From 29cab7a43a3140c8e3c948f1c31ef5e317fd10c8 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 28 Jul 2026 08:59:29 -0400 Subject: [PATCH 1/2] fix: refuse failed rollouts as completed results --- openadapt_evals/action_envelope.py | 171 +++++ openadapt_evals/adapters/__init__.py | 4 + openadapt_evals/adapters/base.py | 164 +++- openadapt_evals/adapters/local/adapter.py | 302 ++++++-- openadapt_evals/adapters/rl_env.py | 309 +++++--- openadapt_evals/adapters/verl_env.py | 90 ++- openadapt_evals/adapters/waa/live.py | 721 +++++++++++++----- openadapt_evals/adapters/waa/mock.py | 82 +- openadapt_evals/agents/api_agent.py | 129 +++- openadapt_evals/agents/demo_executor.py | 45 +- .../agents/planner_grounder_agent.py | 189 ++++- openadapt_evals/analysis/trace_analyzer.py | 102 ++- openadapt_evals/benchmarks/cli.py | 65 +- openadapt_evals/benchmarks/data_collection.py | 43 +- openadapt_evals/benchmarks/pool_viewer.py | 141 +++- openadapt_evals/benchmarks/runner.py | 251 ++++-- openadapt_evals/benchmarks/viewer.py | 134 +++- openadapt_evals/demo_controller.py | 159 +++- openadapt_evals/errors.py | 69 ++ openadapt_evals/evaluation/client.py | 156 +++- openadapt_evals/evaluation/metrics.py | 32 +- openadapt_evals/evaluation/receipts.py | 123 +++ openadapt_evals/integrations/wandb_logger.py | 140 +++- openadapt_evals/server/evaluate_endpoint.py | 399 +++++++--- openadapt_evals/task_config.py | 294 +++++-- openadapt_evals/telemetry.py | 6 + openadapt_evals/training/standalone/config.py | 22 + openadapt_evals/training/standalone/prompt.py | 185 +++-- openadapt_evals/training/standalone/reward.py | 92 ++- .../training/standalone/trainer.py | 206 +++-- openadapt_evals/training/trl_rollout.py | 277 ++++--- openadapt_evals/training/trl_wrapper.py | 13 +- openadapt_evals/vlm_evaluator.py | 111 ++- tests/test_action_envelopes.py | 71 ++ tests/test_agent_failure_contract.py | 5 + tests/test_api_agent_parsing.py | 38 + tests/test_areal_workflow.py | 37 +- tests/test_benchmark_result_contract.py | 230 ++++++ tests/test_demo_controller.py | 170 +++++ tests/test_demo_executor_e2e.py | 28 +- tests/test_demo_executor_http_grounder.py | 8 +- tests/test_dense_rewards.py | 56 ++ tests/test_evaluate_endpoint.py | 41 +- tests/test_evaluator_boundaries.py | 431 +++++++++++ ...est_evaluator_client_failure_visibility.py | 212 +++++ tests/test_false_success_sweep.py | 99 +++ tests/test_local_adapter.py | 196 ++++- tests/test_metrics.py | 26 + tests/test_milestone_screenshot_eval.py | 23 +- tests/test_openenv.py | 1 + tests/test_pixel_action.py | 720 +++++++++++++++-- tests/test_planner_grounder_agent.py | 133 ++++ tests/test_result_artifact_reporting.py | 211 +++++ tests/test_rl_env.py | 233 ++++++ tests/test_runner.py | 409 +++++++++- tests/test_standalone_trainer.py | 356 ++++++++- tests/test_task_config.py | 70 +- tests/test_task_config_receipts.py | 193 +++++ tests/test_trace_analysis.py | 6 +- tests/test_trl_robustness.py | 178 ++--- tests/test_trl_rollout.py | 129 +++- tests/test_verl_env.py | 90 ++- tests/test_waa.py | 119 +++ tests/test_wandb_result_surfaces.py | 107 +++ 64 files changed, 8409 insertions(+), 1443 deletions(-) create mode 100644 openadapt_evals/action_envelope.py create mode 100644 openadapt_evals/errors.py create mode 100644 openadapt_evals/evaluation/receipts.py create mode 100644 tests/test_action_envelopes.py create mode 100644 tests/test_benchmark_result_contract.py create mode 100644 tests/test_evaluator_boundaries.py create mode 100644 tests/test_result_artifact_reporting.py create mode 100644 tests/test_task_config_receipts.py create mode 100644 tests/test_wandb_result_surfaces.py diff --git a/openadapt_evals/action_envelope.py b/openadapt_evals/action_envelope.py new file mode 100644 index 0000000..e7318d1 --- /dev/null +++ b/openadapt_evals/action_envelope.py @@ -0,0 +1,171 @@ +"""Strict parsing helpers for one-action model output envelopes.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from openadapt_evals.errors import ActionParseError + +_THOUGHT_ACTION = re.compile( + r"Thought:\s*[^\r\n]+\r?\nAction:\s*(?P.+)", + re.IGNORECASE | re.DOTALL, +) +_THINK_ACTION = re.compile( + r".+?\s*(?:Action:\s*)?(?P.+)", + re.IGNORECASE | re.DOTALL, +) +_ACTION_PREFIX = re.compile(r"Action:\s*(?P.+)", re.IGNORECASE | re.DOTALL) +_ACTION_CALL = re.compile( + r"(?P[A-Za-z_][A-Za-z0-9_]*)\((?P.*)\)", + re.DOTALL, +) +_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +def parse_single_json_object(text: str) -> dict[str, Any] | None: + """Return one exact JSON object, or None when the envelope is not JSON.""" + stripped = text.strip() + if stripped.startswith("```"): + match = re.fullmatch( + r"```(?:json)?\s*(?P.*?)\s*```", + stripped, + re.IGNORECASE | re.DOTALL, + ) + if match is None: + raise ActionParseError("Malformed fenced JSON action envelope") + stripped = match.group("body").strip() + elif not stripped.startswith("{"): + return None + + def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ActionParseError(f"Duplicate JSON action field: {key}") + result[key] = value + return result + + try: + value = json.loads(stripped, object_pairs_hook=reject_duplicates) + except ActionParseError: + raise + except json.JSONDecodeError as exc: + raise ActionParseError("Malformed or non-exclusive JSON action envelope") from exc + if not isinstance(value, dict): + raise ActionParseError("JSON action must be one object") + return value + + +def parse_single_dsl_action( + text: str, + *, + allowed_commands: set[str], +) -> tuple[str, dict[str, str]]: + """Parse one exact optional-thought plus action DSL envelope.""" + envelope = text.strip() + for pattern in (_THOUGHT_ACTION, _THINK_ACTION, _ACTION_PREFIX): + match = pattern.fullmatch(envelope) + if match is not None: + envelope = match.group("action").strip() + break + + action_match = _ACTION_CALL.fullmatch(envelope) + if action_match is None: + raise ActionParseError("Output must contain exactly one complete action") + + command = action_match.group("command").upper() + if command not in allowed_commands: + raise ActionParseError(f"Unsupported action command: {command!r}") + arguments = _parse_keyword_arguments(action_match.group("arguments")) + return command, arguments + + +def require_exact_fields( + arguments: dict[str, str], + required: set[str], + command: str, +) -> None: + """Require the exact action-specific keyword set.""" + missing = required - arguments.keys() + extra = arguments.keys() - required + if missing: + raise ActionParseError( + f"{command} requires {', '.join(sorted(missing))}" + ) + if extra: + raise ActionParseError( + f"{command} has unsupported fields: {', '.join(sorted(extra))}" + ) + + +def _parse_keyword_arguments(arguments: str) -> dict[str, str]: + """Parse exact comma-separated keyword arguments and reject duplicates.""" + if not arguments.strip(): + return {} + + result: dict[str, str] = {} + position = 0 + length = len(arguments) + while position < length: + while position < length and arguments[position].isspace(): + position += 1 + key_match = _KEY.match(arguments, position) + if key_match is None: + raise ActionParseError("Action arguments must be named fields") + key = key_match.group() + position = key_match.end() + while position < length and arguments[position].isspace(): + position += 1 + if position >= length or arguments[position] != "=": + raise ActionParseError(f"Action field {key!r} is missing '='") + position += 1 + while position < length and arguments[position].isspace(): + position += 1 + if position >= length: + raise ActionParseError(f"Action field {key!r} has no value") + + if arguments[position] in ('"', "'"): + quote = arguments[position] + position += 1 + value_start = position + escaped = False + while position < length: + char = arguments[position] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == quote: + break + position += 1 + if position >= length: + raise ActionParseError(f"Action field {key!r} has an open quote") + value = arguments[value_start:position] + position += 1 + else: + value_start = position + while position < length and arguments[position] not in ",()": + if arguments[position].isspace(): + break + position += 1 + value = arguments[value_start:position] + if not value: + raise ActionParseError(f"Action field {key!r} has no value") + + if key in result: + raise ActionParseError(f"Duplicate action field: {key}") + result[key] = value + + while position < length and arguments[position].isspace(): + position += 1 + if position == length: + break + if arguments[position] != ",": + raise ActionParseError("Action fields must be comma-separated") + position += 1 + if not arguments[position:].strip(): + raise ActionParseError("Action arguments cannot end with a comma") + + return result diff --git a/openadapt_evals/adapters/__init__.py b/openadapt_evals/adapters/__init__.py index 2e49389..06f9ee8 100644 --- a/openadapt_evals/adapters/__init__.py +++ b/openadapt_evals/adapters/__init__.py @@ -34,6 +34,7 @@ """ from openadapt_evals.adapters.base import ( + BENCHMARK_ERROR_TYPES, BenchmarkAction, BenchmarkAdapter, BenchmarkObservation, @@ -42,6 +43,7 @@ EvaluationUnavailableError, StaticDatasetAdapter, UIElement, + normalize_benchmark_result, ) from openadapt_evals.adapters.local import LocalAdapter from openadapt_evals.adapters.rl_env import ( @@ -73,6 +75,8 @@ "BenchmarkObservation", "BenchmarkAction", "BenchmarkResult", + "BENCHMARK_ERROR_TYPES", + "normalize_benchmark_result", "EvaluationUnavailableError", "StaticDatasetAdapter", "UIElement", diff --git a/openadapt_evals/adapters/base.py b/openadapt_evals/adapters/base.py index 3d6d19d..061c370 100644 --- a/openadapt_evals/adapters/base.py +++ b/openadapt_evals/adapters/base.py @@ -13,8 +13,11 @@ from __future__ import annotations +import math from abc import ABC, abstractmethod -from dataclasses import dataclass, field +from collections.abc import Mapping +from dataclasses import dataclass, field, replace +from numbers import Real from typing import TYPE_CHECKING, Any, Iterator # Canonical Benchmark* task/observation/action types now live in the shared @@ -80,6 +83,165 @@ class BenchmarkResult: total_time_seconds: float = 0.0 +BENCHMARK_ERROR_TYPES = frozenset({None, "agent", "infrastructure", "evaluation"}) +UNSCORED_BENCHMARK_ERROR_TYPES = frozenset({"infrastructure", "evaluation"}) + + +def normalize_benchmark_result( + result: object, + *, + expected_task_id: str | None = None, + context: str = "benchmark result", +) -> BenchmarkResult: + """Return a valid result or an explicit unscored evaluation failure. + + Adapters are plugin boundaries. A malformed adapter result must not reach a + done gate, a report, or aggregate metrics as a measured outcome. Partial + scores are valid, so coherence only fixes the unambiguous endpoint cases: + success cannot have a zero score and failure cannot have a full score. + """ + issues: list[str] = [] + + if not isinstance(result, BenchmarkResult): + issues.append(f"expected BenchmarkResult, got {type(result).__name__}") + candidate: BenchmarkResult | None = None + else: + candidate = result + + task_id = expected_task_id + if candidate is not None: + if not isinstance(candidate.task_id, str) or not candidate.task_id: + issues.append("task_id must be a non-empty string") + elif expected_task_id is not None and candidate.task_id != expected_task_id: + issues.append( + f"task_id {candidate.task_id!r} does not match {expected_task_id!r}" + ) + elif task_id is None: + task_id = candidate.task_id + + if type(candidate.success) is not bool: + issues.append("success must be a bool") + + score_valid = ( + not isinstance(candidate.score, bool) + and isinstance(candidate.score, Real) + and math.isfinite(float(candidate.score)) + and 0.0 <= float(candidate.score) <= 1.0 + ) + if not score_valid: + issues.append("score must be a finite number in [0, 1]") + + error_type_valid = ( + candidate.error_type is None + or ( + isinstance(candidate.error_type, str) + and candidate.error_type in BENCHMARK_ERROR_TYPES + ) + ) + if not error_type_valid: + issues.append(f"unknown error_type {candidate.error_type!r}") + + if ( + isinstance(candidate.num_steps, bool) + or not isinstance(candidate.num_steps, int) + or candidate.num_steps < 0 + ): + issues.append("num_steps must be a non-negative integer") + + if ( + isinstance(candidate.total_time_seconds, bool) + or not isinstance(candidate.total_time_seconds, Real) + or not math.isfinite(float(candidate.total_time_seconds)) + or float(candidate.total_time_seconds) < 0.0 + ): + issues.append("total_time_seconds must be a finite non-negative number") + + if candidate.error is not None and not isinstance(candidate.error, str): + issues.append("error must be a string or null") + if candidate.reason is not None and not isinstance(candidate.reason, str): + issues.append("reason must be a string or null") + + if ( + type(candidate.success) is bool + and score_valid + and error_type_valid + ): + score = float(candidate.score) + if candidate.error_type is None and candidate.success and score == 0.0: + issues.append("a successful result cannot have score=0.0") + elif ( + candidate.error_type is None + and not candidate.success + and score == 1.0 + ): + issues.append("a failed result cannot have score=1.0") + + if not issues and candidate is not None: + if candidate.error_type is not None and ( + candidate.success or float(candidate.score) != 0.0 + ): + return replace(candidate, success=False, score=0.0) + return candidate + + safe_task_id = task_id if isinstance(task_id, str) and task_id else "unknown" + message = f"Malformed {context}: {'; '.join(issues)}" + return BenchmarkResult( + task_id=safe_task_id, + success=False, + score=0.0, + error=message, + reason=message, + error_type="evaluation", + ) + + +def normalize_benchmark_result_artifact( + record: object, + *, + expected_task_id: str | None = None, + context: str = "benchmark result artifact", +) -> BenchmarkResult: + """Parse one persisted result row through the runtime result contract. + + JSON decoders preserve strings such as ``"false"``. Constructing reports + directly from their Python truthiness turns those strings into successful + outcomes. This parser keeps raw artifact types intact until + :func:`normalize_benchmark_result` validates them. + """ + if not isinstance(record, Mapping): + return normalize_benchmark_result( + record, + expected_task_id=expected_task_id, + context=context, + ) + + raw_steps = record.get("num_steps", record.get("steps", 0)) + num_steps = len(raw_steps) if isinstance(raw_steps, list) else raw_steps + candidate = BenchmarkResult( + task_id=record.get("task_id", expected_task_id), # type: ignore[arg-type] + success=record.get("success"), # type: ignore[arg-type] + score=record.get("score"), # type: ignore[arg-type] + num_steps=num_steps, # type: ignore[arg-type] + error=record.get("error"), # type: ignore[arg-type] + reason=record.get("reason"), # type: ignore[arg-type] + error_type=record.get("error_type"), # type: ignore[arg-type] + total_time_seconds=record.get( + "total_time_seconds", + record.get("elapsed_seconds", 0.0), + ), # type: ignore[arg-type] + ) + return normalize_benchmark_result( + candidate, + expected_task_id=expected_task_id, + context=context, + ) + + +def benchmark_result_is_scored(result: BenchmarkResult) -> bool: + """Return whether a result belongs in measured-outcome denominators.""" + return result.error_type not in UNSCORED_BENCHMARK_ERROR_TYPES + + @dataclass class UIElement: """Normalized UI element for cross-platform use. diff --git a/openadapt_evals/adapters/local/adapter.py b/openadapt_evals/adapters/local/adapter.py index 24c622f..0a757f2 100644 --- a/openadapt_evals/adapters/local/adapter.py +++ b/openadapt_evals/adapters/local/adapter.py @@ -24,6 +24,7 @@ import io import logging +import math import platform import time from typing import Any @@ -35,44 +36,80 @@ BenchmarkResult, BenchmarkTask, ) +from openadapt_evals.errors import ( + ActionDeliveredObservationError, + ActionDeliveryUncertainError, + ActionExecutionError, +) logger = logging.getLogger(__name__) -def _get_display_scale_factor() -> float: - """Detect the HiDPI / Retina scale factor for the primary display. - - On macOS Retina displays, ``mss`` captures at physical pixel resolution - while ``pynput`` operates in logical points. This function returns the - ratio (physical / logical) so callers can convert between the two - coordinate systems. - - Returns: - Scale factor (e.g. 2.0 on Retina, 1.0 on standard displays). - """ +def _get_input_monitor_geometry( + monitor_index: int, + capture_monitor: dict[str, int], +) -> dict[str, float]: + """Return the selected monitor bounds in the input controller's space.""" if platform.system() != "Darwin": - return 1.0 - - try: - import mss - - with mss.mss() as sct: - monitor = sct.monitors[1] # primary monitor - physical_width = monitor["width"] + return { + name: float(capture_monitor[name]) + for name in ("left", "top", "width", "height") + } - # Quartz gives us the logical width - from Quartz import CGDisplayBounds, CGMainDisplayID # type: ignore[import-not-found] + if monitor_index <= 0: + raise ActionExecutionError( + "Local pointer actions cannot safely use the combined macOS monitor capture" + ) - main_display = CGMainDisplayID() - bounds = CGDisplayBounds(main_display) - logical_width = int(bounds.size.width) + try: + import Quartz # type: ignore[import-not-found] - if logical_width > 0: - return physical_width / logical_width - except Exception: - logger.debug("Could not detect display scale factor, defaulting to 1.0") + error, display_ids, display_count = Quartz.CGGetActiveDisplayList( + 32, None, None + ) + if error != Quartz.kCGErrorSuccess: + raise RuntimeError(f"CGGetActiveDisplayList returned {error}") + active_displays = list(display_ids)[: int(display_count)] + display_id = active_displays[monitor_index - 1] + rotation = float(Quartz.CGDisplayRotation(display_id)) % 360.0 + if not math.isclose(rotation, 0.0, abs_tol=1e-6): + raise RuntimeError( + f"rotated macOS display ({rotation:g} degrees) is not supported" + ) + bounds = Quartz.CGDisplayBounds(display_id) + logical = { + "left": float(bounds.origin.x), + "top": float(bounds.origin.y), + "width": float(bounds.size.width), + "height": float(bounds.size.height), + } - return 1.0 + # MSS and Quartz both enumerate CGGetActiveDisplayList in this order. + # Check that the selected capture still resembles either the logical + # or backing-pixel size. This rejects a stale or incompatible mapping. + capture_size = ( + int(capture_monitor["width"]), + int(capture_monitor["height"]), + ) + logical_size = (round(logical["width"]), round(logical["height"])) + backing_size = ( + int(Quartz.CGDisplayPixelsWide(display_id)), + int(Quartz.CGDisplayPixelsHigh(display_id)), + ) + valid_sizes = { + logical_size, + backing_size, + } + if capture_size not in valid_sizes: + raise RuntimeError( + f"capture size {capture_size!r} does not match selected display " + f"logical/backing sizes {logical_size!r}/{backing_size!r}" + ) + return logical + except Exception as exc: + raise ActionExecutionError( + "Local capture could not bind the selected macOS display to input geometry" + ) from exc class LocalAdapter(BenchmarkAdapter): @@ -95,9 +132,12 @@ def __init__( ): self._action_delay = action_delay self._monitor_index = monitor_index - self._scale: float | None = None self._current_task: BenchmarkTask | None = None self._step_count = 0 + self._last_viewport: tuple[int, int] | None = None + self._capture_origin: tuple[float, float] | None = None + self._capture_scale: tuple[float, float] | None = None + self._capture_pixel_size: tuple[int, int] | None = None # ------------------------------------------------------------------ # BenchmarkAdapter properties @@ -115,18 +155,78 @@ def benchmark_type(self) -> str: # Internal helpers # ------------------------------------------------------------------ - def _ensure_scale(self) -> float: - """Lazily compute and cache the display scale factor.""" - if self._scale is None: - self._scale = _get_display_scale_factor() - if self._scale != 1.0: - logger.info("HiDPI detected: scale factor = %.1f", self._scale) - return self._scale + def _record_capture_geometry( + self, + capture_monitor: dict[str, int], + pixel_size: tuple[int, int], + input_monitor: dict[str, float] | None = None, + ) -> None: + """Bind screenshot pixels to the captured monitor's global coordinates. + + Screenshot bounds and input-controller bounds are independent on + platforms such as macOS Retina. The scale comes from the exact image + and the selected display's OS input bounds, never from the primary + display or from the MSS dimensions alone. + """ + monitor = input_monitor or { + name: float(capture_monitor[name]) + for name in ("left", "top", "width", "height") + } + try: + left = float(monitor["left"]) + top = float(monitor["top"]) + logical_width = float(monitor["width"]) + logical_height = float(monitor["height"]) + pixel_width = int(pixel_size[0]) + pixel_height = int(pixel_size[1]) + except (KeyError, TypeError, ValueError, IndexError) as exc: + raise ActionExecutionError( + "Local capture did not provide usable monitor geometry" + ) from exc + + values = (left, top, logical_width, logical_height) + if not all(math.isfinite(value) for value in values): + raise ActionExecutionError( + "Local capture provided non-finite monitor geometry" + ) + if logical_width <= 0 or logical_height <= 0: + raise ActionExecutionError( + "Local capture provided non-positive monitor dimensions" + ) + if pixel_width <= 0 or pixel_height <= 0: + raise ActionExecutionError( + "Local capture provided non-positive screenshot dimensions" + ) + + scale_x = pixel_width / logical_width + scale_y = pixel_height / logical_height + if not math.isfinite(scale_x) or not math.isfinite(scale_y): + raise ActionExecutionError("Local capture scale is not finite") + + self._capture_origin = (left, top) + self._capture_scale = (scale_x, scale_y) + self._capture_pixel_size = (pixel_width, pixel_height) + self._last_viewport = self._capture_pixel_size + + def _require_capture_geometry(self) -> None: + if ( + self._capture_origin is None + or self._capture_scale is None + or self._capture_pixel_size is None + or self._last_viewport != self._capture_pixel_size + ): + raise ActionExecutionError( + "Local pointer action requires fresh, matching capture geometry" + ) def _to_logical(self, x: float, y: float) -> tuple[float, float]: - """Convert physical pixel coordinates to logical points.""" - s = self._ensure_scale() - return x / s, y / s + """Map monitor-relative screenshot pixels to global input coordinates.""" + self._require_capture_geometry() + assert self._capture_origin is not None + assert self._capture_scale is not None + origin_x, origin_y = self._capture_origin + scale_x, scale_y = self._capture_scale + return origin_x + x / scale_x, origin_y + y / scale_y # ------------------------------------------------------------------ # Observation @@ -153,9 +253,15 @@ def observe(self) -> BenchmarkObservation: pil_img.save(buf, format="PNG") png_bytes = buf.getvalue() + pixel_size = (int(img.size.width), int(img.size.height)) + input_monitor = _get_input_monitor_geometry( + self._monitor_index, + monitor, + ) + self._record_capture_geometry(monitor, pixel_size, input_monitor) return BenchmarkObservation( screenshot=png_bytes, - viewport=(monitor["width"], monitor["height"]), + viewport=self._last_viewport, ) # ------------------------------------------------------------------ @@ -193,10 +299,25 @@ def reset(self, task: BenchmarkTask) -> BenchmarkObservation: Returns: Initial observation (screenshot of current screen). """ - self._current_task = task + # A failed fresh observation must invalidate the prior task and its + # coordinate transform before any later input can be delivered. + self._current_task = None self._step_count = 0 + self._last_viewport = None + self._capture_origin = None + self._capture_scale = None + self._capture_pixel_size = None logger.info("LocalAdapter reset for task: %s", task.task_id) - return self.observe() + try: + observation = self.observe() + except Exception: + self._last_viewport = None + self._capture_origin = None + self._capture_scale = None + self._capture_pixel_size = None + raise + self._current_task = task + return observation def step( self, action: BenchmarkAction @@ -219,19 +340,32 @@ def step( Returns: Tuple of (observation, done, info). """ - self._step_count += 1 + if self._current_task is None and action.type not in {"done", "error"}: + raise ActionExecutionError("Call reset() before a local action") + + self._validate_action(action) done = action.type in ("done", "error") try: self._execute_action(action) + except ActionExecutionError: + raise except Exception as e: logger.error("Failed to execute action %s: %s", action.type, e) - return self.observe(), True, {"step": self._step_count, "error": str(e)} + raise ActionDeliveryUncertainError( + f"Failed to execute local action {action.type!r}: {e}" + ) from e + self._step_count += 1 if self._action_delay > 0 and not done: time.sleep(self._action_delay) - obs = self.observe() + try: + obs = self.observe() + except Exception as exc: + raise ActionDeliveredObservationError( + f"Local action {action.type!r} was delivered, but observation failed: {exc}" + ) from exc return obs, done, {"step": self._step_count} def evaluate(self, task: BenchmarkTask) -> BenchmarkResult: @@ -249,6 +383,7 @@ def evaluate(self, task: BenchmarkTask) -> BenchmarkResult: success=False, score=0.0, num_steps=self._step_count, + error_type="evaluation", reason="LocalAdapter does not implement built-in evaluation. " "Use an external evaluator or VLM judge.", ) @@ -257,6 +392,79 @@ def evaluate(self, task: BenchmarkTask) -> BenchmarkResult: # Action execution # ------------------------------------------------------------------ + def _validate_coordinate( + self, value: float | None, field: str, action_type: str + ) -> None: + if value is None: + raise ActionExecutionError( + f"Local {action_type!r} action requires {field}" + ) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ActionExecutionError( + f"Local {action_type!r} action requires numeric {field}" + ) + if not math.isfinite(float(value)): + raise ActionExecutionError( + f"Local {action_type!r} action requires finite {field}" + ) + if value < 0: + raise ActionExecutionError( + f"Local {action_type!r} action requires non-negative {field}" + ) + self._require_capture_geometry() + assert self._last_viewport is not None + width, height = self._last_viewport + limit = width if field in {"x", "end_x"} else height + if value >= limit: + raise ActionExecutionError( + f"Local {action_type!r} action {field} is outside the captured viewport" + ) + + def _validate_action(self, action: BenchmarkAction) -> None: + """Reject incomplete actions before input delivery or step accounting.""" + supported = { + "click", + "double_click", + "right_click", + "type", + "key", + "scroll", + "drag", + "done", + "wait", + "error", + } + if action.type not in supported: + raise ActionExecutionError(f"Unsupported local action type: {action.type!r}") + + if action.type in {"click", "double_click", "right_click"}: + self._validate_coordinate(action.x, "x", action.type) + self._validate_coordinate(action.y, "y", action.type) + elif action.type == "type" and action.text is None: + raise ActionExecutionError("Local 'type' action requires text") + elif action.type == "key" and not action.key: + raise ActionExecutionError("Local 'key' action requires key") + elif action.type == "scroll": + if action.scroll_direction not in {"up", "down", "left", "right"}: + raise ActionExecutionError( + "Local 'scroll' action requires direction up, down, left, or right" + ) + if action.scroll_amount is not None: + if ( + isinstance(action.scroll_amount, bool) + or not isinstance(action.scroll_amount, (int, float)) + or not math.isfinite(float(action.scroll_amount)) + or action.scroll_amount <= 0 + ): + raise ActionExecutionError( + "Local 'scroll' action requires a positive finite amount" + ) + elif action.type == "drag": + self._validate_coordinate(action.x, "x", action.type) + self._validate_coordinate(action.y, "y", action.type) + self._validate_coordinate(action.end_x, "end_x", action.type) + self._validate_coordinate(action.end_y, "end_y", action.type) + def _execute_action(self, action: BenchmarkAction) -> None: """Dispatch and execute a single action via pynput.""" action_type = action.type @@ -274,7 +482,7 @@ def _execute_action(self, action: BenchmarkAction) -> None: elif action_type in ("done", "wait", "error"): pass # No-op else: - logger.warning("Unknown action type: %s", action_type) + raise ActionExecutionError(f"Unsupported local action type: {action_type!r}") def _do_click(self, action: BenchmarkAction) -> None: """Execute a mouse click action.""" diff --git a/openadapt_evals/adapters/rl_env.py b/openadapt_evals/adapters/rl_env.py index dadd7bc..338c1c8 100644 --- a/openadapt_evals/adapters/rl_env.py +++ b/openadapt_evals/adapters/rl_env.py @@ -57,6 +57,7 @@ import hashlib import logging +import math from dataclasses import dataclass, field from typing import Any, Callable @@ -67,6 +68,7 @@ BenchmarkResult, BenchmarkTask, EvaluationUnavailableError, + normalize_benchmark_result, ) # Avoid circular import — TaskConfig imported lazily. @@ -80,6 +82,32 @@ from openadapt_evals.task_config import TaskConfig + +def _fraction_to_pixel(value: float, size: int, name: str) -> int: + """Map an inclusive normalized coordinate to one valid pixel index.""" + if size <= 0: + raise ValueError(f"{name} screen dimension must be positive") + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + raise ValueError(f"{name}_frac must be a finite value between 0 and 1") + return min(int(value * size), size - 1) + + +def _measured_score(result: BenchmarkResult) -> float: + """Return one finite benchmark score in the declared unit interval.""" + score = result.score + if ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(score) + or not 0.0 <= score <= 1.0 + ): + raise EvaluationUnavailableError( + f"task {result.task_id} returned invalid score {score!r}", + error_type="evaluation", + ) + return float(score) + + logger = logging.getLogger(__name__) @@ -151,6 +179,7 @@ def __init__( self._current_task: BenchmarkTask | None = None self._step_count = 0 self._done = False + self._terminal_error = False self._trajectory: list[RolloutStep] = [] self._last_obs: BenchmarkObservation | None = None @@ -159,6 +188,7 @@ def __init__( # This is the "high-water mark" — once a milestone passes, it # stays passed even if the transient state disappears. self._milestone_passed: set[int] = set() + self._milestone_evaluated: set[int] = set() @property def adapter(self) -> BenchmarkAdapter: @@ -182,21 +212,36 @@ def trajectory(self) -> list[RolloutStep]: @property def screen_size(self) -> tuple[int, int]: - """Actual screen dimensions (width, height) from the last observation. + """Measured screen dimensions from the last observation. - Falls back to the adapter's configured size if no observation has been - captured yet. + Coordinate conversion must use the frame that the agent observed. A + configured or conventional desktop size is not actuation evidence. """ - if self._last_obs is not None and self._last_obs.viewport is not None: - return self._last_obs.viewport - if hasattr(self._adapter, "screen_size"): - return self._adapter.screen_size - config = getattr(self._adapter, "config", None) - if config is not None: - width = getattr(config, "screen_width", 1920) - height = getattr(config, "screen_height", 1200) - return (width, height) - return (1920, 1200) + if self._last_obs is None or self._last_obs.viewport is None: + raise RuntimeError( + "No measured viewport is available; capture an observation before " + "converting coordinates" + ) + viewport = self._last_obs.viewport + if len(viewport) != 2: + raise ValueError(f"Invalid measured viewport: {viewport!r}") + width, height = viewport + if ( + isinstance(width, bool) + or isinstance(height, bool) + or not isinstance(width, (int, float)) + or not isinstance(height, (int, float)) + or not math.isfinite(float(width)) + or not math.isfinite(float(height)) + or int(width) != width + or int(height) != height + ): + raise ValueError(f"Invalid measured viewport: {viewport!r}") + if width <= 0 or height <= 0: + raise ValueError( + f"Measured viewport dimension must be positive: {viewport!r}" + ) + return int(width), int(height) def load_task_config(self, task_config: TaskConfig) -> None: """Set a TaskConfig for dense reward evaluation. @@ -232,6 +277,17 @@ def reset(self, config: ResetConfig | None = None) -> BenchmarkObservation: ValueError: If no task_id is provided and no default was set. RuntimeError: If the adapter cannot connect to the WAA server. """ + # Invalidate the prior episode before any operation that can fail. A + # failed reset must not leave an old task, frame, or trajectory usable. + self._current_task = None + self._step_count = 0 + self._done = False + self._terminal_error = False + self._trajectory = [] + self._last_obs = None + self._milestone_passed = set() + self._milestone_evaluated = set() + config = config or ResetConfig() task_id = config.task_id or self._default_task_id @@ -243,26 +299,23 @@ def reset(self, config: ResetConfig | None = None) -> BenchmarkObservation: # Load the task — prefer TaskConfig if available (avoids server lookup) if self._task_config and self._task_config.id == task_id: - self._current_task = self._task_config.to_benchmark_task() + candidate_task = self._task_config.to_benchmark_task() elif hasattr(self._adapter, "load_task_from_json") and self._task_config: - self._current_task = self._adapter.load_task_from_json( + candidate_task = self._adapter.load_task_from_json( task_id, self._task_config.to_waa_config() ) else: - self._current_task = self._adapter.load_task(task_id) - obs = self._adapter.reset(self._current_task) + candidate_task = self._adapter.load_task(task_id) + obs = self._adapter.reset(candidate_task) - # Reset episode state - self._step_count = 0 - self._done = False - self._trajectory = [] + # Publish the task and observation only after reset succeeds. + self._current_task = candidate_task self._last_obs = obs - self._milestone_passed = set() logger.info( "Environment reset: task=%s, instruction=%s", task_id, - self._current_task.instruction[:80], + candidate_task.instruction[:80], ) return obs @@ -285,9 +338,7 @@ def step(self, action: BenchmarkAction) -> RolloutStep: raise RuntimeError("Call reset() before step().") if self._done: - raise RuntimeError( - "Episode is done. Call reset() to start a new episode." - ) + raise RuntimeError("Episode is done. Call reset() to start a new episode.") obs, done, info = self._adapter.step(action) self._step_count += 1 @@ -296,14 +347,23 @@ def step(self, action: BenchmarkAction) -> RolloutStep: # Check for terminal actions if action.type in ("done", "error"): done = True + if action.type == "error": + self._terminal_error = True self._done = done info["step"] = self._step_count # Optional per-step evaluation for RL training loops - if self._evaluate_every_step and self._current_task is not None: + if self._terminal_error: + info["evaluation_error"] = "agent returned a terminal error action" + info["evaluation_error_type"] = "agent" + elif self._evaluate_every_step and self._current_task is not None: try: - result = self._adapter.evaluate(self._current_task) + result = normalize_benchmark_result( + self._adapter.evaluate(self._current_task), + expected_task_id=self._current_task.task_id, + context="per-step adapter evaluation", + ) if result.error_type: # Not scored. Recording a 0.0 here would look identical to # a step the agent genuinely failed. @@ -312,7 +372,7 @@ def step(self, action: BenchmarkAction) -> RolloutStep: ) info["evaluation_error_type"] = result.error_type else: - info["evaluation_score"] = result.score + info["evaluation_score"] = _measured_score(result) info["evaluation_success"] = result.success except Exception as e: logger.warning("Per-step evaluation failed at step %d: %s", self._step_count, e) @@ -362,32 +422,45 @@ def pixel_action( if self._current_task is None: raise RuntimeError("Call reset() before pixel_action().") if self._done: - raise RuntimeError( - "Episode is done. Call reset() to start a new episode." - ) + raise RuntimeError("Episode is done. Call reset() to start a new episode.") + + fractional_size = None + dispatch_x = x + dispatch_y = y + if x_frac is not None or y_frac is not None: + fractional_size = self.screen_size + width, height = fractional_size + if x_frac is not None: + dispatch_x = _fraction_to_pixel(x_frac, width, "x") + if y_frac is not None: + dispatch_y = _fraction_to_pixel(y_frac, height, "y") obs, done, info = self._adapter.pixel_action( - x=x, - y=y, + x=dispatch_x, + y=dispatch_y, action_type=action_type, text=text, key=key, - x_frac=x_frac, - y_frac=y_frac, + x_frac=None, + y_frac=None, ) self._step_count += 1 self._last_obs = obs self._done = done + if action_type == "error": + done = True + self._done = True + self._terminal_error = True + info["evaluation_error"] = "agent returned a terminal error action" + info["evaluation_error_type"] = "agent" info["step"] = self._step_count # Reconstruct the action for the trajectory record resolved_x = x resolved_y = y - if x_frac is not None or y_frac is not None: - if hasattr(self._adapter, "screen_size"): - w, h = self._adapter.screen_size - resolved_x = int((x_frac or 0.0) * w) - resolved_y = int((y_frac or 0.0) * h) + if fractional_size is not None: + resolved_x = dispatch_x + resolved_y = dispatch_y action = BenchmarkAction( type=action_type, x=resolved_x, y=resolved_y, text=text, key=key @@ -404,19 +477,11 @@ def pixel_action( # Fallback: convert fracs to pixels manually and use step() if x_frac is not None or y_frac is not None: - # Try to get screen size from adapter - if hasattr(self._adapter, "screen_size"): - w, h = self._adapter.screen_size - else: - w, h = 1920, 1200 # WAA default - logger.warning( - "Adapter has no screen_size property; " - "using default %dx%d for frac conversion", - w, - h, - ) - x = int((x_frac or 0.0) * w) - y = int((y_frac or 0.0) * h) + w, h = self.screen_size + if x_frac is not None: + x = _fraction_to_pixel(x_frac, w, "x") + if y_frac is not None: + y = _fraction_to_pixel(y_frac, h, "y") action = BenchmarkAction(type=action_type, x=x, y=y, text=text, key=key) return self.step(action) @@ -486,12 +551,33 @@ def evaluate_result(self) -> BenchmarkResult: if self._current_task is None: raise RuntimeError("Call reset() before evaluate().") - result = self._adapter.evaluate(self._current_task) + if self._terminal_error: + result = BenchmarkResult( + task_id=self._current_task.task_id, + success=False, + score=0.0, + num_steps=self._step_count, + error_type="agent", + reason="Agent returned a terminal error action", + ) + if self._trajectory: + self._trajectory[-1].reward = 0.0 + self._trajectory[-1].info["evaluation_error_type"] = "agent" + self._trajectory[-1].info["evaluation_error"] = result.reason + return result + + result = normalize_benchmark_result( + self._adapter.evaluate(self._current_task), + expected_task_id=self._current_task.task_id, + context="adapter evaluation", + ) # Backfill reward on the last trajectory step only when the score is a # real measurement; an infrastructure 0.0 is not a reward signal. if self._trajectory and not result.error_type: - self._trajectory[-1].reward = result.score + self._trajectory[-1].reward = _measured_score(result) + elif not result.error_type: + _measured_score(result) logger.info( "Evaluation: task=%s, success=%s, score=%.2f, error_type=%s", @@ -526,14 +612,14 @@ def evaluate(self) -> float: """ result = self.evaluate_result() - if result.error_type in ("infrastructure", "evaluation"): + if result.error_type: raise EvaluationUnavailableError( f"task {result.task_id} was not scored " f"({result.error_type}): {result.reason or result.error}", error_type=result.error_type, ) - return result.score + return _measured_score(result) def check_milestones_incremental( self, @@ -570,9 +656,7 @@ def check_milestones_incremental( except Exception: screenshot = b"" - server_url = getattr( - getattr(self._adapter, "config", None), "server_url", "" - ) or "" + server_url = getattr(getattr(self._adapter, "config", None), "server_url", "") or "" for i, ms in enumerate(self._task_config.milestones): if i in self._milestone_passed: @@ -580,28 +664,40 @@ def check_milestones_incremental( try: if ms.check.check == "screenshot": from openadapt_evals.vlm_evaluator import vlm_judge + success, _ = vlm_judge(screenshot, ms.check.description or "") if success: self._milestone_passed.add(i) logger.info( "Milestone %d/%d PASSED (high-water): %s", - i + 1, total, ms.name, + i + 1, + total, + ms.name, ) + self._milestone_evaluated.add(i) elif ms.check.check == "command": result = self._task_config._run_vm_command( - ms.check.run or "", server_url, + ms.check.run or "", + server_url, ) if self._task_config._check_match( - result, ms.check.expect or "", ms.check.match, + result, + ms.check.expect or "", + ms.check.match, ): self._milestone_passed.add(i) logger.info( "Milestone %d/%d PASSED (high-water): %s", - i + 1, total, ms.name, + i + 1, + total, + ms.name, ) + self._milestone_evaluated.add(i) except Exception as exc: logger.debug( - "Milestone %d check failed (non-fatal): %s", i, exc, + "Milestone %d check failed (non-fatal): %s", + i, + exc, ) passed = len(self._milestone_passed) @@ -623,6 +719,9 @@ def evaluate_dense(self) -> float: if self._current_task is None: raise RuntimeError("Call reset() before evaluate_dense().") + if self._terminal_error: + return self.evaluate() + # Try milestone evaluation first if self._task_config and self._task_config.milestones: # Take a FRESH screenshot for the final evaluation pass. @@ -637,8 +736,8 @@ def evaluate_dense(self) -> float: ) except Exception as e: logger.warning( - "evaluate_dense: failed to take fresh screenshot, " - "falling back to cached: %s", e, + "evaluate_dense: failed to take fresh screenshot, falling back to cached: %s", + e, ) if self._last_obs and self._last_obs.screenshot: screenshot = self._last_obs.screenshot @@ -669,40 +768,45 @@ def evaluate_dense(self) -> float: # publishes the milestone high-water mark as a task score. binary_unavailable = exc logger.warning( - "evaluate_dense: binary evaluation did not run: %s", exc, + "evaluate_dense: binary evaluation did not run: %s", + exc, ) # If binary eval returned 0.0 (endpoint down or task # failed), try local evaluation via task config checks. local_check_ran = False - if ( - binary_score == 0.0 - and self._task_config.checks - and screenshot - ): - server_url = getattr( - getattr(self._adapter, "config", None), - "server_url", "", - ) or "" + if binary_score == 0.0 and self._task_config.checks and screenshot: + server_url = ( + getattr( + getattr(self._adapter, "config", None), + "server_url", + "", + ) + or "" + ) try: - binary_score = ( - self._task_config.evaluate_checks_local( - screenshot, server_url, - ) + binary_score = self._task_config.evaluate_checks_local( + screenshot, + server_url, ) local_check_ran = True if binary_score > 0: logger.info( - "evaluate_dense: local check fallback " - "returned %.2f", binary_score, + "evaluate_dense: local check fallback returned %.2f", + binary_score, ) except Exception as exc: logger.debug( - "evaluate_dense: local check fallback " - "failed: %s", exc, + "evaluate_dense: local check fallback failed: %s", + exc, ) - if binary_unavailable is not None and not local_check_ran: + milestones_measured = len(self._milestone_evaluated) == total + if ( + binary_unavailable is not None + and not local_check_ran + and not milestones_measured + ): # Nothing scored this task: the binary evaluator could not # run and no local check stood in for it. The milestone # high-water mark is a progress signal, not a task score, @@ -712,10 +816,13 @@ def evaluate_dense(self) -> float: f"task {self._current_task.task_id} was not scored: " f"binary evaluation did not run " f"({binary_unavailable}) and no local check fallback " - f"was available; the milestone high-water mark " - f"({milestone_score:.2f}) is not a task score", + f"was available; only " + f"{len(self._milestone_evaluated)}/{total} milestone " + "contracts were measured", error_type=getattr( - binary_unavailable, "error_type", "infrastructure", + binary_unavailable, + "error_type", + "infrastructure", ), ) from binary_unavailable @@ -733,7 +840,11 @@ def evaluate_dense(self) -> float: logger.info( "Dense evaluation: milestones=%d/%d (%.2f) [high-water], " "binary=%.2f, final=%.2f", - passed, total, milestone_score, binary_score, score, + passed, + total, + milestone_score, + binary_score, + score, ) return score @@ -769,6 +880,9 @@ def collect_rollout( List of RolloutStep objects. The last step's reward contains the evaluation score; all other rewards are 0.0. """ + if isinstance(max_steps, bool) or not isinstance(max_steps, int) or max_steps <= 0: + raise ValueError("max_steps must be a positive integer") + config = ResetConfig(task_id=task_id) obs = self.reset(config) @@ -804,6 +918,19 @@ def collect_rollout( if rollout_step.done: break + if self._terminal_error: + if self._trajectory: + self._trajectory[-1].reward = 0.0 + self._trajectory[-1].info["evaluation_error_type"] = "agent" + self._trajectory[-1].info["evaluation_error"] = ( + "Agent returned a terminal error action" + ) + logger.info( + "Rollout stopped after terminal agent error: %d steps", + self._step_count, + ) + return list(self._trajectory) + # Evaluate and backfill reward — use dense rewards if milestones exist if self._task_config and self._task_config.milestones: score = self.evaluate_dense() diff --git a/openadapt_evals/adapters/verl_env.py b/openadapt_evals/adapters/verl_env.py index 796c3f7..be0640c 100644 --- a/openadapt_evals/adapters/verl_env.py +++ b/openadapt_evals/adapters/verl_env.py @@ -53,12 +53,18 @@ import hashlib import io import logging +import math import re from pathlib import Path from typing import Any +from openadapt_evals.action_envelope import ( + parse_single_dsl_action, + require_exact_fields, +) from openadapt_evals.adapters.base import BenchmarkAction, BenchmarkObservation -from openadapt_evals.adapters.rl_env import RLEnvironment +from openadapt_evals.adapters.rl_env import RLEnvironment, _fraction_to_pixel +from openadapt_evals.errors import ActionParseError logger = logging.getLogger(__name__) @@ -84,6 +90,19 @@ ) +def _required_fraction(kwargs: dict[str, str], key: str, command: str) -> float: + """Read one required finite normalized coordinate.""" + if key not in kwargs: + raise ActionParseError(f"{command} requires {key}") + try: + value = float(kwargs[key]) + except (TypeError, ValueError) as exc: + raise ActionParseError(f"{command} has invalid {key}: {kwargs[key]!r}") from exc + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + raise ActionParseError(f"{command} {key} must be between 0 and 1") + return value + + def _parse_action_str(action_str: str) -> BenchmarkAction: """Parse a VLM action string into a BenchmarkAction. @@ -96,51 +115,54 @@ def _parse_action_str(action_str: str) -> BenchmarkAction: WAIT() DONE() """ - match = _ACTION_PATTERN.search(action_str) - if not match: - logger.warning("Could not parse action: %s", action_str[:100]) - return BenchmarkAction(type="done") - - cmd = match.group(1) - args_str = match.group(2) - - # Extract key=value pairs - kwargs: dict[str, str] = {} - for kv in re.finditer(r'(\w+)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|([^\s,)]+))', args_str): - key = kv.group(1) - kwargs[key] = kv.group(2) if kv.group(2) is not None else kv.group(3) + cmd, kwargs = parse_single_dsl_action( + action_str, + allowed_commands={"CLICK", "TYPE", "SCROLL", "KEY", "WAIT", "DONE", "DRAG"}, + ) if cmd == "CLICK": + require_exact_fields(kwargs, {"x", "y"}, cmd) return BenchmarkAction( type="click", - x=float(kwargs.get("x", 0.5)), - y=float(kwargs.get("y", 0.5)), + x=_required_fraction(kwargs, "x", cmd), + y=_required_fraction(kwargs, "y", cmd), ) elif cmd == "TYPE": - return BenchmarkAction(type="type", text=kwargs.get("text", "")) + require_exact_fields(kwargs, {"text"}, cmd) + return BenchmarkAction(type="type", text=kwargs["text"]) elif cmd == "KEY": - return BenchmarkAction(type="key", key=kwargs.get("key", "enter")) + require_exact_fields(kwargs, {"key"}, cmd) + if not kwargs.get("key"): + raise ActionParseError("KEY requires key") + return BenchmarkAction(type="key", key=kwargs["key"]) elif cmd == "SCROLL": + require_exact_fields(kwargs, {"x", "y", "direction"}, cmd) + direction = kwargs.get("direction") + if direction not in ("up", "down"): + raise ActionParseError("SCROLL requires direction=up or direction=down") return BenchmarkAction( type="scroll", - x=float(kwargs.get("x", 0.5)), - y=float(kwargs.get("y", 0.5)), - scroll_direction=kwargs.get("direction", "down"), + x=_required_fraction(kwargs, "x", cmd), + y=_required_fraction(kwargs, "y", cmd), + scroll_direction=direction, ) elif cmd == "WAIT": + require_exact_fields(kwargs, set(), cmd) return BenchmarkAction(type="wait") elif cmd == "DONE": + require_exact_fields(kwargs, set(), cmd) return BenchmarkAction(type="done") elif cmd == "DRAG": + require_exact_fields(kwargs, {"x", "y", "end_x", "end_y"}, cmd) return BenchmarkAction( type="drag", - x=float(kwargs.get("x", 0.5)), - y=float(kwargs.get("y", 0.5)), - end_x=float(kwargs["end_x"]) if "end_x" in kwargs else None, - end_y=float(kwargs["end_y"]) if "end_y" in kwargs else None, + x=_required_fraction(kwargs, "x", cmd), + y=_required_fraction(kwargs, "y", cmd), + end_x=_required_fraction(kwargs, "end_x", cmd), + end_y=_required_fraction(kwargs, "end_y", cmd), ) else: - return BenchmarkAction(type="done") + raise ActionParseError(f"Unsupported action command: {cmd!r}") def _obs_to_pil(obs: BenchmarkObservation) -> Image.Image | None: @@ -377,13 +399,13 @@ async def step( if self._use_fractional and action.type in ("click", "scroll", "drag"): w, h = env.screen_size if action.x is not None: - action.x = int(action.x * w) + action.x = _fraction_to_pixel(action.x, w, "x") if action.y is not None: - action.y = int(action.y * h) + action.y = _fraction_to_pixel(action.y, h, "y") if action.end_x is not None: - action.end_x = int(action.end_x * w) + action.end_x = _fraction_to_pixel(action.end_x, w, "end_x") if action.end_y is not None: - action.end_y = int(action.end_y * h) + action.end_y = _fraction_to_pixel(action.end_y, h, "end_y") # Execute action in a thread rollout_step = await asyncio.to_thread(env.step, action) @@ -398,12 +420,8 @@ async def step( info["is_action_valid"] = _ACTION_PATTERN.search(action_str) is not None if done and self._evaluate_at_done: - try: - reward = await asyncio.to_thread(env.evaluate) - info["success"] = reward > 0.5 - except Exception: - logger.exception("Evaluation failed") - info["success"] = False + reward = await asyncio.to_thread(env.evaluate) + info["success"] = reward >= 1.0 elif done: info["success"] = False diff --git a/openadapt_evals/adapters/waa/live.py b/openadapt_evals/adapters/waa/live.py index 2080373..afd774e 100644 --- a/openadapt_evals/adapters/waa/live.py +++ b/openadapt_evals/adapters/waa/live.py @@ -27,6 +27,7 @@ import base64 import json import logging +import math import re import time from dataclasses import dataclass @@ -39,6 +40,12 @@ BenchmarkResult, BenchmarkTask, ) +from openadapt_evals.errors import ( + ActionDeliveredObservationError, + ActionDeliveryState, + ActionDeliveryUncertainError, + ActionExecutionError, +) if TYPE_CHECKING: # Both are type-only: `requests` and Pillow are imported inside the method @@ -52,6 +59,15 @@ logger = logging.getLogger(__name__) +def _fraction_to_pixel(value: float, size: int, name: str) -> int: + """Map an inclusive normalized coordinate to one valid pixel index.""" + if size <= 0: + raise ValueError(f"{name} screen dimension must be positive") + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + raise ValueError(f"{name}_frac must be a finite value between 0 and 1") + return min(int(value * size), size - 1) + + # WAA task IDs are UUIDs with a domain suffix, e.g., "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx-WOS" # Common suffixes: WOS (Windows OS), CHR (Chrome), NTP (Notepad), etc. WAA_TASK_ID_PATTERN = re.compile( @@ -672,16 +688,22 @@ def reset(self, task: BenchmarkTask) -> BenchmarkObservation: Raises: RuntimeError: If server is not reachable. """ + # Invalidate every task-bound observation before any reset work. A + # failed reset must not leave an earlier task actionable. + self._current_task = None + self._step_count = 0 + self._actions = [] + self._actual_screen_size = None + self._current_screenshot = None + self._current_a11y = None + self._current_rects = {} + if not self.check_connection(): raise RuntimeError( f"Cannot connect to WAA server at {self.config.server_url}. " f"Ensure Windows VM is running and server is started." ) - self._current_task = task - self._step_count = 0 - self._actions = [] - import requests if self.config.lightweight: @@ -691,7 +713,9 @@ def reset(self, task: BenchmarkTask) -> BenchmarkObservation: if task.raw_config: self._run_task_setup(task.raw_config) time.sleep(2.0) - return self._get_observation() + observation = self._get_observation() + self._current_task = task + return observation # Legacy mode (lightweight=False): full cleanup + focus checks. # Kept for backward compatibility but not recommended — the cleanup @@ -737,14 +761,17 @@ def reset(self, task: BenchmarkTask) -> BenchmarkObservation: "Marking task as infrastructure failure before step budget." ) - return self._get_observation() + observation = self._get_observation() + self._current_task = task + return observation def _send_command(self, command: str) -> None: """Send a pyautogui command to WAA via /execute_windows. - Handles fail-safe detection and recovery. If PyAutoGUI's fail-safe is - triggered (mouse at screen corner), recovery is attempted automatically - via the /execute endpoint before retrying the command once. + A fail-safe response can occur after a multi-call command delivered a + prefix. The command is retried only when the server explicitly reports + ``delivery_state=not_delivered``. Otherwise recovery only prepares a + later separately authorized attempt. Args: command: Python command string to execute on the WAA server. @@ -756,69 +783,180 @@ def _send_command(self, command: str) -> None: resp = requests.post( f"{self.config.server_url}/execute_windows", json={"command": command}, - timeout=self.config.timeout + timeout=self.config.timeout, ) - # Check ALL responses (200 and non-200) for fail-safe errors. - # WAA returns 500 with a JSON "message" field for fail-safe, - # and sometimes 200 with stderr containing the exception. - response_text = resp.text - if resp.status_code == 200: - result = resp.json() - stderr = result.get("stderr", "") - stdout = result.get("stdout", "") - response_text = stderr + stdout - if stderr: - logger.warning(f"Command stderr: {stderr}") - else: - logger.error(f"Execute failed ({resp.status_code}): {resp.text}") + except Exception as exc: + raise ActionDeliveryUncertainError( + "WAA command request failed; delivery outcome is uncertain: " + f"{exc}" + ) from exc + + response_text, stderr = self._command_response_details(resp) + + # WAA can report the fail-safe through HTTP 500 or through stderr on + # HTTP 200. Recovery is bounded to one attempt and must itself succeed. + if _is_failsafe_error(response_text): + logger.warning("PyAutoGUI fail-safe detected; attempting recovery...") + delivery_state = self._command_delivery_state(resp) + if not self._recover_failsafe(): + if delivery_state is ActionDeliveryState.NOT_DELIVERED: + raise ActionExecutionError( + "WAA fail-safe recovery failed after confirmed non-delivery" + ) + raise ActionDeliveryUncertainError( + "WAA fail-safe recovery failed; delivery could not be confirmed" + ) - # Detect PyAutoGUI fail-safe and attempt recovery (once per step) - if _is_failsafe_error(response_text): - logger.warning( - "PyAutoGUI fail-safe detected; attempting recovery..." + if delivery_state is not ActionDeliveryState.NOT_DELIVERED: + raise ActionDeliveryUncertainError( + "WAA fail-safe may have occurred after part of the command was " + "delivered; refusing a blind retry" ) - if self._recover_failsafe(): - logger.info( - "Fail-safe cleared; retrying command: %s", command - ) - retry_resp = requests.post( - f"{self.config.server_url}/execute_windows", - json={"command": command}, - timeout=self.config.timeout, - ) - if retry_resp.status_code == 200: - retry_result = retry_resp.json() - if retry_result.get("stderr"): - logger.warning( - f"Retry stderr: {retry_result['stderr']}" - ) - else: - logger.error( - f"Retry failed ({retry_resp.status_code}): " - f"{retry_resp.text}" - ) - else: - logger.error( - "Fail-safe recovery failed; step will proceed " - "with degraded state" + + logger.info( + "Fail-safe cleared after an explicit non-delivery receipt; " + "retrying command: %s", + command, + ) + try: + retry_resp = requests.post( + f"{self.config.server_url}/execute_windows", + json={"command": command}, + timeout=self.config.timeout, + ) + except Exception as exc: + raise ActionDeliveryUncertainError( + "WAA command retry failed; delivery outcome is uncertain: " + f"{exc}" + ) from exc + + retry_text, retry_stderr = self._command_response_details(retry_resp) + retry_delivery_state = self._command_delivery_state(retry_resp) + if _is_failsafe_error(retry_text): + if retry_delivery_state is ActionDeliveryState.NOT_DELIVERED: + raise ActionExecutionError( + "WAA command retry triggered a fail-safe before delivery" ) - elif resp.status_code == 200: - logger.debug(f"Executed: {command}") - except Exception as e: - logger.error(f"Execute request failed: {e}") + raise ActionDeliveryUncertainError( + "WAA command retry triggered a fail-safe; refusing another retry" + ) + if retry_resp.status_code != 200: + message = ( + "WAA command retry was not confirmed: " + f"HTTP {retry_resp.status_code}: {retry_resp.text}" + ) + if retry_delivery_state is ActionDeliveryState.NOT_DELIVERED: + raise ActionExecutionError(message) + raise ActionDeliveryUncertainError(message) + self._validate_command_receipt(retry_resp, context="retry") + if retry_stderr: + raise ActionDeliveryUncertainError( + f"WAA command retry returned stderr: {retry_stderr}" + ) + logger.debug("Executed after fail-safe recovery: %s", command) + return - def step( - self, action: BenchmarkAction - ) -> tuple[BenchmarkObservation, bool, dict[str, Any]]: + if resp.status_code != 200: + message = ( + "WAA command was not confirmed: " + f"HTTP {resp.status_code}: {resp.text}" + ) + if self._command_delivery_state(resp) is ActionDeliveryState.NOT_DELIVERED: + raise ActionExecutionError(message) + raise ActionDeliveryUncertainError(message) + self._validate_command_receipt(resp, context="command") + if stderr: + raise ActionDeliveryUncertainError( + f"WAA command returned stderr: {stderr}" + ) + logger.debug("Executed: %s", command) + + @classmethod + def _validate_command_receipt(cls, response: Any, *, context: str) -> None: + """Reject an explicit failed or incomplete HTTP-200 command receipt.""" + try: + result = response.json() + except Exception: + return + if not isinstance(result, dict): + return + + reason: str | None = None + if "success" in result and result["success"] is not True: + reason = f"success={result['success']!r}" + if result.get("error"): + reason = f"error={result['error']!r}" + if "returncode" in result and ( + isinstance(result["returncode"], bool) + or not isinstance(result["returncode"], int) + or result["returncode"] != 0 + ): + reason = f"returncode={result['returncode']!r}" + + delivery_state = cls._command_delivery_state(response) + if "delivery_state" in result and delivery_state is None: + reason = f"invalid delivery_state={result['delivery_state']!r}" + if reason is None and delivery_state in { + ActionDeliveryState.NOT_DELIVERED, + ActionDeliveryState.UNCERTAIN, + }: + reason = f"delivery_state={delivery_state.value}" + if reason is None: + return + + message = f"WAA {context} receipt did not confirm execution: {reason}" + if delivery_state is ActionDeliveryState.NOT_DELIVERED: + raise ActionExecutionError(message) + raise ActionDeliveryUncertainError(message) + + @staticmethod + def _command_delivery_state(response: Any) -> ActionDeliveryState | None: + """Read an explicit typed delivery state from one server receipt.""" + try: + result = response.json() + except Exception: + return None + if not isinstance(result, dict): + return None + try: + return ActionDeliveryState(result.get("delivery_state")) + except (TypeError, ValueError): + return None + + @staticmethod + def _command_response_details(response: Any) -> tuple[str, str]: + """Return response diagnostics and reject malformed success receipts.""" + response_text = str(response.text or "") + try: + result = response.json() + except Exception as exc: + if response.status_code == 200: + raise ActionDeliveryUncertainError( + "WAA returned an invalid command receipt; delivery outcome is uncertain" + ) from exc + return response_text, "" + + if not isinstance(result, dict): + if response.status_code == 200: + raise ActionDeliveryUncertainError( + "WAA returned an invalid command receipt; delivery outcome is uncertain" + ) + return response_text, "" + + stderr = str(result.get("stderr") or "") + stdout = str(result.get("stdout") or "") + message = str(result.get("message") or "") + return "\n".join((response_text, stderr, stdout, message)), stderr + + def step(self, action: BenchmarkAction) -> tuple[BenchmarkObservation, bool, dict[str, Any]]: """Execute action and return new observation. Uses element-based grounding via WAA's Computer class. Click actions are translated to computer.mouse.move_id(id) commands that WAA executes using the rects we POSTed to /update_computer. - If PyAutoGUI's fail-safe is triggered (mouse at screen corner), recovery - is attempted automatically via the /execute endpoint before retrying the - command once. + If PyAutoGUI's fail-safe is triggered, the adapter refuses a blind + retry unless the server proves that the command was not dispatched. Args: action: Action to execute. @@ -826,26 +964,48 @@ def step( Returns: Tuple of (observation, done, info). """ - self._step_count += 1 - self._actions.append(action) + if self._current_task is None: + raise ActionExecutionError("WAA action requires a successfully loaded task") + + pointer_action = action.type in { + "click", + "double_click", + "right_click", + "scroll", + "drag", + } or (action.type == "type" and action.target_node_id is not None) + if pointer_action: + self._require_pointer_context() # Translate action to element-based command for WAA's Computer command = self._translate_action(action) + if action.type not in ("done", "error") and ( + not command or command.lstrip().startswith("pass") + ): + raise ActionExecutionError( + f"WAA action {action.type!r} did not produce an executable command" + ) + # Execute command via /execute_windows (has access to computer object) if command: self._send_command(command) + self._step_count += 1 + self._actions.append(action) + # Wait for UI to settle time.sleep(self.config.action_delay) # Check if done (error actions are also terminal) - done = ( - action.type in ("done", "error") or - self._step_count >= self.config.max_steps - ) + done = action.type in ("done", "error") or self._step_count >= self.config.max_steps - obs = self._get_observation() + try: + obs = self._get_observation() + except Exception as exc: + raise ActionDeliveredObservationError( + f"WAA action {action.type!r} was delivered, but observation failed: {exc}" + ) from exc info = { "step": self._step_count, "command": command, @@ -867,12 +1027,19 @@ def _recover_failsafe(self) -> bool: """ import requests + recovery_script = ( + "import pyautogui\n" + "previous = pyautogui.FAILSAFE\n" + "try:\n" + " pyautogui.FAILSAFE = False\n" + " pyautogui.moveTo(500, 400)\n" + "finally:\n" + " pyautogui.FAILSAFE = previous\n" + ) + recovery_payload = base64.b64encode(recovery_script.encode()).decode() recovery_command = ( - 'python -c "' - "import pyautogui; " - "pyautogui.FAILSAFE=False; " - "pyautogui.moveTo(500, 400)" - '"' + 'python -c "import base64;' + f"exec(base64.b64decode('{recovery_payload}'))\"" ) try: resp = requests.post( @@ -880,15 +1047,22 @@ def _recover_failsafe(self) -> bool: json={"command": recovery_command}, timeout=10.0, ) - if resp.status_code == 200: - logger.info("Fail-safe recovery command succeeded") - return True - else: + if resp.status_code != 200: logger.error( - f"Fail-safe recovery request returned HTTP {resp.status_code}: " - f"{resp.text}" + f"Fail-safe recovery request returned HTTP {resp.status_code}: {resp.text}" ) return False + try: + _, stderr = self._command_response_details(resp) + self._validate_command_receipt(resp, context="fail-safe recovery") + except ActionExecutionError as exc: + logger.error("Fail-safe recovery returned an invalid receipt: %s", exc) + return False + if stderr: + logger.error("Fail-safe recovery returned stderr: %s", stderr) + return False + logger.info("Fail-safe recovery command succeeded") + return True except Exception as e: logger.error(f"Fail-safe recovery request failed: {e}") return False @@ -1142,7 +1316,7 @@ def evaluate(self, task: BenchmarkTask) -> BenchmarkResult: return result def _result_from_evaluate_response( - self, task: BenchmarkTask, result: dict, + self, task: BenchmarkTask, result: Any, ) -> BenchmarkResult: """Build a BenchmarkResult from an /evaluate 200 body. @@ -1153,13 +1327,54 @@ def _result_from_evaluate_response( servers have neither field and are treated as measurements, which is what they were. """ + def invalid(reason: str) -> BenchmarkResult: + return BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + num_steps=self._step_count, + reason=f"Malformed evaluation response: {reason}", + error_type="evaluation", + ) + + if not isinstance(result, dict): + return invalid("body must be an object") + + success = result.get("success") + if not isinstance(success, bool): + return invalid("success must be a boolean") + + score = result.get("score") + if ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(float(score)) + or not 0.0 <= float(score) <= 1.0 + ): + return invalid("score must be a finite number between 0 and 1") + score = float(score) + + scored = result.get("scored") + if scored is not None and not isinstance(scored, bool): + return invalid("scored must be a boolean when present") + error_type = result.get("error_type") - if result.get("scored") is False and not error_type: + if error_type not in (None, "infrastructure", "evaluation"): + return invalid("error_type must be infrastructure, evaluation, or null") + if scored is True and error_type is not None: + return invalid("a scored result cannot carry an error_type") + if success != (score >= 1.0): + return invalid("success contradicts score") + + if scored is False and error_type is None: error_type = "evaluation" + if error_type is not None and (success or score != 0.0): + return invalid("an unscored result must have success=false and score=0") + return BenchmarkResult( task_id=task.task_id, - success=result.get("success", False), - score=result.get("score", 0.0), + success=success, + score=score, num_steps=self._step_count, reason=result.get("reason"), error_type=error_type, @@ -1219,14 +1434,24 @@ def close(self) -> None: def screen_size(self) -> tuple[int, int]: """Actual screen dimensions (width, height) detected from screenshots. - Defaults to config values until first screenshot is taken. - RL agents should verify this matches their coordinate normalization - assumption (e.g., 1920x1080 for models trained on that resolution). + Configured dimensions are not observation evidence. The property + refuses access until a screenshot establishes the live viewport. """ - return self._actual_screen_size or ( - self.config.screen_width, - self.config.screen_height, - ) + if self._actual_screen_size is None: + raise ActionExecutionError( + "WAA screen size requires an exact measured viewport" + ) + return self._actual_screen_size + + def _require_pointer_context(self) -> tuple[int, int]: + """Return the current measured viewport or refuse pointer actuation.""" + if self._current_task is None: + raise ActionExecutionError("WAA pointer action requires a loaded task") + if self._actual_screen_size is None: + raise ActionExecutionError( + "WAA pointer action requires an exact measured viewport" + ) + return self.screen_size def observe(self) -> BenchmarkObservation: """Get current observation (screenshot + a11y tree) without stepping. @@ -1298,34 +1523,68 @@ def pixel_action( Returns: Tuple of (observation, done, info) -- same as step(). """ - if x_frac is not None or y_frac is not None: - w, h = self.screen_size - x = int((x_frac or 0.0) * w) - y = int((y_frac or 0.0) * h) + if action_type not in ("done", "error") and self._current_task is None: + raise ActionExecutionError( + "WAA pixel action requires a successfully loaded task" + ) - # Build the action for bookkeeping (step count, action history) - action = BenchmarkAction(type=action_type, x=x, y=y, text=text, key=key) - self._step_count += 1 - self._actions.append(action) + pointer_actions = { + "click", + "double_click", + "right_click", + "type", + "scroll", + "drag", + } + if action_type in pointer_actions: + measured_viewport = self._require_pointer_context() - # Build pyautogui command directly -- no element resolution needed + if x_frac is not None or y_frac is not None: + # Pointer actions above require this exact measured value. Do not + # substitute configured dimensions as actuation evidence. + if action_type not in pointer_actions: + measured_viewport = self._require_pointer_context() + w, h = measured_viewport + if x_frac is not None: + x = _fraction_to_pixel(x_frac, w, "x") + if y_frac is not None: + y = _fraction_to_pixel(y_frac, h, "y") + + # Build and validate the command before changing rollout state. + action = BenchmarkAction(type=action_type, x=x, y=y, text=text, key=key) command = self._build_pixel_command( - action_type=action_type, x=x, y=y, text=text, key=key, + action_type=action_type, + x=x, + y=y, + text=text, + key=key, ) + if action_type not in ("done", "error") and ( + not command or command.lstrip().startswith("pass") + ): + raise ActionExecutionError( + f"WAA pixel action {action_type!r} did not produce an executable command" + ) + if command: self._send_command(command) + self._step_count += 1 + self._actions.append(action) + # Wait for UI to settle time.sleep(self.config.action_delay) # Check if done (error actions are also terminal) - done = ( - action_type in ("done", "error") - or self._step_count >= self.config.max_steps - ) + done = action_type in ("done", "error") or self._step_count >= self.config.max_steps - obs = self._get_observation() + try: + obs = self._get_observation() + except Exception as exc: + raise ActionDeliveredObservationError( + f"WAA pixel action {action_type!r} was delivered, but observation failed: {exc}" + ) from exc info = { "step": self._step_count, "command": command, @@ -1365,22 +1624,37 @@ def _build_pixel_command( if action_type == "wait": return "import time; time.sleep(1)" - # Resolve pixel coordinates and clamp to safe margin - px = int(x) if x is not None else 0 - py = int(y) if y is not None else 0 - px, py = self._clamp_pixel_coords(px, py) - - if action_type == "click": - return f"import pyautogui; pyautogui.click({px}, {py})" - - if action_type == "double_click": - return f"import pyautogui; pyautogui.doubleClick({px}, {py})" - - if action_type == "right_click": - return f"import pyautogui; pyautogui.rightClick({px}, {py})" + if action_type in { + "click", + "double_click", + "right_click", + "type", + "scroll", + "drag", + }: + if x is None or y is None: + raise ActionExecutionError( + f"WAA pixel {action_type!r} action requires x and y coordinates" + ) + try: + px, py = self._clamp_pixel_coords(int(x), int(y)) + except (TypeError, ValueError, OverflowError) as exc: + raise ActionExecutionError( + f"WAA pixel {action_type!r} action has invalid coordinates" + ) from exc + + click_method = { + "click": "click", + "double_click": "doubleClick", + "right_click": "rightClick", + }.get(action_type) + if click_method: + return f"import pyautogui; pyautogui.{click_method}({px}, {py})" if action_type == "type": - type_body = _build_type_commands(text or "") + if text is None: + raise ActionExecutionError("WAA pixel 'type' action requires text") + type_body = _build_type_commands(text) return ( f"import pyautogui; import time; " f"pyautogui.click({px}, {py}); " @@ -1389,6 +1663,8 @@ def _build_pixel_command( ) if action_type == "key": + if not key: + raise ActionExecutionError("WAA pixel 'key' action requires key") # Reuse the key translation logic temp_action = BenchmarkAction(type="key", key=key) return self._translate_key_action(temp_action) @@ -1398,11 +1674,21 @@ def _build_pixel_command( if action_type == "drag": # drag needs end coordinates passed via text as "end_x,end_y" - ex, ey = px, py - if text: - parts = text.split(",") - if len(parts) == 2: - ex, ey = self._clamp_pixel_coords(int(parts[0]), int(parts[1])) + if not text: + raise ActionExecutionError( + "WAA pixel 'drag' action requires end coordinates" + ) + parts = text.split(",") + if len(parts) != 2: + raise ActionExecutionError( + "WAA pixel 'drag' end coordinates must be 'x,y'" + ) + try: + ex, ey = self._clamp_pixel_coords(int(parts[0]), int(parts[1])) + except (TypeError, ValueError, OverflowError) as exc: + raise ActionExecutionError( + "WAA pixel 'drag' action has invalid end coordinates" + ) from exc return ( f"import pyautogui; import time; " f"pyautogui.moveTo({px}, {py}); " @@ -1414,8 +1700,7 @@ def _build_pixel_command( f"pyautogui.mouseUp(button='left')" ) - logger.warning(f"Unknown pixel action type: {action_type}") - return None + raise ActionExecutionError(f"Unsupported WAA pixel action type: {action_type!r}") def _get_observation(self) -> BenchmarkObservation: """Fetch current observation from WAA server. @@ -1428,6 +1713,12 @@ def _get_observation(self) -> BenchmarkObservation: """ import requests + # Observation evidence is one atomic generation. Do not combine a new + # screenshot with accessibility data or geometry from an older frame. + self._current_screenshot = None + self._current_a11y = None + self._current_rects = {} + self._actual_screen_size = None screenshot = None a11y_tree = None @@ -1488,6 +1779,11 @@ def _get_observation(self) -> BenchmarkObservation: if self._current_rects: self._update_waa_computer() + if not screenshot and not a11y_tree: + raise AdapterInfrastructureError( + "WAA observation failed: no screenshot or accessibility evidence was available" + ) + return BenchmarkObservation( screenshot=screenshot, viewport=self._actual_screen_size or (self.config.screen_width, self.config.screen_height), @@ -1640,7 +1936,7 @@ def _config_entry_to_command(entry: dict) -> str | None: escaped = shell_cmd.replace("\\", "\\\\").replace("'", "\\'") return ( "import subprocess; " - f"subprocess.run('{escaped}', shell=True, timeout=60)" + f"subprocess.run('{escaped}', shell=True, timeout=60, check=True)" ) if entry_type == "launch": @@ -2017,10 +2313,15 @@ def _run_task_setup(self, raw_config: dict) -> None: timeout=http_timeout, ) if resp.status_code == 200: - result = resp.json() - stderr = result.get("stderr", "") + _, stderr = self._command_response_details(resp) if stderr: - logger.warning("Setup %s stderr: %s", entry_type, stderr) + raise ActionDeliveryUncertainError( + f"WAA setup command returned stderr: {stderr}" + ) + self._validate_command_receipt( + resp, + context=f"setup {entry_type}", + ) logger.info("Setup %s: ok", entry_type) self._last_setup_results.append( {"type": entry_type, "status": "ok"} @@ -2869,21 +3170,30 @@ def _check_foreground_matches( return False def _clamp_pixel_coords(self, x: int, y: int) -> tuple[int, int]: - """Clamp pixel coordinates to a safe margin from screen edges. + """Validate pixel coordinates without moving the requested target. - Prevents PyAutoGUI fail-safe by keeping the mouse at least 5px from - any screen corner. If both coordinates are 0, the action would - target the top-left corner -- the most common fail-safe trigger. + PyAutoGUI reserves the four exact screen corners for its fail-safe. + Refuse those targets instead of shifting a valid target to a different + control. Other edge coordinates remain unchanged. Returns: - Clamped (x, y) tuple. + The unchanged validated ``(x, y)`` tuple. """ - screen_w, screen_h = self._actual_screen_size or ( - self.config.screen_width, self.config.screen_height, - ) - margin = 5 - x = max(margin, min(x, screen_w - margin)) - y = max(margin, min(y, screen_h - margin)) + screen_w, screen_h = self._require_pointer_context() + if x < 0 or x >= screen_w or y < 0 or y >= screen_h: + raise ActionExecutionError( + f"WAA coordinates ({x}, {y}) are outside the " + f"{screen_w}x{screen_h} viewport" + ) + if (x, y) in { + (0, 0), + (0, screen_h - 1), + (screen_w - 1, 0), + (screen_w - 1, screen_h - 1), + }: + raise ActionExecutionError( + f"WAA coordinates ({x}, {y}) are reserved for the input fail-safe" + ) return x, y def _translate_action(self, action: BenchmarkAction) -> str | None: @@ -2917,7 +3227,9 @@ def _translate_action(self, action: BenchmarkAction) -> str | None: return self._translate_click_action(action, "right_click") if action.type == "type": - text = action.text or "" + if action.text is None: + raise ActionExecutionError("WAA 'type' action requires text") + text = action.text type_body = _build_type_commands(text) # If target_node_id is set (from type_element), click element first to focus it if action.target_node_id is not None: @@ -2926,18 +3238,24 @@ def _translate_action(self, action: BenchmarkAction) -> str | None: rect = self._current_rects[elem_id] cx = (rect[0] + rect[2]) // 2 cy = (rect[1] + rect[3]) // 2 - return ( - f"import pyautogui; import time; " - f"pyautogui.click({cx}, {cy}); " - f"time.sleep(0.2); " - f"{type_body}" - ) + cx, cy = self._clamp_pixel_coords(cx, cy) else: - logger.warning(f"Element ID '{elem_id}' not found for type_element, typing without focus") + raise ActionExecutionError( + f"WAA element {elem_id!r} is stale; recorded coordinates " + "are not authorized as current type evidence" + ) + return ( + f"import pyautogui; import time; " + f"pyautogui.click({cx}, {cy}); " + f"time.sleep(0.2); " + f"{type_body}" + ) return f"import pyautogui; {type_body}" if action.type == "key": - key = action.key or "" + if not action.key: + raise ActionExecutionError("WAA 'key' action requires key") + key = action.key modifiers = action.modifiers or [] # Handle modifier+key combos (Ctrl+A, Alt+F4, etc.) @@ -2956,7 +3274,9 @@ def _translate_action(self, action: BenchmarkAction) -> str | None: return f"import pyautogui; pyautogui.press('{key.lower()}')" if action.type == "scroll": - direction = action.scroll_direction or "down" + direction = action.scroll_direction + if direction not in ("up", "down"): + raise ActionExecutionError("WAA scroll direction must be 'up' or 'down'") # pyautogui.scroll(clicks) - positive is up, negative is down clicks = 3 if direction == "up" else -3 return f"import pyautogui; pyautogui.scroll({clicks})" @@ -2970,31 +3290,55 @@ def _translate_action(self, action: BenchmarkAction) -> str | None: rect = self._current_rects[elem_id] start_x = (rect[0] + rect[2]) // 2 start_y = (rect[1] + rect[3]) // 2 + else: + raise ActionExecutionError( + f"WAA element {elem_id!r} is stale; recorded coordinates " + "are not authorized as current drag evidence" + ) if start_x is None and action.x is not None and action.y is not None: - screen_w, screen_h = self._actual_screen_size or (self.config.screen_width, self.config.screen_height) - start_x = action.x if not isinstance(action.x, float) or action.x > 1 else int(action.x * screen_w) - start_y = action.y if not isinstance(action.y, float) or action.y > 1 else int(action.y * screen_h) + screen_w, screen_h = self._actual_screen_size or ( + self.config.screen_width, + self.config.screen_height, + ) + start_x = ( + action.x + if not isinstance(action.x, float) or action.x > 1 + else _fraction_to_pixel(action.x, screen_w, "x") + ) + start_y = ( + action.y + if not isinstance(action.y, float) or action.y > 1 + else _fraction_to_pixel(action.y, screen_h, "y") + ) # Get end position - screen_w, screen_h = self._actual_screen_size or (self.config.screen_width, self.config.screen_height) + screen_w, screen_h = self._actual_screen_size or ( + self.config.screen_width, + self.config.screen_height, + ) end_x = action.end_x end_y = action.end_y if end_x is not None and isinstance(end_x, float) and 0 <= end_x <= 1: - end_x = int(end_x * screen_w) + end_x = _fraction_to_pixel(end_x, screen_w, "end_x") if end_y is not None and isinstance(end_y, float) and 0 <= end_y <= 1: - end_y = int(end_y * screen_h) + end_y = _fraction_to_pixel(end_y, screen_h, "end_y") # Skip drag if coordinates are missing or both are at origin if start_x is None or end_x is None or end_y is None: - logger.warning("Drag action missing coordinates, skipping") - return "pass # drag skipped: missing coordinates" + raise ActionExecutionError("WAA drag action requires start and end coordinates") if int(start_x) == 0 and int(start_y) == 0 and int(end_x) == 0 and int(end_y) == 0: - logger.warning("Drag action has all-zero coordinates, skipping") - return "pass # drag skipped: all-zero coordinates" + raise ActionExecutionError("WAA drag action cannot use all-zero coordinates") # Clamp to safe margin - start_x, start_y = self._clamp_pixel_coords(int(start_x), int(start_y)) - end_x, end_y = self._clamp_pixel_coords(int(end_x), int(end_y)) + try: + start_x, start_y = self._clamp_pixel_coords( + int(start_x), int(start_y) + ) + end_x, end_y = self._clamp_pixel_coords(int(end_x), int(end_y)) + except (TypeError, ValueError, OverflowError) as exc: + raise ActionExecutionError( + "WAA drag action has invalid or out-of-viewport coordinates" + ) from exc return ( f"import pyautogui; import time; " @@ -3007,8 +3351,7 @@ def _translate_action(self, action: BenchmarkAction) -> str | None: f"pyautogui.mouseUp(button='left')" ) - logger.warning(f"Unknown action type: {action.type}") - return None + raise ActionExecutionError(f"Unsupported WAA action type: {action.type!r}") def _translate_click_action(self, action: BenchmarkAction, click_method: str) -> str: """Translate click-type action to pyautogui command. @@ -3038,26 +3381,33 @@ def _translate_click_action(self, action: BenchmarkAction, click_method: str) -> cx, cy = self._clamp_pixel_coords(cx, cy) return f"import pyautogui; pyautogui.{pyautogui_method}({cx}, {cy})" else: - logger.warning(f"Element ID '{elem_id}' not found in rects, falling back to coordinates") - # If no coordinates were provided alongside the element ID, - # skip the click entirely rather than clicking (0,0) which - # triggers PyAutoGUI fail-safe. - if action.x is None and action.y is None: - logger.warning(f"No fallback coordinates for '{elem_id}', skipping click") - return "pass # element not found and no coordinates" + raise ActionExecutionError( + f"WAA element {elem_id!r} is stale; recorded coordinates " + "are not authorized as current click evidence" + ) # Fallback: use coordinates if provided - x = action.x if action.x is not None else 0 - y = action.y if action.y is not None else 0 + if action.x is None or action.y is None: + raise ActionExecutionError("WAA click action requires x and y coordinates") + x = action.x + y = action.y # Convert normalized coordinates to pixels - screen_w, screen_h = self._actual_screen_size or (self.config.screen_width, self.config.screen_height) + screen_w, screen_h = self._actual_screen_size or ( + self.config.screen_width, + self.config.screen_height, + ) if isinstance(x, float) and 0 <= x <= 1: - x = int(x * screen_w) + x = _fraction_to_pixel(x, screen_w, "x") if isinstance(y, float) and 0 <= y <= 1: - y = int(y * screen_h) + y = _fraction_to_pixel(y, screen_h, "y") - x, y = self._clamp_pixel_coords(int(x), int(y)) + try: + x, y = self._clamp_pixel_coords(int(x), int(y)) + except (TypeError, ValueError, OverflowError) as exc: + raise ActionExecutionError( + "WAA click action has invalid or out-of-viewport coordinates" + ) from exc return f"import pyautogui; pyautogui.{pyautogui_method}({x}, {y})" def _translate_key_action(self, action: BenchmarkAction) -> str: @@ -3083,9 +3433,18 @@ def _translate_key_action(self, action: BenchmarkAction) -> str: "End": "end", "PageUp": "pageup", "PageDown": "pagedown", - "F1": "f1", "F2": "f2", "F3": "f3", "F4": "f4", - "F5": "f5", "F6": "f6", "F7": "f7", "F8": "f8", - "F9": "f9", "F10": "f10", "F11": "f11", "F12": "f12", + "F1": "f1", + "F2": "f2", + "F3": "f3", + "F4": "f4", + "F5": "f5", + "F6": "f6", + "F7": "f7", + "F8": "f8", + "F9": "f9", + "F10": "f10", + "F11": "f11", + "F12": "f12", } key = key_map.get(key, key.lower()) diff --git a/openadapt_evals/adapters/waa/mock.py b/openadapt_evals/adapters/waa/mock.py index 60f80fa..1e25366 100644 --- a/openadapt_evals/adapters/waa/mock.py +++ b/openadapt_evals/adapters/waa/mock.py @@ -17,9 +17,11 @@ import json import logging +import math import sys import time from dataclasses import dataclass +from numbers import Real from pathlib import Path from typing import Any @@ -292,20 +294,86 @@ def evaluate(self, task: BenchmarkTask) -> BenchmarkResult: # Run WAA's evaluator try: result = self._desktop_env.evaluate() - success = result.get("success", False) - score = 1.0 if success else 0.0 - reason = result.get("reason", None) except Exception as e: logger.error(f"Evaluation failed for task {task.task_id}: {e}") - success = False - score = 0.0 - reason = str(e) + error_type = getattr(e, "error_type", "evaluation") + if error_type not in ("infrastructure", "evaluation"): + error_type = "evaluation" + return BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + reason=str(e), + error=str(e), + error_type=error_type, + ) + + return self._result_from_evaluate_response(task, result) + + def _result_from_evaluate_response( + self, task: BenchmarkTask, result: Any + ) -> BenchmarkResult: + """Validate one native WAA evaluator result without inventing a score.""" + + def invalid(reason: str) -> BenchmarkResult: + return BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + reason=f"Malformed evaluation response: {reason}", + error_type="evaluation", + ) + + if isinstance(result, Real) and not isinstance(result, bool): + score = float(result) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + return invalid("scalar score must be a finite number between 0 and 1") + return BenchmarkResult( + task_id=task.task_id, + success=score >= 1.0, + score=score, + ) + + if not isinstance(result, dict): + return invalid("body must be an object") + + success = result.get("success") + if not isinstance(success, bool): + return invalid("success must be a boolean") + + score = result.get("score") + if ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(float(score)) + or not 0.0 <= float(score) <= 1.0 + ): + return invalid("score must be a finite number between 0 and 1") + score = float(score) + + scored = result.get("scored") + if scored is not None and not isinstance(scored, bool): + return invalid("scored must be a boolean when present") + + error_type = result.get("error_type") + if error_type not in (None, "infrastructure", "evaluation"): + return invalid("error_type must be infrastructure, evaluation, or null") + if scored is True and error_type is not None: + return invalid("a scored result cannot carry an error_type") + if success != (score >= 1.0): + return invalid("success contradicts score") + + if scored is False and error_type is None: + error_type = "evaluation" + if error_type is not None and (success or score != 0.0): + return invalid("an unscored result must have success=false and score=0") return BenchmarkResult( task_id=task.task_id, success=success, score=score, - reason=reason, + reason=result.get("reason"), + error_type=error_type, ) def close(self) -> None: diff --git a/openadapt_evals/agents/api_agent.py b/openadapt_evals/agents/api_agent.py index 780acdd..611b878 100644 --- a/openadapt_evals/agents/api_agent.py +++ b/openadapt_evals/agents/api_agent.py @@ -33,12 +33,14 @@ from PIL import Image +from openadapt_evals.action_envelope import parse_single_json_object from openadapt_evals.adapters.base import ( BenchmarkAction, BenchmarkObservation, BenchmarkTask, ) from openadapt_evals.agents.base import BenchmarkAgent +from openadapt_evals.errors import ActionParseError logger = logging.getLogger("openadapt_evals.agents.api") @@ -758,23 +760,96 @@ def _parse_api_response( - error: Error message (if failed) """ try: - # Defensive: ensure response_text is actually a string + # Model output is an action protocol. Coercing arbitrary objects to + # strings can manufacture a parseable action from malformed output. if not isinstance(response_text, str): + return { + "status": "failed", + "error": "API response must be a string action envelope", + } + + decision_matches = re.findall( + r"```decision\s*\n(.*?)```", response_text, re.DOTALL | re.IGNORECASE + ) + if len(decision_matches) > 1: + return {"status": "failed", "error": "Multiple decision blocks"} + + action_calls = re.findall( + r"\bcomputer\.[A-Za-z_][A-Za-z0-9_]*\s*\(", response_text + ) + json_objects = re.findall(r"\{[^{}]*\}", response_text, re.DOTALL) + fenced_pattern = re.compile( + r"^```([A-Za-z0-9_-]*)[ \t]*\n(.*?)^```[ \t]*$", + re.DOTALL | re.IGNORECASE | re.MULTILINE, + ) + fenced_matches = list(fenced_pattern.finditer(response_text)) + action_blocks = [ + match.group(2) + for match in fenced_matches + if match.group(1).lower() in {"", "python", "json"} + ] + if len(action_calls) > 1 or len(json_objects) > 1 or len(action_blocks) > 1: + return { + "status": "failed", + "error": "Response must contain exactly one action envelope", + } + + if action_blocks: + block = action_blocks[0].strip() try: - # Handle case where response might be a dict or other object - if hasattr(response_text, 'get'): - # It's a dict-like object - response_text = str(response_text) - logs["response_type_coerced"] = "dict_to_str" - elif hasattr(response_text, 'text'): - # It might be a response object with a text attribute - response_text = response_text.text - logs["response_type_coerced"] = "obj_text_attr" - else: - response_text = str(response_text) - logs["response_type_coerced"] = "generic_str" - except Exception as e: - return {"status": "failed", "error": f"Response type coercion failed: {e}"} + block_json = parse_single_json_object(block) + except ActionParseError as exc: + return {"status": "failed", "error": str(exc)} + if block_json is None and not self._validate_action( + block, width, height + ): + return { + "status": "failed", + "error": "Fenced block must contain one exact action", + } + + outside_blocks = fenced_pattern.sub("", response_text) + outside_calls = re.findall( + r"\bcomputer\.[A-Za-z_][A-Za-z0-9_]*\s*\(", outside_blocks + ) + outside_json = re.findall(r"\{[^{}]*\}", outside_blocks, re.DOTALL) + if action_blocks and (outside_calls or outside_json): + return { + "status": "failed", + "error": "Competing action envelopes use different formats", + } + if ( + not action_blocks + and action_calls + and json_objects + and "computer." not in json_objects[0] + ): + return { + "status": "failed", + "error": "Competing action envelopes use different formats", + } + + if json_objects: + try: + parse_single_json_object(json_objects[0]) + except ActionParseError as exc: + return {"status": "failed", "error": str(exc)} + + decision = None + if decision_matches: + decision = decision_matches[0].strip().upper() + if decision not in {"DONE", "FAIL", "WAIT", "CONTINUE"}: + return { + "status": "failed", + "error": f"Unknown decision: {decision_matches[0].strip()!r}", + } + if decision in {"DONE", "FAIL", "WAIT"}: + if action_calls or json_objects or action_blocks: + return { + "status": "failed", + "error": "Terminal decision cannot include an action", + } + return {"status": "terminal", "action": decision} # Extract memory block (update internal state) try: @@ -784,20 +859,6 @@ def _parse_api_response( except Exception as e: logger.warning(f"Memory extraction failed: {e}") - # Strategy 0: Check for terminal decisions (DONE, FAIL, WAIT) - try: - decision_match = re.search(r"```decision\n(.*?)```", response_text, re.DOTALL) - if decision_match: - decision = decision_match.group(1).strip().upper() - if "DONE" in decision: - return {"status": "terminal", "action": "DONE"} - elif "FAIL" in decision: - return {"status": "terminal", "action": "FAIL"} - elif "WAIT" in decision: - return {"status": "terminal", "action": "WAIT"} - except Exception as e: - logger.warning(f"Decision extraction failed: {e}") - # Strategy 1: Standard Python code block try: code_match = re.search(r"```python\n(.*?)```", response_text, re.DOTALL) @@ -1170,8 +1231,14 @@ def _parse_computer_action( end_x=float(x2), end_y=float(y2), raw_action=raw_action, ) - logger.warning(f"Unrecognized action, treating as no-op: {code}") - return BenchmarkAction(type="done", raw_action=raw_action) + logger.warning("Unrecognized computer action: %s", code) + raw_action.update( + { + "parse_error": "unrecognized computer action", + "error_type": "agent", + } + ) + return BenchmarkAction(type="error", raw_action=raw_action) def _add_to_history(self, entry: str) -> None: """Add an entry to the rich history (reasoning + action).""" diff --git a/openadapt_evals/agents/demo_executor.py b/openadapt_evals/agents/demo_executor.py index ca3bea6..c1efa8d 100644 --- a/openadapt_evals/agents/demo_executor.py +++ b/openadapt_evals/agents/demo_executor.py @@ -29,6 +29,7 @@ from openadapt_evals.adapters.base import BenchmarkAction, BenchmarkObservation from openadapt_evals.demo_library import Demo, DemoStep +from openadapt_evals.errors import ActionExecutionError from openadapt_evals.grounding import ( GroundingTarget, check_state_preconditions, @@ -110,6 +111,9 @@ def run( _tier1 = 0 _tier2 = 0 + if not demo.steps: + raise ActionExecutionError("Demo has no executable steps") + try: from openadapt_evals.telemetry import track_demo_execution track_demo_execution( @@ -156,8 +160,14 @@ def run( action = self._execute_step(step, obs) if action is None: - logger.warning("Step %d: no action produced, skipping", i + 1) - continue + raise ActionExecutionError( + f"Demo step {i + 1} did not produce an action" + ) + if action.type == "error": + raise ActionExecutionError( + f"Demo step {i + 1} failed before actuation: " + f"{action.raw_action or {}}" + ) # Count tiers for telemetry tier = (action.raw_action or {}).get("tier", 2) @@ -437,7 +447,10 @@ def _ground_click_http( raw = resp.json()["choices"][0]["message"]["content"] except Exception as exc: logger.error("HTTP grounder failed: %s", exc) - return BenchmarkAction(type="click", x=0.5, y=0.5) + return BenchmarkAction( + type="error", + raw_action={"grounder_error": str(exc)}, + ) logger.info("HTTP grounder: %s", raw[:200]) @@ -475,21 +488,33 @@ def _ground_click_vlm( cost_label="demo_executor_grounder", ) + from openadapt_evals.errors import ActionParseError from openadapt_evals.training.trl_rollout import parse_action_json - action = parse_action_json(raw) - if action.type == "done": - logger.warning( - "Grounder could not find %r -- returning click at center", - description, + try: + action = parse_action_json(raw) + except ActionParseError as exc: + return BenchmarkAction( + type="error", + raw_action={"grounder_error": str(exc)}, ) - return BenchmarkAction(type="click", x=0.5, y=0.5) + if action.type != "click": + return BenchmarkAction( + type="error", + raw_action={ + "grounder_error": ( + f"grounder returned {action.type!r}; click required" + ) + }, + ) return action def _dispatch_action(self, env, action: BenchmarkAction): """Execute an action through the environment.""" - if action.x is not None and action.y is not None: + if action.type in ("click", "double_click", "right_click"): + if action.x is None or action.y is None: + raise ActionExecutionError(f"{action.type} requires x and y") x, y = float(action.x), float(action.y) if 0 <= x <= 1 and 0 <= y <= 1: return env.pixel_action( diff --git a/openadapt_evals/agents/planner_grounder_agent.py b/openadapt_evals/agents/planner_grounder_agent.py index 211d243..5722be5 100644 --- a/openadapt_evals/agents/planner_grounder_agent.py +++ b/openadapt_evals/agents/planner_grounder_agent.py @@ -56,6 +56,7 @@ import logging from typing import TYPE_CHECKING, Any +from openadapt_evals.action_envelope import parse_single_json_object from openadapt_evals.adapters.base import ( BenchmarkAction, BenchmarkObservation, @@ -66,6 +67,7 @@ action_to_string, format_accessibility_tree, ) +from openadapt_evals.errors import ActionParseError try: from openadapt_evals.integrations.weave_integration import weave_op @@ -329,12 +331,16 @@ def act( # -- Step 1: Call planner ------------------------------------------ planner_output = self._call_planner(observation, task) - decision = planner_output.get("decision", "COMMAND").upper() + raw_decision = planner_output.get("decision", "COMMAND") + decision = raw_decision.upper() if isinstance(raw_decision, str) else "" reasoning = planner_output.get("reasoning", "") # Extract structured fields (new format) with backward-compat # fallback to the old "instruction" field. - action_type = planner_output.get("action_type", "").lower() + raw_action_type = planner_output.get("action_type", "") + action_type = ( + raw_action_type.lower() if isinstance(raw_action_type, str) else "" + ) action_value = planner_output.get("action_value", "") target_description = planner_output.get("target_description", "") instruction = planner_output.get("instruction", target_description) @@ -393,6 +399,70 @@ def act( }, ) + if decision != "COMMAND": + logger.warning("Planner returned an unknown decision: %r", raw_decision) + self._action_history.append("ERROR() [invalid decision]") + return BenchmarkAction( + type="error", + raw_action={ + "planner_output": planner_output, + "parse_error": "invalid_decision", + "error": f"unknown planner decision: {raw_decision!r}", + "error_type": "agent", + }, + ) + + supported_action_types = {"click", "double_click", "type", "key", "scroll"} + if not isinstance(raw_action_type, str) or ( + action_type and action_type not in supported_action_types + ): + logger.warning( + "Planner returned an unknown structured action_type: %r", + raw_action_type, + ) + self._action_history.append("ERROR() [invalid action type]") + return BenchmarkAction( + type="error", + raw_action={ + "planner_output": planner_output, + "parse_error": "invalid_action_type", + "error": f"unknown structured action type: {raw_action_type!r}", + "error_type": "agent", + }, + ) + + if action_type == "scroll" and ( + not isinstance(action_value, str) + or action_value.strip().lower() not in {"up", "down"} + ): + logger.warning("Planner returned an invalid scroll direction: %r", action_value) + self._action_history.append("ERROR() [invalid scroll direction]") + return BenchmarkAction( + type="error", + raw_action={ + "planner_output": planner_output, + "parse_error": "invalid_scroll_direction", + "error": f"invalid scroll direction: {action_value!r}", + "error_type": "agent", + }, + ) + + if action_type in {"click", "double_click"} and ( + not isinstance(target_description, str) + or not target_description.strip() + ): + logger.warning("Structured pointer action has no target description") + self._action_history.append("ERROR() [missing target description]") + return BenchmarkAction( + type="error", + raw_action={ + "planner_output": planner_output, + "parse_error": "missing_target_description", + "error": "structured pointer action requires a target description", + "error_type": "agent", + }, + ) + if not instruction and not action_type: logger.warning("Planner returned an empty instruction") self._action_history.append("ERROR() [empty instruction]") @@ -559,7 +629,7 @@ def _build_action_from_structured( return BenchmarkAction(type="key", key=key_str.lower()) if action_type == "scroll": - direction = action_value.lower() if action_value else "down" + direction = action_value.strip().lower() logger.info("Structured planner output: SCROLL %s", direction) return BenchmarkAction(type="scroll", scroll_direction=direction) @@ -813,13 +883,17 @@ def _call_grounder( logger.debug("Grounder raw output: %s", raw[:500]) + from openadapt_evals.errors import ActionParseError from openadapt_evals.training.trl_rollout import parse_action_json - action = parse_action_json(raw) + try: + action = parse_action_json(raw) + except ActionParseError: + action = BenchmarkAction(type="error") # If parsing fell through to "done" (no JSON found) and we haven't # retried yet, try once more with a simplified prompt. - if action.type == "done" and _retry: + if action.type in ("done", "error") and _retry: logger.info("Grounder parse failed, retrying with simplified prompt") simplified_prompt = ( f"Where should I click to: {instruction}\n" @@ -834,7 +908,10 @@ def _call_grounder( max_tokens=128, cost_label="grounder_retry", ) - action = parse_action_json(raw2) + try: + action = parse_action_json(raw2) + except ActionParseError: + action = BenchmarkAction(type="error") if action.type != "done": return action @@ -924,59 +1001,91 @@ def _parse_bbox_to_action(raw: str) -> BenchmarkAction: Returns BenchmarkAction with the center of the bbox. """ + import math import re - # Try JSON parse first (for non-bbox grounders) + def parse_error(reason: str) -> BenchmarkAction: + return BenchmarkAction( + type="error", + raw_action={"parse_error": reason}, + ) + + def finite_number(value: Any, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a number") + parsed = float(value) + if not math.isfinite(parsed): + raise ValueError(f"{name} must be finite") + return parsed + + # Try one strict JSON action first (for non-bbox grounders). try: - import json - data = json.loads(raw.strip()) - if isinstance(data, dict) and "x" in data and "y" in data: + data = parse_single_json_object(raw) + if data is not None: + if set(data) != {"type", "x", "y"}: + return parse_error( + "grounder JSON must contain exactly type, x, and y" + ) + if data["type"] != "click": + return parse_error("grounder JSON must contain type='click'") + x = finite_number(data["x"], "x") + y = finite_number(data["y"], "y") + if not (0.0 <= x <= 1.0 and 0.0 <= y <= 1.0): + return parse_error( + "grounder JSON coordinates must both be normalized to [0, 1]" + ) return BenchmarkAction( - type=data.get("type", "click"), - x=float(data["x"]), - y=float(data["y"]), - text=data.get("text"), - key=data.get("key"), + type="click", + x=x, + y=y, ) - except (json.JSONDecodeError, ValueError, KeyError): - pass + except ActionParseError as exc: + return parse_error(str(exc)) + except (TypeError, ValueError, KeyError) as exc: + return parse_error(str(exc)) # Find list of numbers (bbox format) - match = re.search(r"\[?\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*)\s*(?:,\s*(\d+\.?\d*)\s*,\s*(\d+\.?\d*))?\s*\]?", raw) + number = r"[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?" + match = re.fullmatch( + rf"\s*\[\s*({number})\s*,\s*({number})" + rf"(?:\s*,\s*({number})\s*,\s*({number}))?\s*\]\s*", + raw, + ) if not match: # Last resort: try parse_action_json from openadapt_evals.training.trl_rollout import parse_action_json - action = parse_action_json(raw) - if action.type == "done": - return BenchmarkAction( - type="error", - raw_action={"parse_error": "grounder output was not actionable"}, - ) + try: + action = parse_action_json(raw) + except ActionParseError: + return parse_error("grounder output was not actionable") + if action.type != "click": + return parse_error("grounder output must resolve to one click") return action nums = [float(x) for x in match.groups() if x is not None] if len(nums) == 4: - # [x1, y1, x2, y2] → center - x = (nums[0] + nums[2]) / 2 - y = (nums[1] + nums[3]) / 2 + x1, y1, x2, y2 = nums + if not all(0.0 <= value <= 1000.0 for value in nums): + return parse_error("grounder bbox values must be in [0, 1000]") + if x1 > x2 or y1 > y2: + return parse_error("grounder bbox corners are reversed") + # UI-Venus uses a fixed 1000 by 1000 canvas. + x = ((x1 + x2) / 2) / 1000 + y = ((y1 + y2) / 2) / 1000 elif len(nums) == 2: x, y = nums[0], nums[1] + normalized = 0.0 <= x <= 1.0 and 0.0 <= y <= 1.0 + canvas = 1.0 < x <= 1000.0 and 1.0 < y <= 1000.0 + if canvas: + x, y = x / 1000, y / 1000 + elif not normalized: + return parse_error( + "grounder coordinate pair mixes coordinate spaces or is out of range" + ) else: logger.warning("Unexpected number count in bbox: %s", nums) - return BenchmarkAction( - type="error", - raw_action={"parse_error": f"unexpected bbox values: {nums}"}, - ) - - # Normalize coordinates - if x > 1 and y > 1: - if x <= 1000 and y <= 1000: - # Canvas [0-1000] - x, y = x / 1000, y / 1000 - else: - # Pixel coords — leave as-is, run script handles conversion - pass + return parse_error(f"unexpected bbox values: {nums}") logger.info("Grounder: bbox=%s → click=(%.3f, %.3f)", nums, x, y) return BenchmarkAction(type="click", x=x, y=y) diff --git a/openadapt_evals/analysis/trace_analyzer.py b/openadapt_evals/analysis/trace_analyzer.py index 96fe144..22538e2 100644 --- a/openadapt_evals/analysis/trace_analyzer.py +++ b/openadapt_evals/analysis/trace_analyzer.py @@ -31,6 +31,11 @@ from pathlib import Path from typing import Any +from openadapt_evals.adapters.base import ( + UNSCORED_BENCHMARK_ERROR_TYPES, + normalize_benchmark_result_artifact, +) + logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -182,16 +187,21 @@ def _load_full_eval_jsonl(path: Path) -> tuple[list[Episode], dict[str, Any]]: if not task_id: continue + result = normalize_benchmark_result_artifact( + record, + expected_task_id=task_id, + context=f"full-eval artifact for {task_id}", + ) episodes.append( Episode( episode_id=task_id, task_id=task_id, - score=record.get("score", 0.0), - success=record.get("success", False), - num_steps=record.get("steps", 0), - elapsed_seconds=record.get("elapsed_seconds", 0.0), - error=record.get("error"), - error_type=record.get("error_type"), + score=result.score, + success=result.success, + num_steps=result.num_steps, + elapsed_seconds=result.total_time_seconds, + error=result.error, + error_type=result.error_type, started_at=record.get("started_at"), finished_at=record.get("finished_at"), milestones_passed=record.get("milestones_passed"), @@ -264,17 +274,35 @@ def _load_trajectory_dir(path: Path) -> tuple[list[Episode], dict[str, Any]]: task_instruction = rec.get("task_instruction", "") reward = records[0].get("episode_reward") - success = (reward is not None and reward > 0) if reward is not None else False - score = reward if reward is not None else 0.0 + reward_is_number = ( + not isinstance(reward, bool) + and isinstance(reward, (int, float)) + and 0.0 <= float(reward) <= 1.0 + ) + result_record = { + "task_id": eid, + "score": reward, + "success": reward_is_number and float(reward) >= 1.0, + "num_steps": len(steps), + "error_type": None if reward is not None else "evaluation", + "reason": None if reward is not None else "episode reward is missing", + } + result = normalize_benchmark_result_artifact( + result_record, + expected_task_id=eid, + context=f"trajectory artifact for {eid}", + ) episodes.append( Episode( episode_id=eid, task_id=eid, task_instruction=task_instruction, - score=score, - success=success, - num_steps=len(steps), + score=result.score, + success=result.success, + num_steps=result.num_steps, + error=result.error, + error_type=result.error_type, steps=steps, ) ) @@ -300,7 +328,7 @@ def _load_benchmark_dir(path: Path) -> tuple[list[Episode], dict[str, Any]]: task_id = task_dir.name definition: dict[str, Any] = {} - execution: dict[str, Any] = {} + execution_artifact: object = {} task_json = task_dir / "task.json" if task_json.exists(): @@ -308,7 +336,10 @@ def _load_benchmark_dir(path: Path) -> tuple[list[Episode], dict[str, Any]]: exec_json = task_dir / "execution.json" if exec_json.exists(): - execution = json.loads(exec_json.read_text(encoding="utf-8")) + execution_artifact = json.loads(exec_json.read_text(encoding="utf-8")) + execution = ( + execution_artifact if isinstance(execution_artifact, dict) else {} + ) # Collect screenshot paths screenshots_dir = task_dir / "screenshots" @@ -333,15 +364,26 @@ def _load_benchmark_dir(path: Path) -> tuple[list[Episode], dict[str, Any]]: ) ) + result = normalize_benchmark_result_artifact( + ( + {**execution, "task_id": execution.get("task_id", task_id)} + if isinstance(execution_artifact, dict) + else execution_artifact + ), + expected_task_id=task_id, + context=f"benchmark artifact for {task_id}", + ) episodes.append( Episode( episode_id=f"waa_{task_id}", task_id=task_id, task_instruction=definition.get("instruction", ""), - score=execution.get("score", 0.0), - success=execution.get("success", False), - num_steps=len(steps), - elapsed_seconds=execution.get("total_time_seconds", 0.0), + score=result.score, + success=result.success, + num_steps=result.num_steps, + elapsed_seconds=result.total_time_seconds, + error=result.error, + error_type=result.error_type, steps=steps, model=metadata.get("model_id"), raw={"definition": definition, "execution": execution}, @@ -384,6 +426,8 @@ def _load_mixed_dir(path: Path) -> tuple[list[Episode], dict[str, Any]]: episode_id=path.name, task_id=path.name, num_steps=len(steps), + error="No benchmark result artifact was present", + error_type="evaluation", steps=steps, ) return [ep], {} @@ -539,6 +583,8 @@ def summary(self) -> dict[str, Any]: if not self._episodes: return { "total_episodes": 0, + "outcome_episodes": 0, + "unscored_episodes": 0, "total_steps": 0, "success_rate": 0.0, "avg_score": 0.0, @@ -551,10 +597,16 @@ def summary(self) -> dict[str, Any]: } total = len(self._episodes) - successes = sum(1 for ep in self._episodes if ep.success) + outcomes = [ + ep + for ep in self._episodes + if ep.error_type not in UNSCORED_BENCHMARK_ERROR_TYPES + ] + outcome_total = len(outcomes) + successes = sum(1 for ep in outcomes if ep.success) total_steps = sum(ep.num_steps for ep in self._episodes) total_time = sum(ep.elapsed_seconds for ep in self._episodes) - scores = [ep.score for ep in self._episodes] + scores = [ep.score for ep in outcomes] # Status breakdown status_counts: dict[str, int] = Counter() @@ -563,6 +615,8 @@ def summary(self) -> dict[str, Any]: status_counts["passed"] += 1 elif ep.error_type == "infrastructure": status_counts["infra_error"] += 1 + elif ep.error_type == "evaluation": + status_counts["evaluation_error"] += 1 elif ep.error: status_counts["error"] += 1 else: @@ -576,9 +630,15 @@ def summary(self) -> dict[str, Any]: return { "total_episodes": total, + "outcome_episodes": outcome_total, + "unscored_episodes": total - outcome_total, "total_steps": total_steps, - "success_rate": round(successes / total, 4) if total else 0.0, - "avg_score": round(sum(scores) / total, 4) if total else 0.0, + "success_rate": ( + round(successes / outcome_total, 4) if outcome_total else 0.0 + ), + "avg_score": ( + round(sum(scores) / outcome_total, 4) if outcome_total else 0.0 + ), "avg_steps_per_episode": round(total_steps / total, 2) if total else 0.0, "avg_time_per_episode": round(total_time / total, 2) if total else 0.0, "total_time": round(total_time, 2), diff --git a/openadapt_evals/benchmarks/cli.py b/openadapt_evals/benchmarks/cli.py index 3ce2fb8..8e9883a 100644 --- a/openadapt_evals/benchmarks/cli.py +++ b/openadapt_evals/benchmarks/cli.py @@ -186,6 +186,26 @@ def _write_run_environment_metadata( metadata_path.write_text(json.dumps(metadata, indent=2)) +def _print_evaluation_metrics(metrics: dict, title: str = "Evaluation Results") -> None: + """Print benchmark outcomes without hiding unavailable attempts.""" + print("\n" + "=" * 50) + print(title) + print("=" * 50) + print(f"Attempts: {metrics['num_tasks']}") + print(f"Outcomes: {metrics['num_outcome_tasks']}") + print(f"Errors: {metrics['error_count']}") + print(f"Success rate: {metrics['success_rate']:.1%} (outcomes only)") + print(f"Avg score: {metrics['avg_score']:.3f} (outcomes only)") + print(f"Avg steps: {metrics['avg_steps']:.1f} (all attempts)") + if metrics["error_count"]: + print( + "Error types: " + f"infrastructure={metrics['num_infrastructure_failures']}, " + f"evaluation={metrics['num_evaluation_failures']}, " + f"agent={metrics['num_agent_failures']}" + ) + + def cmd_mock(args: argparse.Namespace) -> int: """Run mock evaluation (no Windows VM required).""" from openadapt_evals.agents import ApiAgent @@ -304,16 +324,7 @@ def cmd_mock(args: argparse.Namespace) -> int: # Compute and display metrics metrics = compute_metrics(results) - print("\n" + "=" * 50) - print("Evaluation Results") - print("=" * 50) - print(f"Tasks: {metrics['num_tasks']}") - print(f"Success rate: {metrics['success_rate']:.1%}") - print(f"Avg score: {metrics['avg_score']:.3f}") - print(f"Avg steps: {metrics['avg_steps']:.1f}") - if metrics.get("num_infrastructure_failures", 0): - print(f"Infra fails: {metrics['num_infrastructure_failures']}") - print(f"Adj success: {metrics.get('success_rate_excluding_infra', 0.0):.1%}") + _print_evaluation_metrics(metrics) if config: print(f"\nResults saved to: {config.output_dir}/{config.run_name}") @@ -519,16 +530,7 @@ def cmd_run(args: argparse.Namespace) -> int: # Compute and display metrics metrics = compute_metrics(results) - print("\n" + "=" * 50) - print("Evaluation Results") - print("=" * 50) - print(f"Tasks: {metrics['num_tasks']}") - print(f"Success rate: {metrics['success_rate']:.1%}") - print(f"Avg score: {metrics['avg_score']:.3f}") - print(f"Avg steps: {metrics['avg_steps']:.1f}") - if metrics.get("num_infrastructure_failures", 0): - print(f"Infra fails: {metrics['num_infrastructure_failures']}") - print(f"Adj success: {metrics.get('success_rate_excluding_infra', 0.0):.1%}") + _print_evaluation_metrics(metrics) benchmark_dir = Path(eval_config.output_dir) / eval_config.run_name _write_run_environment_metadata( benchmark_dir, @@ -779,16 +781,7 @@ def cmd_live(args: argparse.Namespace) -> int: # Compute and display metrics metrics = compute_metrics(results) - print("\n" + "=" * 50) - print("Evaluation Results") - print("=" * 50) - print(f"Tasks: {metrics['num_tasks']}") - print(f"Success rate: {metrics['success_rate']:.1%}") - print(f"Avg score: {metrics['avg_score']:.3f}") - print(f"Avg steps: {metrics['avg_steps']:.1f}") - if metrics.get("num_infrastructure_failures", 0): - print(f"Infra fails: {metrics['num_infrastructure_failures']}") - print(f"Adj success: {metrics.get('success_rate_excluding_infra', 0.0):.1%}") + _print_evaluation_metrics(metrics) if eval_config: benchmark_dir = Path(eval_config.output_dir) / eval_config.run_name @@ -1063,13 +1056,7 @@ def patch_evaluate_endpoint() -> bool: ) metrics = compute_metrics(results) - print("\n" + "=" * 50) - print("Smoke Live Results") - print("=" * 50) - print(f"Tasks: {metrics['num_tasks']}") - print(f"Success rate: {metrics['success_rate']:.1%}") - print(f"Avg score: {metrics['avg_score']:.3f}") - print(f"Avg steps: {metrics['avg_steps']:.1f}") + _print_evaluation_metrics(metrics, "Smoke Live Results") benchmark_dir = Path(eval_config.output_dir) / eval_config.run_name _write_run_environment_metadata( benchmark_dir, @@ -1079,10 +1066,6 @@ def patch_evaluate_endpoint() -> bool: evaluate_url=None, ) print(f"\nResults saved to: {benchmark_dir}") - if metrics.get("num_infrastructure_failures", 0): - print(f"Infra fails: {metrics['num_infrastructure_failures']}") - print(f"Adj success: {metrics.get('success_rate_excluding_infra', 0.0):.1%}") - return 0 finally: diff --git a/openadapt_evals/benchmarks/data_collection.py b/openadapt_evals/benchmarks/data_collection.py index e9b895f..6fb2a8c 100644 --- a/openadapt_evals/benchmarks/data_collection.py +++ b/openadapt_evals/benchmarks/data_collection.py @@ -351,31 +351,26 @@ def save_summary(self, all_results: list[BenchmarkResult]) -> None: Args: all_results: List of all BenchmarkResult objects from the run. """ + # Keep one aggregation contract across CLI, traces, and integrations. + # Import locally to avoid a module cycle during runner initialization. + from openadapt_evals.adapters import normalize_benchmark_result + from openadapt_evals.benchmarks.runner import compute_metrics + + all_results = [ + normalize_benchmark_result(result, context=f"trace result {index}") + for index, result in enumerate(all_results) + ] + + metrics = compute_metrics(all_results) summary = { "benchmark_name": self.benchmark_name, "run_name": self.run_name, "model_id": self.model_id, - "num_tasks": len(all_results), - "num_success": sum(1 for r in all_results if r.success), - "success_rate": sum(1 for r in all_results if r.success) / len(all_results) if all_results else 0.0, - "avg_score": sum(r.score for r in all_results) / len(all_results) if all_results else 0.0, - "avg_steps": sum(r.num_steps for r in all_results) / len(all_results) if all_results else 0.0, - "avg_time_seconds": sum(r.total_time_seconds for r in all_results) / len(all_results) if all_results else 0.0, - "num_infrastructure_failures": sum( - 1 for r in all_results if r.error_type == "infrastructure" - ), - "num_tasks_excluding_infra": sum( - 1 for r in all_results if r.error_type != "infrastructure" - ), - "num_success_excluding_infra": sum( - 1 for r in all_results if r.error_type != "infrastructure" and r.success - ), - "success_rate_excluding_infra": ( - sum(1 for r in all_results if r.error_type != "infrastructure" and r.success) - / sum(1 for r in all_results if r.error_type != "infrastructure") - if any(r.error_type != "infrastructure" for r in all_results) - else 0.0 - ), + **metrics, + # Existing trace consumers use these names. Evaluator and + # infrastructure outages do not enter the outcome denominator. + "num_success": metrics["success_count"], + "num_success_excluding_infra": metrics["success_count"], "tasks": [ { "task_id": r.task_id, @@ -395,8 +390,10 @@ def save_summary(self, all_results: list[BenchmarkResult]) -> None: json.dump(summary, f, indent=2) logger.info( - f"Saved summary: {summary['num_success']}/{summary['num_tasks']} tasks succeeded " - f"({summary['success_rate']:.1%})" + f"Saved summary: {summary['num_success']}/" + f"{summary['num_outcome_tasks']} outcomes succeeded " + f"({summary['success_rate']:.1%}); {summary['error_count']} errors " + f"across {summary['num_tasks']} attempts" ) def _save_screenshot(self, step_idx: int, screenshot_bytes: bytes) -> str: diff --git a/openadapt_evals/benchmarks/pool_viewer.py b/openadapt_evals/benchmarks/pool_viewer.py index 8ce5995..c653bbb 100644 --- a/openadapt_evals/benchmarks/pool_viewer.py +++ b/openadapt_evals/benchmarks/pool_viewer.py @@ -20,6 +20,35 @@ from pathlib import Path from typing import Any +from openadapt_evals.adapters.base import ( + UNSCORED_BENCHMARK_ERROR_TYPES, + normalize_benchmark_result_artifact, +) + + +def _normalize_pool_task_artifact(task: object) -> tuple[dict[str, Any], bool]: + """Return a display row and whether it is a measured outcome.""" + raw = dict(task) if isinstance(task, dict) else {} + if "score" not in raw and "result" in raw: + raw["score"] = raw["result"] + task_id = str(raw.get("task_id") or "unknown") + result = normalize_benchmark_result_artifact( + raw, + expected_task_id=task_id, + context=f"pool artifact for {task_id}", + ) + raw.update( + { + "task_id": result.task_id, + "result": result.score, + "score": result.score, + "success": result.success, + "error": result.error, + "reason": result.reason, + "error_type": result.error_type, + } + ) + return raw, result.error_type not in UNSCORED_BENCHMARK_ERROR_TYPES def parse_pool_logs(pool_dir: Path) -> dict[str, Any]: """Parse WAA pool log files to extract task results. @@ -61,7 +90,13 @@ def parse_pool_logs(pool_dir: Path) -> dict[str, Any]: for log_file in log_files: worker_id = log_file.stem.replace("waa-pool-", "") - workers[worker_id] = {"tasks": 0, "successes": 0, "failures": 0} + workers[worker_id] = { + "tasks": 0, + "outcomes": 0, + "successes": 0, + "failures": 0, + "unscored": 0, + } current_task = None last_result = None @@ -133,15 +168,35 @@ def parse_pool_logs(pool_dir: Path) -> dict[str, Any]: current_task["domain"] = domain current_task["task_id"] = task_id - current_task["result"] = last_result if last_result is not None else 0.0 - current_task["success"] = last_result is not None and last_result > 0 + current_task.update( + { + "result": last_result, + "score": last_result, + "success": last_result is not None and last_result >= 1.0, + "error_type": ( + None if last_result is not None else "evaluation" + ), + "reason": ( + None + if last_result is not None + else "pool log did not contain a result" + ), + } + ) current_task["timestamp"] = metadata["last_timestamp"] + current_task, is_scored = _normalize_pool_task_artifact( + current_task + ) # Update worker stats workers[worker_id]["tasks"] += 1 - if current_task["success"]: + if not is_scored: + workers[worker_id]["unscored"] += 1 + elif current_task["success"]: + workers[worker_id]["outcomes"] += 1 workers[worker_id]["successes"] += 1 else: + workers[worker_id]["outcomes"] += 1 workers[worker_id]["failures"] += 1 tasks.append(current_task) @@ -162,12 +217,23 @@ def get_domain_stats(tasks: list[dict]) -> dict[str, dict[str, int]]: for task in tasks: domain = task.get("domain", "unknown") if domain not in domain_stats: - domain_stats[domain] = {"total": 0, "success": 0, "fail": 0} + domain_stats[domain] = { + "total": 0, + "outcomes": 0, + "success": 0, + "fail": 0, + "unscored": 0, + } domain_stats[domain]["total"] += 1 - if task.get("success"): + result, is_scored = _normalize_pool_task_artifact(task) + if not is_scored: + domain_stats[domain]["unscored"] += 1 + elif result["success"]: + domain_stats[domain]["outcomes"] += 1 domain_stats[domain]["success"] += 1 else: + domain_stats[domain]["outcomes"] += 1 domain_stats[domain]["fail"] += 1 return domain_stats @@ -192,14 +258,23 @@ def generate_pool_results_viewer( # Parse logs data = parse_pool_logs(pool_dir) - tasks = data["tasks"] + tasks = [_normalize_pool_task_artifact(task)[0] for task in data["tasks"]] workers = data["workers"] metadata = data["metadata"] # Calculate stats num_tasks = len(tasks) - num_success = sum(1 for t in tasks if t.get("success")) - success_rate = (num_success / num_tasks * 100) if num_tasks > 0 else 0 + outcome_tasks = [ + task + for task in tasks + if task.get("error_type") not in UNSCORED_BENCHMARK_ERROR_TYPES + ] + num_outcomes = len(outcome_tasks) + num_unscored = num_tasks - num_outcomes + num_success = sum(1 for task in outcome_tasks if task["success"] is True) + success_rate = ( + num_success / num_outcomes * 100 if num_outcomes > 0 else 0 + ) # Domain stats domain_stats = get_domain_stats(tasks) @@ -246,6 +321,8 @@ def generate_pool_results_viewer( metadata=metadata, domain_stats=domain_stats, num_tasks=num_tasks, + num_outcomes=num_outcomes, + num_unscored=num_unscored, num_success=num_success, success_rate=success_rate, elapsed_str=elapsed_str, @@ -266,6 +343,8 @@ def _generate_pool_viewer_html( metadata: dict, domain_stats: dict, num_tasks: int, + num_outcomes: int, + num_unscored: int, num_success: int, success_rate: float, elapsed_str: str, @@ -276,13 +355,18 @@ def _generate_pool_viewer_html( # Worker rows HTML worker_rows = "" for worker_id, stats in sorted(workers.items()): - rate = (stats["successes"] / stats["tasks"] * 100) if stats["tasks"] > 0 else 0 + rate = ( + stats["successes"] / stats["outcomes"] * 100 + if stats["outcomes"] > 0 + else 0 + ) worker_rows += f""" Worker {worker_id} {stats["tasks"]} {stats["successes"]} {stats["failures"]} + {stats["unscored"]} {rate:.1f}% """ @@ -291,19 +375,32 @@ def _generate_pool_viewer_html( domain_tags = "" for domain in sorted(domain_stats.keys()): stats = domain_stats[domain] - rate = (stats["success"] / stats["total"] * 100) if stats["total"] > 0 else 0 + rate = ( + stats["success"] / stats["outcomes"] * 100 + if stats["outcomes"] > 0 + else 0 + ) domain_tags += f"""
{domain} - {stats["success"]}/{stats["total"]} ({rate:.0f}%) + {stats["success"]}/{stats["outcomes"]} ({rate:.0f}%) · {stats["total"]} attempts
""" # Task rows HTML task_rows = "" for i, task in enumerate(tasks): - status_class = "success" if task.get("success") else "fail" - status_text = "PASS" if task.get("success") else "FAIL" + is_unscored = task.get("error_type") in UNSCORED_BENCHMARK_ERROR_TYPES + status_class = ( + "success" + if task["success"] is True + else ("unscored" if is_unscored else "fail") + ) + status_text = ( + "PASS" + if task["success"] is True + else ("UNSCORED" if is_unscored else "FAIL") + ) result = task.get("result", 0) task_rows += f""" @@ -492,6 +589,10 @@ def _generate_pool_viewer_html( background: rgba(255, 95, 95, 0.2); color: var(--error); }} + .status-badge.unscored {{ + background: rgba(245, 158, 11, 0.2); + color: #f59e0b; + }} .domain-badge {{ font-size: 0.75rem; color: var(--accent); @@ -567,14 +668,22 @@ def _generate_pool_viewer_html(
{num_tasks}
Total Tasks
+
+
{num_outcomes}
+
Measured Outcomes
+
{num_success}
Passed
-
{num_tasks - num_success}
+
{num_outcomes - num_success}
Failed
+
+
{num_unscored}
+
Unscored Errors
+
= 50 else "error"}">{success_rate:.1f}%
Success Rate
@@ -601,6 +710,7 @@ def _generate_pool_viewer_html( Tasks Passed Failed + Unscored Success Rate @@ -628,6 +738,7 @@ def _generate_pool_viewer_html( +
{num_tasks} tasks diff --git a/openadapt_evals/benchmarks/runner.py b/openadapt_evals/benchmarks/runner.py index 29d2310..7615158 100644 --- a/openadapt_evals/benchmarks/runner.py +++ b/openadapt_evals/benchmarks/runner.py @@ -27,8 +27,11 @@ BenchmarkObservation, BenchmarkResult, BenchmarkTask, + EvaluationUnavailableError, + normalize_benchmark_result, ) from openadapt_evals.agents import BenchmarkAgent +from openadapt_evals.errors import RolloutEvaluationError, RolloutInfrastructureError from openadapt_evals.telemetry import ( track_action_executed, track_agent_run, @@ -41,6 +44,22 @@ logger = logging.getLogger(__name__) +_ERROR_TYPES = frozenset({"agent", "infrastructure", "evaluation"}) + + +def _exception_error_type(error: Exception, fallback: str) -> str: + """Return one allowed error type for an escaped exception.""" + claimed = getattr(error, "error_type", None) + if claimed is None: + claimed = fallback + if isinstance(claimed, str) and claimed in _ERROR_TYPES: + return claimed + logger.warning( + "Exception claimed unknown error_type=%r; classifying it as evaluation", + claimed, + ) + return "evaluation" + @dataclass class EvaluationConfig: @@ -155,6 +174,11 @@ def evaluate_agent_on_benchmark( else: results = _evaluate_sequential(agent, adapter, tasks, config, trace_collector, live_tracker) + results = [ + normalize_benchmark_result(result, context=f"returned result {index}") + for index, result in enumerate(results) + ] + # Save summary if trace collection is enabled if trace_collector is not None: trace_collector.save_summary(results) @@ -163,23 +187,27 @@ def evaluate_agent_on_benchmark( if live_tracker is not None: live_tracker.finish() - # Log summary + metrics = compute_metrics(results) + success_count = metrics["success_count"] + avg_steps = metrics["avg_steps"] + + # Exclude evaluator and infrastructure outages from the outcome rate. + # Agent errors remain failed outcomes and also remain visible as errors. if config.verbose: - success_count = sum(1 for r in results if r.success) - success_rate = success_count / len(results) if results else 0 - avg_steps = sum(r.num_steps for r in results) / len(results) if results else 0 logger.info( - f"Evaluation complete: {success_count}/{len(results)} " - f"({success_rate:.1%}) success, {avg_steps:.1f} avg steps" + f"Evaluation complete: {success_count}/{metrics['num_outcome_tasks']} " + f"outcomes ({metrics['success_rate']:.1%}) success; " + f"{metrics['error_count']} errors across {metrics['num_tasks']} attempts; " + f"{avg_steps:.1f} avg steps" ) - else: - success_count = sum(1 for r in results if r.success) - avg_steps = sum(r.num_steps for r in results) / len(results) if results else 0 track_agent_run_completed( adapter=adapter.name, agent_class=type(agent).__name__, - num_tasks=len(results), + num_tasks=metrics["num_tasks"], + attempt_count=metrics["num_tasks"], + outcome_count=metrics["num_outcome_tasks"], + error_count=metrics["error_count"], success_count=success_count, avg_steps=round(avg_steps, 2), run_name=config.run_name or "unspecified", @@ -214,7 +242,13 @@ def _evaluate_sequential( if config.verbose: logger.info(f"Task {i + 1}/{len(tasks)}: {task.task_id}") - result = _run_single_task(agent, adapter, task, config, trace_collector, live_tracker) + try: + result = _run_single_task( + agent, adapter, task, config, trace_collector, live_tracker + ) + except Exception as e: + logger.error(f"Task {task.task_id} failed with error: {e}") + result = _failed_task_result(task, e) results.append(result) if config.on_task_complete: @@ -272,18 +306,23 @@ def _evaluate_parallel( except Exception as e: logger.error(f"Task {task.task_id} failed with error: {e}") - results.append( - BenchmarkResult( - task_id=task.task_id, - success=False, - score=0.0, - error=str(e), - ) - ) + results.append(_failed_task_result(task, e)) return results +def _failed_task_result(task: BenchmarkTask, error: Exception) -> BenchmarkResult: + """Convert an escaped worker error into an explicit unmeasured attempt.""" + return BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + error=str(error), + reason=str(error), + error_type=_exception_error_type(error, "infrastructure"), + ) + + def _run_single_task( agent: BenchmarkAgent, adapter: BenchmarkAdapter, @@ -307,6 +346,7 @@ def _run_single_task( """ start_time = time.perf_counter() history: list[tuple[BenchmarkObservation, BenchmarkAction]] = [] + failure_error_type = "infrastructure" # Start trace collection if enabled if trace_collector is not None: @@ -334,6 +374,7 @@ def _run_single_task( # Get action from agent try: + failure_error_type = "agent" think_start = time.perf_counter() action = agent.act(obs, task, history if config.save_trajectories else None) think_end = time.perf_counter() @@ -390,26 +431,30 @@ def _run_single_task( f"(override {done_gate_overrides + 1}/{config.done_gate_max_overrides})" ) try: - gate_result = adapter.evaluate(task) - gate_score = gate_result.score + failure_error_type = "evaluation" + gate_result = normalize_benchmark_result( + adapter.evaluate(task), + expected_task_id=task.task_id, + context="done-gate result", + ) except Exception as e: - logger.warning( - f"Step {steps}: Done-gate evaluation failed: {e}. " - "Accepting 'done' to avoid infinite loop." + if getattr(e, "error_type", None) is not None: + raise + raise RolloutEvaluationError( + f"Done-gate evaluation failed: {e}" + ) from e + + if gate_result.error_type is not None: + message = ( + gate_result.reason + or gate_result.error + or "done-gate evaluation did not produce a measured result" ) - done = True - break - - # If evaluate endpoint is unreachable, accept "done" - # rather than forcing the agent to continue pointlessly - if gate_result.error_type == "infrastructure": - logger.warning( - f"Step {steps}: Done-gate skipped — evaluate " - f"returned infrastructure error: {gate_result.reason}. " - "Accepting 'done'." + raise EvaluationUnavailableError( + str(message), error_type=gate_result.error_type ) - done = True - break + + gate_score = gate_result.score if gate_score >= config.done_gate_threshold: logger.info( @@ -450,23 +495,28 @@ def _run_single_task( task.instruction = task.instruction[:marker_idx] task.instruction = task.instruction + continuation_msg - # Get a fresh observation for the agent's next step - # Use a no-op key press to trigger a new screenshot + # Refresh only through an observation-only adapter method. + # Never synthesize input to request a screenshot. + failure_error_type = "infrastructure" + observe = getattr(adapter, "observe", None) + if not callable(observe): + raise RolloutInfrastructureError( + "Done-gate rejected completion, but the adapter cannot " + "provide a fresh observation" + ) try: - noop_action = BenchmarkAction(type="key", key="") - obs, env_done, _info = adapter.step(noop_action) - if env_done: - logger.info( - f"Step {steps}: Environment signaled done during " - "done-gate screenshot refresh" - ) - done = True - break + refreshed_obs = observe() except Exception as e: - logger.warning( - f"Step {steps}: Failed to get fresh observation " - f"after done-gate override: {e}. Using previous obs." + raise RolloutInfrastructureError( + "Done-gate rejected completion, but the fresh observation " + f"failed: {e}" + ) from e + if not isinstance(refreshed_obs, BenchmarkObservation): + raise RolloutInfrastructureError( + "Done-gate rejected completion, but the adapter returned " + "an invalid fresh observation" ) + obs = refreshed_obs steps += 1 continue @@ -481,6 +531,7 @@ def _run_single_task( # Execute action try: + failure_error_type = "infrastructure" logger.info(f"Step {steps}: Executing action in environment") exec_start = time.perf_counter() obs, done, info = adapter.step(action) @@ -505,16 +556,20 @@ def _run_single_task( if steps >= max_steps: logger.warning(f"Task reached maximum steps ({max_steps})") - # Evaluate result - logger.info("Evaluating task result") - result = adapter.evaluate(task) - # An evaluator observes target state, not whether the agent itself # completed this attempt safely. A pre-existing target state can score - # as successful even after the agent reported a provider or parse - # failure, so the terminal agent error remains authoritative. + # as successful after an agent error. Do not call the evaluator for a + # terminal error because evaluator unavailability must not hide a valid + # agent failure from the benchmark denominator. if action is not None and action.type == "error": raw_action = action.raw_action or {} + claimed_error_type = raw_action.get("error_type") + if claimed_error_type not in (None, "agent"): + logger.warning( + "Agent terminal error claimed error_type=%r; classifying the " + "attempt as an agent outcome", + claimed_error_type, + ) error_reason = ( raw_action.get("reason") or raw_action.get("error") @@ -522,16 +577,37 @@ def _run_single_task( or raw_action.get("fail_reason") or "agent reported a terminal error" ) - result.success = False - result.score = 0.0 - result.error_type = raw_action.get("error_type", "agent") - result.error = str(error_reason) - result.reason = str(error_reason) + result = BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + error_type="agent", + error=str(error_reason), + reason=str(error_reason), + ) + else: + logger.info("Evaluating task result") + failure_error_type = "evaluation" + try: + result = normalize_benchmark_result( + adapter.evaluate(task), + expected_task_id=task.task_id, + context="final evaluator result", + ) + except Exception as e: + if getattr(e, "error_type", None) is not None: + raise + raise RolloutEvaluationError(f"Task evaluation failed: {e}") from e # Update result with trajectory info result.steps = history if config.save_trajectories else [] result.num_steps = steps result.total_time_seconds = time.perf_counter() - start_time + result = normalize_benchmark_result( + result, + expected_task_id=task.task_id, + context="final task result", + ) # Log final result if result.success: @@ -553,7 +629,7 @@ def _run_single_task( except Exception as e: logger.error(f"Error running task {task.task_id}: {e}") - error_type = getattr(e, "error_type", None) + error_type = _exception_error_type(e, failure_error_type) result = BenchmarkResult( task_id=task.task_id, success=False, @@ -570,11 +646,15 @@ def _run_single_task( if trace_collector is not None: trace_collector.finish_task(result) - return result + return normalize_benchmark_result( + result, + expected_task_id=task.task_id, + context="escaped task result", + ) def compute_metrics(results: list[BenchmarkResult]) -> dict: - """Compute aggregate metrics from evaluation results. + """Compute outcomes without scoring unavailable evaluation attempts. Args: results: List of BenchmarkResult from evaluation. @@ -585,36 +665,61 @@ def compute_metrics(results: list[BenchmarkResult]) -> dict: if not results: return { "num_tasks": 0, + "num_attempts": 0, + "num_outcome_tasks": 0, + "error_count": 0, "success_rate": 0.0, "avg_score": 0.0, "avg_steps": 0.0, "avg_time_seconds": 0.0, "num_infrastructure_failures": 0, + "num_evaluation_failures": 0, + "num_agent_failures": 0, "num_tasks_excluding_infra": 0, "success_rate_excluding_infra": 0.0, + "success_count": 0, + "fail_count": 0, } + results = [ + normalize_benchmark_result(result, context=f"aggregate result {index}") + for index, result in enumerate(results) + ] num_tasks = len(results) - success_count = sum(1 for r in results if r.success) - total_score = sum(r.score for r in results) + unavailable_types = {"infrastructure", "evaluation"} + outcomes = [ + result for result in results if result.error_type not in unavailable_types + ] + success_count = sum(1 for result in outcomes if result.success) + total_score = sum(result.score for result in outcomes) total_steps = sum(r.num_steps for r in results) total_time = sum(r.total_time_seconds for r in results) infra_failures = [r for r in results if r.error_type == "infrastructure"] - non_infra = [r for r in results if r.error_type != "infrastructure"] - non_infra_success = sum(1 for r in non_infra if r.success) + evaluation_failures = [r for r in results if r.error_type == "evaluation"] + agent_failures = [r for r in results if r.error_type == "agent"] + outcome_count = len(outcomes) + outcome_fail_count = outcome_count - success_count + error_count = sum(1 for result in results if result.error_type is not None) return { "num_tasks": num_tasks, - "success_rate": success_count / num_tasks, - "avg_score": total_score / num_tasks, + "num_attempts": num_tasks, + "num_outcome_tasks": outcome_count, + "error_count": error_count, + "success_rate": success_count / outcome_count if outcomes else 0.0, + "avg_score": total_score / outcome_count if outcomes else 0.0, "avg_steps": total_steps / num_tasks, "avg_time_seconds": total_time / num_tasks, "success_count": success_count, - "fail_count": num_tasks - success_count, + "fail_count": outcome_fail_count, "num_infrastructure_failures": len(infra_failures), - "num_tasks_excluding_infra": len(non_infra), + "num_evaluation_failures": len(evaluation_failures), + "num_agent_failures": len(agent_failures), + # Compatibility aliases. Evaluator and infrastructure outages are + # excluded. Agent errors remain failed benchmark outcomes. + "num_tasks_excluding_infra": outcome_count, "success_rate_excluding_infra": ( - non_infra_success / len(non_infra) if non_infra else 0.0 + success_count / outcome_count if outcomes else 0.0 ), } diff --git a/openadapt_evals/benchmarks/viewer.py b/openadapt_evals/benchmarks/viewer.py index d3a034f..08c37ea 100644 --- a/openadapt_evals/benchmarks/viewer.py +++ b/openadapt_evals/benchmarks/viewer.py @@ -37,6 +37,10 @@ from pathlib import Path from typing import Any +from openadapt_evals.adapters.base import ( + UNSCORED_BENCHMARK_ERROR_TYPES, + normalize_benchmark_result_artifact, +) from openadapt_evals.shared_ui import ( get_keyboard_shortcuts_css, get_keyboard_shortcuts_js, @@ -45,6 +49,32 @@ logger = logging.getLogger(__name__) +def _normalize_execution_artifact( + execution: object, + task_id: str, +) -> dict[str, Any]: + """Preserve trace fields while normalizing its result envelope.""" + raw = dict(execution) if isinstance(execution, dict) else {} + result = normalize_benchmark_result_artifact( + execution, + expected_task_id=task_id, + context=f"viewer execution artifact for {task_id}", + ) + raw.update( + { + "task_id": result.task_id, + "success": result.success, + "score": result.score, + "num_steps": result.num_steps, + "total_time_seconds": result.total_time_seconds, + "error": result.error, + "reason": result.reason, + "error_type": result.error_type, + } + ) + return raw + + def load_benchmark_metadata(benchmark_dir: Path) -> dict[str, Any]: """Load benchmark metadata from metadata.json. @@ -125,9 +155,13 @@ def load_task_results(benchmark_dir: Path) -> list[dict[str, Any]]: execution_json = task_dir / "execution.json" if execution_json.exists(): with open(execution_json) as f: - task_data["execution"] = json.load(f) + execution = json.load(f) else: - task_data["execution"] = {"steps": [], "success": False, "num_steps": 0} + execution = None + task_data["execution"] = _normalize_execution_artifact( + execution, + task_dir.name, + ) # Load screenshot paths screenshots_dir = task_dir / "screenshots" @@ -174,15 +208,30 @@ def _get_domain_stats(tasks: list[dict[str, Any]]) -> dict[str, dict[str, int]]: for task in tasks: domain = task.get("definition", {}).get("domain", "unknown") - success = task.get("execution", {}).get("success", False) + task_id = str(task.get("task_id") or "unknown") + result = normalize_benchmark_result_artifact( + task.get("execution"), + expected_task_id=task_id, + context=f"viewer domain artifact for {task_id}", + ) if domain not in domain_stats: - domain_stats[domain] = {"total": 0, "success": 0, "fail": 0} + domain_stats[domain] = { + "total": 0, + "outcomes": 0, + "success": 0, + "fail": 0, + "unscored": 0, + } domain_stats[domain]["total"] += 1 - if success: + if result.error_type in UNSCORED_BENCHMARK_ERROR_TYPES: + domain_stats[domain]["unscored"] += 1 + elif result.success: + domain_stats[domain]["outcomes"] += 1 domain_stats[domain]["success"] += 1 else: + domain_stats[domain]["outcomes"] += 1 domain_stats[domain]["fail"] += 1 return domain_stats @@ -371,21 +420,26 @@ def _generate_benchmark_viewer_html( # Calculate aggregate metrics num_tasks = len(tasks) - num_success = sum(1 for t in tasks if t.get("execution", {}).get("success", False)) - success_rate = (num_success / num_tasks * 100) if num_tasks > 0 else 0 - num_infra = sum( - 1 - for t in tasks - if (t.get("execution", {}) or {}).get("error_type") == "infrastructure" + executions = [ + normalize_benchmark_result_artifact( + task.get("execution"), + expected_task_id=str(task.get("task_id") or "unknown"), + context="viewer aggregate artifact", + ) + for task in tasks + ] + outcomes = [ + result + for result in executions + if result.error_type not in UNSCORED_BENCHMARK_ERROR_TYPES + ] + num_outcomes = len(outcomes) + num_success = sum(1 for result in outcomes if result.success) + num_failed = num_outcomes - num_success + num_unscored = num_tasks - num_outcomes + success_rate = ( + num_success / num_outcomes * 100 if num_outcomes > 0 else 0 ) - num_non_infra = max(0, num_tasks - num_infra) - non_infra_success = sum( - 1 - for t in tasks - if (t.get("execution", {}) or {}).get("error_type") != "infrastructure" - and (t.get("execution", {}) or {}).get("success", False) - ) - adj_success_rate = (non_infra_success / num_non_infra * 100) if num_non_infra > 0 else 0 body_class = ' class="compact"' if compact else '' @@ -654,6 +708,10 @@ def _generate_benchmark_viewer_html( background: rgba(245, 158, 11, 0.2); color: #f59e0b; }} + .task-item .task-status.unscored {{ + background: rgba(245, 158, 11, 0.2); + color: #f59e0b; + }} .task-item .task-info {{ font-size: 0.75rem; color: var(--text-secondary); @@ -1177,26 +1235,26 @@ def _generate_benchmark_viewer_html(
{num_tasks}
Total Tasks
+
+
{num_outcomes}
+
Measured Outcomes
+
{num_success}
Passed
-
{num_tasks - num_success}
+
{num_failed}
Failed
-
{num_infra}
-
Infra Fails
+
{num_unscored}
+
Unscored Errors
{success_rate:.1f}%
Success Rate
-
-
{adj_success_rate:.1f}%
-
Adj Success
-
@@ -1226,6 +1284,7 @@ def _generate_benchmark_viewer_html( + {num_tasks} tasks @@ -1282,11 +1341,11 @@ def _generate_benchmark_viewer_html( const container = document.getElementById('domain-breakdown'); let html = ''; for (const [domain, stats] of Object.entries(domainStats)) {{ - const rate = stats.total > 0 ? (stats.success / stats.total * 100).toFixed(0) : 0; + const rate = stats.outcomes > 0 ? (stats.success / stats.outcomes * 100).toFixed(0) : 0; html += `
${{domain}} - ${{stats.success}}/${{stats.total}} (${{rate}}%) + ${{stats.success}}/${{stats.outcomes}} (${{rate}}%) · ${{stats.total}} attempts
`; }} @@ -1309,11 +1368,12 @@ def _generate_benchmark_viewer_html( tasks.forEach((task, idx) => {{ const def = task.definition || {{}}; const exec = task.execution || {{}}; - const success = exec.success || false; + const success = exec.success === true; const errorType = exec.error_type || ''; const isInfra = errorType === 'infrastructure'; - const statusKey = success ? 'success' : (isInfra ? 'infra' : 'fail'); - const statusLabel = success ? 'PASS' : (isInfra ? 'INFRA' : 'FAIL'); + const isUnscored = isInfra || errorType === 'evaluation'; + const statusKey = success ? 'success' : (isInfra ? 'infra' : (isUnscored ? 'unscored' : 'fail')); + const statusLabel = success ? 'PASS' : (isInfra ? 'INFRA' : (isUnscored ? 'UNSCORED' : 'FAIL')); const domain = def.domain || 'unknown'; const numSteps = exec.num_steps || 0; @@ -1473,10 +1533,11 @@ def _generate_benchmark_viewer_html( const def = task.definition || {{}}; const exec = task.execution || {{}}; const steps = exec.steps || []; - const success = exec.success || false; + const success = exec.success === true; const isInfra = (exec.error_type || '') === 'infrastructure'; - const statusColor = success ? 'var(--success)' : (isInfra ? '#f59e0b' : 'var(--error)'); - const statusLabel = success ? 'PASSED' : (isInfra ? 'INFRA FAILURE' : 'FAILED'); + const isUnscored = isInfra || (exec.error_type || '') === 'evaluation'; + const statusColor = success ? 'var(--success)' : (isUnscored ? '#f59e0b' : 'var(--error)'); + const statusLabel = success ? 'PASSED' : (isInfra ? 'INFRA ERROR' : (isUnscored ? 'UNSCORED' : 'FAILED')); const container = document.getElementById('task-detail-content'); container.innerHTML = ` @@ -2192,11 +2253,6 @@ def _generate_benchmark_viewer_html( if (liveData.total_tasks) {{ document.querySelector('.stat-card:nth-child(1) .stat-value').textContent = liveData.total_tasks; }} - if (liveData.tasks_completed !== undefined) {{ - document.querySelector('.stat-card:nth-child(2) .stat-value').textContent = liveData.tasks_completed; - const failed = liveData.total_tasks - liveData.tasks_completed; - document.querySelector('.stat-card:nth-child(3) .stat-value').textContent = failed; - }} // Show live indicator let liveIndicator = document.getElementById('live-indicator'); diff --git a/openadapt_evals/demo_controller.py b/openadapt_evals/demo_controller.py index e190faa..2460dbd 100644 --- a/openadapt_evals/demo_controller.py +++ b/openadapt_evals/demo_controller.py @@ -48,6 +48,7 @@ BenchmarkObservation, BenchmarkResult, BenchmarkTask, + normalize_benchmark_result, ) from openadapt_evals.agents.base import BenchmarkAgent from openadapt_evals.agents.claude_computer_use_agent import _parse_multilevel_demo @@ -245,8 +246,30 @@ def execute( # Reset agent and environment logger.info("Resetting agent and environment for task %s", task.task_id) - self.agent.reset() - obs = self.adapter.reset(task) + try: + self.agent.reset() + except Exception as e: + logger.error("Agent reset failed: %s", e) + return self._failure_result( + task, + start_time, + history, + 0, + f"Agent reset error: {e}", + "agent", + ) + try: + obs = self.adapter.reset(task) + except Exception as e: + logger.error("Environment reset failed: %s", e) + return self._failure_result( + task, + start_time, + history, + 0, + f"Environment reset error: {e}", + "infrastructure", + ) for step_idx in range(max_steps): self.plan_state.total_attempts += 1 @@ -257,13 +280,23 @@ def execute( "All %d plan steps completed, verifying goal", len(self.plan_state.steps), ) - if self._verify_goal(obs): + try: + goal_verified = self._verify_goal(obs) + except Exception as e: + logger.error("Goal verification failed: %s", e) + return self._failure_result( + task, + start_time, + history, + step_idx, + f"Goal verification error: {e}", + self._evaluator_error_type(e), + ) + if goal_verified: logger.info("[SUCCESS] Goal verified after %d agent steps", step_idx) - result = self.adapter.evaluate(task) - result.steps = history - result.num_steps = step_idx - result.total_time_seconds = time.perf_counter() - start_time - return result + return self._evaluate_task( + task, start_time, history, step_idx + ) else: logger.warning( "All steps done but goal not verified; " @@ -317,6 +350,8 @@ def execute( steps=history, num_steps=step_idx, error=f"Agent error: {e}", + reason=f"Agent error: {e}", + error_type="agent", total_time_seconds=time.perf_counter() - start_time, ) @@ -341,18 +376,33 @@ def execute( # Handle error actions if action.type == "error": logger.error("Agent returned error: %s", action.raw_action) + raw_action = ( + action.raw_action if isinstance(action.raw_action, dict) else {} + ) + claimed_error_type = raw_action.get("error_type") + if claimed_error_type not in (None, "agent"): + logger.warning( + "Agent terminal error claimed error_type=%r; classifying the " + "attempt as an agent outcome", + claimed_error_type, + ) + error_reason = ( + raw_action.get("reason") + or raw_action.get("error") + or raw_action.get("parse_error") + or raw_action.get("fail_reason") + or action.raw_action + or "agent reported a terminal error" + ) return BenchmarkResult( task_id=task.task_id, success=False, score=0.0, steps=history, num_steps=step_idx, - error=str(action.raw_action), - error_type=( - action.raw_action.get("error_type", "agent") - if isinstance(action.raw_action, dict) - else "agent" - ), + error=str(error_reason), + reason=str(error_reason), + error_type="agent", total_time_seconds=time.perf_counter() - start_time, ) @@ -368,6 +418,8 @@ def execute( steps=history, num_steps=step_idx, error=f"Environment error: {e}", + reason=f"Environment error: {e}", + error_type="infrastructure", total_time_seconds=time.perf_counter() - start_time, ) @@ -376,7 +428,18 @@ def execute( # Verify step completion screenshot_bytes = self._get_screenshot_bytes(obs) if screenshot_bytes is not None: - vr = self._verify_step(screenshot_bytes, current.expect) + try: + vr = self._verify_step(screenshot_bytes, current.expect) + except Exception as e: + logger.error("Step verification failed: %s", e) + return self._failure_result( + task, + start_time, + history, + step_idx + 1, + f"Step verification error: {e}", + self._evaluator_error_type(e), + ) current.verification_result = vr if vr.effectively_verified: @@ -437,11 +500,71 @@ def execute( if step_idx >= max_steps - 1: logger.warning("Reached maximum steps (%d)", max_steps) - result = self.adapter.evaluate(task) + return self._evaluate_task(task, start_time, history, len(history)) + + @staticmethod + def _evaluator_error_type(error: Exception) -> str: + """Preserve only typed evaluator availability classifications.""" + error_type = getattr(error, "error_type", "evaluation") + if error_type not in ("infrastructure", "evaluation"): + return "evaluation" + return error_type + + @staticmethod + def _failure_result( + task: BenchmarkTask, + start_time: float, + history: list[tuple[BenchmarkObservation, BenchmarkAction]], + num_steps: int, + message: str, + error_type: str, + ) -> BenchmarkResult: + """Return one explicit failed result for an execution boundary error.""" + return BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + steps=history, + num_steps=num_steps, + error=message, + reason=message, + error_type=error_type, + total_time_seconds=time.perf_counter() - start_time, + ) + + def _evaluate_task( + self, + task: BenchmarkTask, + start_time: float, + history: list[tuple[BenchmarkObservation, BenchmarkAction]], + num_steps: int, + ) -> BenchmarkResult: + """Evaluate the task and keep evaluator failures unscored.""" + try: + result = self.adapter.evaluate(task) + except Exception as e: + logger.error("Task evaluation failed: %s", e) + return self._failure_result( + task, + start_time, + history, + num_steps, + f"Evaluation error: {e}", + self._evaluator_error_type(e), + ) + result = normalize_benchmark_result( + result, + expected_task_id=task.task_id, + context="demo controller adapter result", + ) result.steps = history - result.num_steps = len(history) + result.num_steps = num_steps result.total_time_seconds = time.perf_counter() - start_time - return result + return normalize_benchmark_result( + result, + expected_task_id=task.task_id, + context="demo controller final result", + ) # ------------------------------------------------------------------ # State machine operations diff --git a/openadapt_evals/errors.py b/openadapt_evals/errors.py new file mode 100644 index 0000000..14ea995 --- /dev/null +++ b/openadapt_evals/errors.py @@ -0,0 +1,69 @@ +"""Errors that must remain distinct from valid benchmark outcomes.""" + +from enum import Enum + + +class ActionDeliveryState(str, Enum): + """What the runtime knows about action delivery after an exception.""" + + NOT_DELIVERED = "not_delivered" + UNCERTAIN = "uncertain" + DELIVERED = "delivered" + + +class ActionExecutionError(RuntimeError): + """An action failed with a machine-readable delivery state.""" + + error_type = "infrastructure" + + def __init__( + self, + message: str, + *, + delivery_state: ActionDeliveryState = ActionDeliveryState.NOT_DELIVERED, + ) -> None: + super().__init__(message) + self.delivery_state = delivery_state + + @property + def retry_safe(self) -> bool: + """Return True only when the runtime proves no action was delivered.""" + return self.delivery_state is ActionDeliveryState.NOT_DELIVERED + + +class ActionDeliveryUncertainError(ActionExecutionError): + """An action may have reached the target, so blind retry is unsafe.""" + + def __init__(self, message: str) -> None: + super().__init__(message, delivery_state=ActionDeliveryState.UNCERTAIN) + + +class ActionDeliveredObservationError(ActionExecutionError): + """The action was delivered, but the required next observation failed.""" + + def __init__(self, message: str) -> None: + super().__init__(message, delivery_state=ActionDeliveryState.DELIVERED) + + +class ActionParseError(ValueError): + """Model output did not contain one valid explicit action.""" + + error_type = "agent" + + +class RolloutInfrastructureError(RuntimeError): + """A rollout could not run because its observation infrastructure failed.""" + + error_type = "infrastructure" + + +class RolloutEvaluationError(RuntimeError): + """A rollout outcome could not be measured.""" + + error_type = "evaluation" + + +class RolloutLossError(RuntimeError): + """A rollout did not contain the evidence required to compute loss.""" + + error_type = "evaluation" diff --git a/openadapt_evals/evaluation/client.py b/openadapt_evals/evaluation/client.py index b9c27d8..3098b5a 100644 --- a/openadapt_evals/evaluation/client.py +++ b/openadapt_evals/evaluation/client.py @@ -23,6 +23,7 @@ """ import logging +import math import sys from dataclasses import dataclass, field from pathlib import Path @@ -30,6 +31,12 @@ import requests +from openadapt_evals.evaluation.receipts import ( + ReceiptValidationError, + parse_strict_json_response, + require_successful_receipt, +) + logger = logging.getLogger(__name__) @@ -46,6 +53,10 @@ class EvaluationError(RuntimeError): """An evaluation could not be carried out (as opposed to failing).""" +class EvaluationInfrastructureError(EvaluationError): + """The evaluator could not run because its remote dependency failed.""" + + @dataclass class EvaluationResult: """Result of evaluating a benchmark task. @@ -80,8 +91,10 @@ def to_dict(self) -> Dict[str, Any]: return { "success": self.success, "score": self.score, - "actual": str(self.actual)[:500] if self.actual else None, - "expected": str(self.expected)[:500] if self.expected else None, + "actual": str(self.actual)[:500] if self.actual is not None else None, + "expected": ( + str(self.expected)[:500] if self.expected is not None else None + ), "reason": self.reason, "metrics": self.metrics, "error_type": self.error_type, @@ -260,9 +273,17 @@ def evaluate(self, task_config: Dict[str, Any]) -> EvaluationResult: evaluator yields ``score=0.0`` with ``error_type`` set, which is not the same thing as a task the agent failed. """ + if not isinstance(task_config, dict): + return EvaluationResult( + success=False, + score=0.0, + reason="Task config must be an object", + error_type="evaluation", + evaluator_source=self.evaluator_source, + ) evaluator_config = task_config.get("evaluator", {}) - if not evaluator_config: + if not isinstance(evaluator_config, dict) or not evaluator_config: # A task with no evaluator spec cannot be scored at all. This is a # configuration error, not a task the agent failed. return EvaluationResult( @@ -293,7 +314,7 @@ def evaluate(self, task_config: Dict[str, Any]) -> EvaluationResult: evaluator_source=self.evaluator_source, ) - except requests.RequestException as e: + except (requests.RequestException, EvaluationInfrastructureError) as e: # The VM never answered. Scoring this 0.0 with no marker is how an # unreachable backend gets published as a legitimate 0%. return EvaluationResult( @@ -314,11 +335,27 @@ def evaluate(self, task_config: Dict[str, Any]) -> EvaluationResult: def _get_actual_value(self, evaluator_config: Dict[str, Any]) -> Any: """Get actual value from VM using getter function.""" - result_spec = evaluator_config.get("result", {}) + result_spec = evaluator_config.get("result") + if not isinstance(result_spec, dict) or not result_spec: + raise EvaluationError( + "evaluator result contract must be a non-empty object" + ) getter_type = result_spec.get("type") - if not getter_type: - raise ValueError("No 'type' specified in evaluator.result") + if not isinstance(getter_type, str) or not getter_type.strip(): + raise EvaluationError("No 'type' specified in evaluator.result") + getter_type = getter_type.strip() + + required_fields = { + "file_content": ("path",), + "registry_value": ("key", "value"), + "process_running": ("process",), + "window_exists": ("title",), + "vm_command_line": ("command",), + "vm_file": ("path",), + }.get(getter_type, ()) + for field_name in required_fields: + self._require_getter_parameter(result_spec, field_name, getter_type) # Create a mock env object that the getters expect class HttpEnv: @@ -342,7 +379,18 @@ def execute(self, command: str) -> Dict[str, Any]: timeout=self.timeout, ) response.raise_for_status() - return response.json() + try: + payload = parse_strict_json_response( + response, + context="VM command", + ) + return require_successful_receipt( + payload, + context="VM command", + require_output=True, + ) + except ReceiptValidationError as exc: + raise EvaluationInfrastructureError(str(exc)) from exc env = HttpEnv(self.vm_ip, self.port, self.timeout) @@ -359,23 +407,23 @@ def _fallback_getter(self, env: Any, getter_type: str, spec: Dict[str, Any]) -> """Fallback getter implementation when WAA evaluators not available.""" # Common getter types if getter_type == "file_content": - path = spec.get("path", "") + path = self._require_getter_parameter(spec, "path", getter_type) result = env.execute(f"type {path}") return self._require_output(result, getter_type) elif getter_type == "registry_value": - key = spec.get("key", "") - value = spec.get("value", "") + key = self._require_getter_parameter(spec, "key", getter_type) + value = self._require_getter_parameter(spec, "value", getter_type) result = env.execute(f'reg query "{key}" /v "{value}"') return self._require_output(result, getter_type) elif getter_type == "process_running": - process = spec.get("process", "") + process = self._require_getter_parameter(spec, "process", getter_type) result = env.execute(f'tasklist /FI "IMAGENAME eq {process}"') return process.lower() in self._require_output(result, getter_type).lower() elif getter_type == "window_exists": - title = spec.get("title", "") + title = self._require_getter_parameter(spec, "title", getter_type) result = env.execute(f'powershell "Get-Process | Where-Object {{$_.MainWindowTitle -like \'*{title}*\'}}"') return bool(self._require_output(result, getter_type).strip()) @@ -390,26 +438,51 @@ def _require_output(result: Dict[str, Any], getter_type: str) -> str: thing; treating them alike is how a failed command becomes a measured empty result. """ - if isinstance(result, dict) and result.get("error"): + try: + result = require_successful_receipt( + result, + context=f"getter {getter_type!r}", + require_output=True, + ) + except ReceiptValidationError as exc: + raise EvaluationInfrastructureError(str(exc)) from exc + output = result["output"] + assert isinstance(output, str) + return output + + @staticmethod + def _require_getter_parameter( + spec: Dict[str, Any], name: str, getter_type: str + ) -> str: + """Return one required non-empty string getter parameter.""" + value = spec.get(name) + if not isinstance(value, str) or not value.strip(): raise EvaluationError( - f"getter '{getter_type}' could not run on the VM: {result['error']}" + f"getter '{getter_type}' requires non-empty '{name}'" ) - return (result or {}).get("output", "") or "" + return value.strip() def _get_expected_value(self, evaluator_config: Dict[str, Any]) -> Any: """Extract expected value from evaluator config.""" - expected_spec = evaluator_config.get("expected", {}) + if "expected" not in evaluator_config: + raise EvaluationError("evaluator has no expected-value contract") + expected_spec = evaluator_config["expected"] + if not isinstance(expected_spec, dict) or not expected_spec: + raise EvaluationError("expected-value contract must be a non-empty object") # Direct value if "value" in expected_spec: - return expected_spec["value"] + value = expected_spec["value"] + if value is None: + raise EvaluationError("expected-value contract cannot be null") + return value # Rules-based - rules = expected_spec.get("rules", {}) - if "match" in rules: + rules = expected_spec.get("rules") + if isinstance(rules, dict) and "match" in rules and rules["match"] is not None: return rules["match"] - return None + raise EvaluationError("expected-value contract is malformed") def _run_metric(self, evaluator_config: Dict[str, Any], actual: Any, expected: Any) -> float: """Run metric function to compare actual vs expected. @@ -420,22 +493,38 @@ def _run_metric(self, evaluator_config: Dict[str, Any], actual: Any, expected: A number nobody asked for. """ func_name = evaluator_config.get("func", "exact_match") + if not isinstance(func_name, str) or not func_name: + raise EvaluationError("metric name must be a non-empty string") + options = evaluator_config.get("options", {}) + if not isinstance(options, dict): + raise EvaluationError("metric options must be an object") + if evaluator_config.get("conj", "and") != "and": + raise EvaluationError( + "EvaluatorClient supports only one metric with conj='and'" + ) # Try WAA metric if available if self._metrics: metric_func = getattr(self._metrics, func_name, None) if metric_func: try: - return float(metric_func(actual, expected)) + score = metric_func(actual, expected, **options) except Exception as e: raise EvaluationError( f"WAA metric '{func_name}' raised while scoring: {e}" ) from e + return self._require_metric_score(score, func_name) # Fallback metrics - return self._fallback_metric(func_name, actual, expected) + return self._fallback_metric(func_name, actual, expected, options) - def _fallback_metric(self, func_name: str, actual: Any, expected: Any) -> float: + def _fallback_metric( + self, + func_name: str, + actual: Any, + expected: Any, + options: dict[str, Any] | None = None, + ) -> float: """Fallback metric — delegates to shared evaluation.metrics module. An unknown metric name raises. Silently substituting ``exact_match`` @@ -451,7 +540,24 @@ def _fallback_metric(self, func_name: str, actual: Any, expected: Any) -> float: f"fallback evaluator; scoring with a different metric would " f"publish a number the task config did not ask for" ) - return float(metric_fn(actual, expected)) + return self._require_metric_score( + metric_fn(actual, expected, **(options or {})), + func_name, + ) + + @staticmethod + def _require_metric_score(value: Any, metric_name: str) -> float: + """Return one finite score in the closed unit interval.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise EvaluationError( + f"metric '{metric_name}' returned a non-numeric score" + ) + score = float(value) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise EvaluationError( + f"metric '{metric_name}' returned an invalid score: {value!r}" + ) + return score def health_check(self) -> bool: """Check if WAA server is reachable. diff --git a/openadapt_evals/evaluation/metrics.py b/openadapt_evals/evaluation/metrics.py index 3048da3..4a4b483 100644 --- a/openadapt_evals/evaluation/metrics.py +++ b/openadapt_evals/evaluation/metrics.py @@ -7,12 +7,13 @@ from __future__ import annotations -from pathlib import Path from typing import Any def exact_match(result: Any, expected: Any, **options) -> float: """Exact string/value match.""" + if result is None or expected is None: + raise ValueError("exact_match requires observed and expected values") if result == expected: return 1.0 if str(result).strip() == str(expected).strip(): @@ -27,6 +28,8 @@ def fuzzy_match( Falls back to substring containment when rapidfuzz is not installed. """ + if result is None or expected is None or str(expected).strip() == "": + raise ValueError("fuzzy_match requires a non-empty expected value") try: from rapidfuzz import fuzz @@ -42,6 +45,8 @@ def fuzzy_match( def contains(result: Any, expected: Any, **options) -> float: """Check if result contains expected (case-insensitive).""" + if result is None or expected is None or str(expected).strip() == "": + raise ValueError("contains requires a non-empty expected value") result_str = str(result).lower() expected_str = str(expected).lower() return 1.0 if expected_str in result_str else 0.0 @@ -49,15 +54,28 @@ def contains(result: Any, expected: Any, **options) -> float: def boolean(result: Any, expected: Any, **options) -> float: """Boolean equality check.""" - return 1.0 if bool(result) == bool(expected) else 0.0 + return 1.0 if _parse_boolean(result) == _parse_boolean(expected) else 0.0 def file_exists(result: Any, expected: Any, **options) -> float: - """Check if file exists.""" - path = result if result else expected - if path and Path(path).exists(): - return 1.0 - return 0.0 + """Compare explicit remote file-existence evidence.""" + del options + return 1.0 if _parse_boolean(result) == _parse_boolean(expected) else 0.0 + + +def _parse_boolean(value: Any) -> bool: + """Parse an explicit Boolean value without Python truthiness.""" + if isinstance(value, bool): + return value + if isinstance(value, int) and value in {0, 1}: + return bool(value) + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"true", "1"}: + return True + if normalized in {"false", "0"}: + return False + raise ValueError(f"expected an explicit boolean value, got {value!r}") def get_metric(name: str): diff --git a/openadapt_evals/evaluation/receipts.py b/openadapt_evals/evaluation/receipts.py new file mode 100644 index 0000000..e6245eb --- /dev/null +++ b/openadapt_evals/evaluation/receipts.py @@ -0,0 +1,123 @@ +"""Strict validation for command and setup HTTP receipts.""" + +from __future__ import annotations + +import json +from typing import Any + + +class ReceiptValidationError(ValueError): + """A remote endpoint did not prove that its requested operation succeeded.""" + + +def _reject_duplicate_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build one JSON object and reject ambiguous duplicate fields.""" + parsed: dict[str, Any] = {} + for key, value in pairs: + if key in parsed: + raise ReceiptValidationError(f"duplicate receipt field {key!r}") + parsed[key] = value + return parsed + + +def parse_strict_json_response(response: Any, *, context: str) -> Any: + """Parse an HTTP JSON response without last-key-wins ambiguity. + + Real ``requests.Response`` objects retain the raw body in ``text``. Tests + and compatible response adapters can expose only ``json()``; that fallback + remains useful, but production responses always take the strict raw path. + """ + raw = getattr(response, "text", None) + try: + if isinstance(raw, str): + return json.loads(raw, object_pairs_hook=_reject_duplicate_fields) + return response.json() + except (TypeError, ValueError) as exc: + raise ReceiptValidationError( + f"{context} response was not unambiguous JSON: {exc}" + ) from exc + + +def require_successful_receipt( + payload: object, + *, + context: str, + require_output: bool = False, +) -> dict[str, Any]: + """Return one explicit successful receipt or raise. + + HTTP 200 only proves transport success. The JSON body must include at least + one positive operation signal and no failure or uncertain-delivery signal. + """ + if not isinstance(payload, dict): + raise ReceiptValidationError(f"{context} receipt must be an object") + + positive = False + for field in ("success", "ok"): + if field in payload: + if payload[field] is not True: + raise ReceiptValidationError( + f"{context} receipt has {field}={payload[field]!r}" + ) + positive = True + + if "failed" in payload: + if payload["failed"] is not False: + raise ReceiptValidationError( + f"{context} receipt has failed={payload['failed']!r}" + ) + positive = True + + for field in ("error", "stderr"): + if field not in payload: + continue + value = payload[field] + if not isinstance(value, str) or value.strip(): + raise ReceiptValidationError( + f"{context} receipt has {field}={value!r}" + ) + + if "returncode" in payload: + returncode = payload["returncode"] + if ( + isinstance(returncode, bool) + or not isinstance(returncode, int) + or returncode != 0 + ): + raise ReceiptValidationError( + f"{context} receipt has returncode={returncode!r}" + ) + positive = True + + if "delivery_state" in payload: + delivery_state = payload["delivery_state"] + if delivery_state != "delivered": + raise ReceiptValidationError( + f"{context} receipt has delivery_state={delivery_state!r}" + ) + # Delivery proves that the command was dispatched, not that it + # completed successfully. Require a separate outcome signal below. + + if "status" in payload: + status = payload["status"] + if not isinstance(status, str) or status.lower() not in { + "completed", + "ok", + "success", + "succeeded", + }: + raise ReceiptValidationError( + f"{context} receipt has status={status!r}" + ) + positive = True + + if not positive: + raise ReceiptValidationError( + f"{context} receipt has no explicit success signal" + ) + + if require_output and not isinstance(payload.get("output"), str): + raise ReceiptValidationError( + f"{context} receipt has no string output" + ) + return payload diff --git a/openadapt_evals/integrations/wandb_logger.py b/openadapt_evals/integrations/wandb_logger.py index 6280604..531475d 100644 --- a/openadapt_evals/integrations/wandb_logger.py +++ b/openadapt_evals/integrations/wandb_logger.py @@ -40,7 +40,11 @@ from pathlib import Path from typing import Any -from openadapt_evals.adapters import BenchmarkResult, BenchmarkTask +from openadapt_evals.adapters import ( + BenchmarkResult, + BenchmarkTask, + normalize_benchmark_result, +) logger = logging.getLogger(__name__) @@ -166,17 +170,22 @@ def log_task_result(self, result: BenchmarkResult) -> None: if self._run is None: self.init() + result = normalize_benchmark_result(result, context="W&B task result") self._results_buffer.append(result) self._tasks_logged += 1 - # Log incremental metrics - success_count = sum(1 for r in self._results_buffer if r.success) - current_rate = success_count / len(self._results_buffer) + # Log incremental measured-outcome metrics. Infrastructure and + # evaluator failures are attempts, not failed task outcomes. + metrics = self._compute_metrics(self._results_buffer) + is_outcome = result.error_type not in {"infrastructure", "evaluation"} self._run.log({ - "task/current_success_rate": current_rate, + "task/current_success_rate": metrics["success_rate"], "task/tasks_completed": self._tasks_logged, - "task/last_task_success": result.success, + "task/outcomes_completed": metrics["num_outcome_tasks"], + "task/last_task_is_outcome": is_outcome, + "task/last_task_success": result.success if is_outcome else None, + "task/last_task_error_type": result.error_type, "task/last_task_steps": result.num_steps, }) @@ -198,6 +207,11 @@ def log_results( logger.warning("No results to log") return + results = [ + normalize_benchmark_result(result, context=f"W&B result {index}") + for index, result in enumerate(results) + ] + # Build task_id -> domain mapping task_domains = {} if tasks: @@ -220,7 +234,15 @@ def log_results( "eval/avg_steps": metrics["avg_steps"], "eval/avg_time_seconds": metrics["avg_time_seconds"], "eval/num_tasks": metrics["num_tasks"], + "eval/num_attempts": metrics["num_attempts"], + "eval/num_outcome_tasks": metrics["num_outcome_tasks"], "eval/num_success": metrics["num_success"], + "eval/error_count": metrics["error_count"], + "eval/num_infrastructure_failures": metrics[ + "num_infrastructure_failures" + ], + "eval/num_evaluation_failures": metrics["num_evaluation_failures"], + "eval/num_agent_failures": metrics["num_agent_failures"], }) # Log per-domain metrics @@ -229,6 +251,20 @@ def log_results( f"eval/domain/{domain}/success_rate": dm["success_rate"], f"eval/domain/{domain}/avg_steps": dm["avg_steps"], f"eval/domain/{domain}/num_tasks": dm["num_tasks"], + f"eval/domain/{domain}/num_attempts": dm["num_attempts"], + f"eval/domain/{domain}/num_outcome_tasks": dm[ + "num_outcome_tasks" + ], + f"eval/domain/{domain}/error_count": dm["error_count"], + f"eval/domain/{domain}/num_infrastructure_failures": dm[ + "num_infrastructure_failures" + ], + f"eval/domain/{domain}/num_evaluation_failures": dm[ + "num_evaluation_failures" + ], + f"eval/domain/{domain}/num_agent_failures": dm[ + "num_agent_failures" + ], }) # Log task results table @@ -346,28 +382,11 @@ def finish(self, exit_code: int = 0) -> None: self._run = None def _compute_metrics(self, results: list[BenchmarkResult]) -> dict[str, Any]: - """Compute aggregate metrics from results.""" - if not results: - return { - "num_tasks": 0, - "num_success": 0, - "success_rate": 0.0, - "avg_score": 0.0, - "avg_steps": 0.0, - "avg_time_seconds": 0.0, - } - - num_tasks = len(results) - num_success = sum(1 for r in results if r.success) + """Compute measured outcome metrics and retain error-attempt counts.""" + from openadapt_evals.benchmarks.runner import compute_metrics - return { - "num_tasks": num_tasks, - "num_success": num_success, - "success_rate": num_success / num_tasks, - "avg_score": sum(r.score for r in results) / num_tasks, - "avg_steps": sum(r.num_steps for r in results) / num_tasks, - "avg_time_seconds": sum(r.total_time_seconds for r in results) / num_tasks, - } + metrics = compute_metrics(results) + return {**metrics, "num_success": metrics["success_count"]} def _compute_domain_metrics( self, @@ -399,7 +418,10 @@ def _log_task_table( "score", "num_steps", "time_seconds", + "is_outcome", + "error_type", "error", + "reason", ] data = [] @@ -412,7 +434,10 @@ def _log_task_table( r.score, r.num_steps, r.total_time_seconds, + r.error_type not in {"infrastructure", "evaluation"}, + r.error_type or "", r.error or "", + r.reason or "", ]) table = Table(columns=columns, data=data) @@ -422,10 +447,10 @@ def _log_error_breakdown(self, results: list[BenchmarkResult]) -> None: """Log error type breakdown.""" error_counts: dict[str, int] = defaultdict(int) for r in results: - if not r.success and r.error: - error_counts[r.error] += 1 + if r.error_type is not None: + error_counts[r.error_type] += 1 elif not r.success: - error_counts["unknown"] += 1 + error_counts["measured_failure"] += 1 if error_counts: # Create error breakdown table @@ -450,7 +475,12 @@ def _log_error_breakdown(self, results: list[BenchmarkResult]) -> None: def _log_step_distribution(self, results: list[BenchmarkResult]) -> None: """Log step count distribution.""" success_steps = [r.num_steps for r in results if r.success] - fail_steps = [r.num_steps for r in results if not r.success] + fail_steps = [ + r.num_steps + for r in results + if not r.success + and r.error_type not in {"infrastructure", "evaluation"} + ] if success_steps: self._run.log({ @@ -489,15 +519,51 @@ def load_results_from_summary(summary_path: str | Path) -> list[BenchmarkResult] with open(summary_path) as f: summary = json.load(f) + if not isinstance(summary, dict): + raise ValueError("Benchmark summary must be a JSON object") + task_rows = summary.get("tasks") + if not isinstance(task_rows, list): + raise ValueError("Benchmark summary must contain a tasks list") + results = [] - for task_data in summary.get("tasks", []): - results.append(BenchmarkResult( - task_id=task_data["task_id"], - success=task_data["success"], - score=task_data.get("score", 1.0 if task_data["success"] else 0.0), + for index, task_data in enumerate(task_rows): + fallback_task_id = f"summary-row-{index}" + if not isinstance(task_data, dict): + results.append( + normalize_benchmark_result( + task_data, + expected_task_id=fallback_task_id, + context=f"summary row {index}", + ) + ) + continue + + raw_task_id = task_data.get("task_id") + expected_task_id = ( + raw_task_id + if isinstance(raw_task_id, str) and raw_task_id + else fallback_task_id + ) + candidate = BenchmarkResult( + task_id=raw_task_id, + success=task_data.get("success"), + score=task_data.get("score"), num_steps=task_data.get("num_steps", 0), total_time_seconds=task_data.get("total_time_seconds", 0.0), error=task_data.get("error"), - )) + reason=task_data.get("reason"), + error_type=( + task_data["error_type"] + if "error_type" in task_data + else "__missing__" + ), + ) + results.append( + normalize_benchmark_result( + candidate, + expected_task_id=expected_task_id, + context=f"summary row {index}", + ) + ) return results diff --git a/openadapt_evals/server/evaluate_endpoint.py b/openadapt_evals/server/evaluate_endpoint.py index 0419503..7504b8a 100644 --- a/openadapt_evals/server/evaluate_endpoint.py +++ b/openadapt_evals/server/evaluate_endpoint.py @@ -27,9 +27,17 @@ from __future__ import annotations import logging +import math from pathlib import Path from typing import Any, Callable +import requests + +from openadapt_evals.evaluation.receipts import ( + parse_strict_json_response, + require_successful_receipt, +) + logger = logging.getLogger(__name__) @@ -151,8 +159,23 @@ def get_actual_value( if getters is None: getters, _ = _load_waa_evaluators() - result_spec = evaluator_config.get("result", {}) - result_type = result_spec.get("type", "vm_command_line") + if "result" not in evaluator_config: + raise EvaluationNotRunError("evaluator has no result contract") + result_spec = evaluator_config["result"] + if not isinstance(result_spec, dict) or not result_spec: + raise EvaluationNotRunError("evaluator result contract must be a non-empty object") + result_type = result_spec.get("type") + if not isinstance(result_type, str) or not result_type: + raise EvaluationNotRunError("evaluator result contract requires an explicit type") + required_field = {"vm_command_line": "command", "vm_file": "path"}.get( + result_type + ) + if required_field is not None: + required_value = result_spec.get(required_field) + if not isinstance(required_value, str) or not required_value.strip(): + raise EvaluationNotRunError( + f"evaluator result type {result_type!r} requires {required_field}" + ) # Get the getter function getter_name = f"get_{result_type}" @@ -203,38 +226,57 @@ def get_expected_value( if getters is None: getters, _ = _load_waa_evaluators() - expected_spec = evaluator_config.get("expected", {}) + if "expected" not in evaluator_config: + raise EvaluationNotRunError("evaluator has no expected-value contract") + + expected_spec = evaluator_config["expected"] + if not isinstance(expected_spec, dict): + if expected_spec is None: + raise EvaluationNotRunError("expected-value contract cannot be null") + return expected_spec + if not expected_spec: + raise EvaluationNotRunError("expected-value contract is empty") # Handle different expected value formats expected_type = expected_spec.get("type") if expected_type == "rule": # Rule-based matching - return the rules dict - return expected_spec.get("rules", {}) + rules = expected_spec.get("rules") + if not isinstance(rules, dict) or not rules: + raise EvaluationNotRunError("expected rule contract is missing its rules") + return rules if expected_type == "literal" or "value" in expected_spec: # Direct literal value - return expected_spec.get("value") + if "value" not in expected_spec or expected_spec["value"] is None: + raise EvaluationNotRunError("expected literal contract has no value") + return expected_spec["value"] if expected_type: # Get via getter function getter_name = f"get_{expected_type}" getter_func = getattr(getters, getter_name, None) - if getter_func: - try: - return getter_func(env, expected_spec) - except Exception as e: - logger.error(f"Expected getter {getter_name} failed: {e}") - return None - - # Fallback: check for direct 'expected' value - if "expected" in evaluator_config and not isinstance( - evaluator_config["expected"], dict - ): - return evaluator_config["expected"] + if getter_func is None: + raise EvaluationNotRunError( + f"expected getter '{getter_name}' is not implemented" + ) + try: + value = getter_func(env, expected_spec) + except Exception as e: + logger.error(f"Expected getter {getter_name} failed: {e}") + raise EvaluationNotRunError( + f"expected getter '{getter_name}' raised: {e}", + error_type="infrastructure", + ) from e + if value is None: + raise EvaluationNotRunError( + f"expected getter '{getter_name}' returned no value" + ) + return value - return None + raise EvaluationNotRunError("expected-value contract is malformed") def run_metric( @@ -256,6 +298,8 @@ def run_metric( Returns: Score from 0.0 to 1.0. """ + if not isinstance(metric_name, str) or not metric_name: + raise EvaluationNotRunError("metric name must be a non-empty string") if metrics is None: _, metrics = _load_waa_evaluators() @@ -276,12 +320,21 @@ def run_metric( try: score = metric_func(actual, expected, **options) - return float(score) except Exception as e: logger.error(f"Metric {metric_name} failed: {e}") raise EvaluationNotRunError( f"metric '{metric_name}' raised while scoring: {e}" ) from e + if ( + isinstance(score, bool) + or not isinstance(score, (int, float)) + or not math.isfinite(float(score)) + or not 0.0 <= float(score) <= 1.0 + ): + raise EvaluationNotRunError( + f"metric '{metric_name}' returned an invalid score: {score!r}" + ) + return float(score) def evaluate_task_state( @@ -314,6 +367,8 @@ def evaluate_task_state( "metrics": list, # Per-metric results if multiple } """ + if not isinstance(task_config, dict): + return _not_scored("Task config must be an object", "evaluation") if env is None: env = MockEnv() @@ -325,25 +380,37 @@ def evaluate_task_state( evaluator_config = task_config.get("evaluator", {}) - if not evaluator_config: + if not isinstance(evaluator_config, dict) or not evaluator_config: return _not_scored("No evaluator configuration in task", "evaluation") - # Handle infeasible tasks - # If task is marked infeasible and agent reported FAIL, that's success - if evaluator_config.get("infeasible"): + # Handle infeasible tasks. The marker is part of the scoring contract, so + # truthy strings or numbers must not turn an agent refusal into success. + infeasible = evaluator_config.get("infeasible", False) + if not isinstance(infeasible, bool): + return _not_scored("Evaluator infeasible must be a boolean", "evaluation") + if infeasible: # Check if agent's last action was a FAIL/infeasible signal agent_last_action = task_config.get("agent_last_action", "") + if not isinstance(agent_last_action, str): + return _not_scored("Agent last action must be a string", "evaluation") if agent_last_action.upper() in ("FAIL", "INFEASIBLE", "IMPOSSIBLE"): return { "success": True, "score": 1.0, "reason": "Correctly identified infeasible task", + "scored": True, + "error_type": None, } # Run postconfig if present (e.g., activate windows for inspection) postconfig = evaluator_config.get("postconfig", []) + if "postconfig" in evaluator_config and not isinstance(postconfig, list): + return _not_scored("Evaluator postconfig must be a list", "evaluation") if postconfig: - _run_postconfig(postconfig, env) + try: + _run_postconfig(postconfig, env) + except EvaluationNotRunError as e: + return _not_scored(str(e), e.error_type) # Get actual value from VM try: @@ -358,12 +425,20 @@ def evaluate_task_state( func_spec = evaluator_config.get("func", "exact_match") options = evaluator_config.get("options", {}) conjunction = evaluator_config.get("conj", "and") # "and" or "or" + if not isinstance(options, dict): + return _not_scored("Metric options must be an object", "evaluation") + if conjunction not in ("and", "or"): + return _not_scored("Metric conjunction must be 'and' or 'or'", "evaluation") # Handle single or multiple metrics if isinstance(func_spec, str): func_names = [func_spec] - else: + elif isinstance(func_spec, list) and all( + isinstance(name, str) and name for name in func_spec + ): func_names = func_spec + else: + return _not_scored("Metric func must name one or more metrics", "evaluation") # Run each metric metric_results = [] @@ -448,40 +523,80 @@ def _run_postconfig(postconfig: list, env: MockEnv) -> None: postconfig: List of postconfig command specs. env: Environment object. """ - import requests - - for cmd in postconfig: - try: - cmd_type = cmd.get("type", "") - - if cmd_type == "activate_window": - # Send activate window command to VM - window_name = cmd.get("name", "") - requests.post( - f"http://{env.vm_ip}:{env.port}/setup/activate_window", - json={"name": window_name}, - timeout=10.0, - ) - elif cmd_type == "wait": - import time - - time.sleep(cmd.get("seconds", 1.0)) + for index, cmd in enumerate(postconfig): + if not isinstance(cmd, dict): + raise EvaluationNotRunError( + f"postconfig item {index} must be an object" + ) + cmd_type = cmd.get("type") + if cmd_type not in {"activate_window", "wait", "execute"}: + raise EvaluationNotRunError( + f"postconfig item {index} has unsupported type: {cmd_type!r}" + ) - elif cmd_type == "execute": - # Run a command on the VM - command = cmd.get("command", "") - requests.post( - f"http://{env.vm_ip}:{env.port}/execute_windows", - json={"command": command}, - timeout=30.0, + if cmd_type == "wait": + seconds = cmd.get("seconds", 1.0) + if ( + isinstance(seconds, bool) + or not isinstance(seconds, (int, float)) + or not math.isfinite(float(seconds)) + or seconds < 0 + ): + raise EvaluationNotRunError( + f"postconfig wait item {index} has invalid seconds" ) + import time - else: - logger.debug(f"Unknown postconfig type: {cmd_type}") + time.sleep(float(seconds)) + continue + if cmd_type == "activate_window": + field = "name" + endpoint = "setup/activate_window" + timeout = 10.0 + else: + field = "command" + endpoint = "execute_windows" + timeout = 30.0 + value = cmd.get(field) + if not isinstance(value, str) or not value.strip(): + raise EvaluationNotRunError( + f"postconfig {cmd_type} item {index} requires {field}" + ) + + try: + response = requests.post( + f"http://{env.vm_ip}:{env.port}/{endpoint}", + json={field: value}, + timeout=timeout, + ) except Exception as e: - logger.warning(f"Postconfig command failed: {e}") + raise EvaluationNotRunError( + f"postconfig {cmd_type} item {index} request failed: {e}", + error_type="infrastructure", + ) from e + if not 200 <= response.status_code < 300: + raise EvaluationNotRunError( + f"postconfig {cmd_type} item {index} failed with " + f"HTTP {response.status_code}", + error_type="infrastructure", + ) + try: + payload = parse_strict_json_response( + response, + context=f"postconfig {cmd_type} item {index}", + ) + require_successful_receipt( + payload, + context=f"postconfig {cmd_type} item {index}", + ) + except (TypeError, ValueError) as e: + raise EvaluationNotRunError( + f"postconfig {cmd_type} item {index} returned an invalid " + f"success receipt: {e}", + error_type="infrastructure", + ) from e def _truncate_value(value: Any, max_len: int = 500) -> str: @@ -583,12 +698,12 @@ class StandaloneMetrics: continue to work. """ - from openadapt_evals.evaluation.metrics import ( - contains, - exact_match, - file_exists, - fuzzy_match, - ) + from openadapt_evals.evaluation import metrics as _metrics + + contains = staticmethod(_metrics.contains) + exact_match = staticmethod(_metrics.exact_match) + file_exists = staticmethod(_metrics.file_exists) + fuzzy_match = staticmethod(_metrics.fuzzy_match) class StandaloneGetters: @@ -606,17 +721,21 @@ def get_vm_file(self, env: MockEnv, config: dict) -> str | None: import requests path = config.get("path", "") - try: - resp = requests.post( - f"{self.server_url}/execute_windows", - json={"command": f"Get-Content -Path '{path}'", "shell": "powershell"}, - timeout=30.0, + resp = requests.post( + f"{self.server_url}/execute_windows", + json={"command": f"Get-Content -Path '{path}'", "shell": "powershell"}, + timeout=30.0, + ) + if resp.status_code != 200: + raise RuntimeError( + f"file getter returned HTTP {resp.status_code} for {path!r}" ) - if resp.status_code == 200: - return resp.json().get("output", "") - except Exception as e: - logger.error(f"Failed to get file {path}: {e}") - return None + payload = require_successful_receipt( + parse_strict_json_response(resp, context="standalone file getter"), + context="standalone file getter", + require_output=True, + ) + return payload["output"] def get_vm_command_line(self, env: MockEnv, config: dict) -> str | None: """Execute command on VM and return output.""" @@ -625,17 +744,21 @@ def get_vm_command_line(self, env: MockEnv, config: dict) -> str | None: command = config.get("command", "") shell = config.get("shell", "powershell") - try: - resp = requests.post( - f"{self.server_url}/execute_windows", - json={"command": command, "shell": shell}, - timeout=60.0, + resp = requests.post( + f"{self.server_url}/execute_windows", + json={"command": command, "shell": shell}, + timeout=60.0, + ) + if resp.status_code != 200: + raise RuntimeError( + f"command getter returned HTTP {resp.status_code}" ) - if resp.status_code == 200: - return resp.json().get("output", "") - except Exception as e: - logger.error(f"Failed to run command: {e}") - return None + payload = require_successful_receipt( + parse_strict_json_response(resp, context="standalone command getter"), + context="standalone command getter", + require_output=True, + ) + return payload["output"] def create_standalone_evaluator( @@ -664,46 +787,114 @@ def create_standalone_evaluator( metrics = StandaloneMetrics() def evaluate(task_config: dict) -> dict: + if not isinstance(task_config, dict): + return _not_scored("Standalone task config must be an object", "evaluation") evaluator_config = task_config.get("evaluator", {}) - if not evaluator_config: - return { - "success": False, - "score": 0.0, - "reason": "No evaluator configuration", - } + if not isinstance(evaluator_config, dict) or not evaluator_config: + return _not_scored("No evaluator configuration", "evaluation") # Get result spec - result_spec = evaluator_config.get("result", {}) - result_type = result_spec.get("type", "vm_command_line") + result_spec = evaluator_config.get("result") + if not isinstance(result_spec, dict) or not result_spec: + return _not_scored( + "standalone result contract must be a non-empty object", + "evaluation", + ) + result_type = result_spec.get("type") + if not isinstance(result_type, str) or not result_type: + return _not_scored("standalone result type is invalid", "evaluation") + required_result_field = { + "vm_command_line": "command", + "vm_file": "path", + }.get(result_type) + if required_result_field is None: + return _not_scored( + f"standalone result type {result_type!r} is not supported", + "evaluation", + ) + required_result_value = result_spec.get(required_result_field) + if ( + not isinstance(required_result_value, str) + or not required_result_value.strip() + ): + return _not_scored( + f"standalone result type {result_type!r} requires " + f"{required_result_field}", + "evaluation", + ) + + postconfig = evaluator_config.get("postconfig", []) + if "postconfig" in evaluator_config and not isinstance(postconfig, list): + return _not_scored("standalone postconfig must be a list", "evaluation") + if postconfig: + return _not_scored( + "standalone evaluator does not implement postconfig", + "evaluation", + ) # Get actual value getter_name = f"get_{result_type}" getter_func = getattr(getters, getter_name, None) - - actual = None - if getter_func: - try: - actual = getter_func(MockEnv(), result_spec) - except Exception as e: - logger.error(f"Getter failed: {e}") + if getter_func is None: + return _not_scored( + f"standalone getter '{getter_name}' is not implemented", + "evaluation", + ) + try: + actual = getter_func(MockEnv(), result_spec) + except Exception as e: + return _not_scored( + f"standalone getter '{getter_name}' raised: {e}", + "infrastructure", + ) + if actual is None: + return _not_scored( + f"standalone getter '{getter_name}' returned no value", + "evaluation", + ) # Get expected value - expected_spec = evaluator_config.get("expected", {}) + expected_spec = evaluator_config.get("expected") + if not isinstance(expected_spec, dict): + return _not_scored("standalone expected contract is missing", "evaluation") if expected_spec.get("type") == "rule": - expected = expected_spec.get("rules", {}).get("match") + rules = expected_spec.get("rules") + if not isinstance(rules, dict) or "match" not in rules: + return _not_scored( + "standalone expected rule requires rules.match", "evaluation" + ) + expected = rules["match"] + elif "value" in expected_spec and expected_spec["value"] is not None: + expected = expected_spec["value"] else: - expected = expected_spec.get("value") + return _not_scored( + "standalone expected literal requires value", "evaluation" + ) # Run metric func_name = evaluator_config.get("func", "exact_match") - metric_func = getattr(metrics, func_name, metrics.exact_match) - + if not isinstance(func_name, str) or not func_name: + return _not_scored("standalone metric name is invalid", "evaluation") + options = evaluator_config.get("options", {}) + if not isinstance(options, dict): + return _not_scored("standalone metric options must be an object", "evaluation") + conjunction = evaluator_config.get("conj", "and") + if conjunction != "and": + return _not_scored( + "standalone evaluator supports only one metric with conj='and'", + "evaluation", + ) try: - score = metric_func(actual, expected) - except Exception as e: - logger.error(f"Metric failed: {e}") - score = 0.0 + score = run_metric( + func_name, + actual, + expected, + options=options, + metrics=metrics, + ) + except EvaluationNotRunError as e: + return _not_scored(str(e), e.error_type) success = score >= 1.0 @@ -713,6 +904,8 @@ def evaluate(task_config: dict) -> dict: "actual": _truncate_value(actual), "expected": _truncate_value(expected), "reason": "Standalone evaluation", + "scored": True, + "error_type": None, } return evaluate diff --git a/openadapt_evals/task_config.py b/openadapt_evals/task_config.py index b57b61c..73fb831 100644 --- a/openadapt_evals/task_config.py +++ b/openadapt_evals/task_config.py @@ -35,7 +35,11 @@ import yaml -from openadapt_evals.adapters.base import BenchmarkTask +from openadapt_evals.adapters.base import BenchmarkTask, EvaluationUnavailableError +from openadapt_evals.evaluation.receipts import ( + ReceiptValidationError, + parse_strict_json_response, +) logger = logging.getLogger(__name__) @@ -391,53 +395,193 @@ def evaluate_milestones( passed = 0 total = len(self.milestones) if total == 0: - return 0, 0 + raise EvaluationUnavailableError( + "No milestone evaluation contract is configured", + error_type="evaluation", + ) for ms in self.milestones: try: if ms.check.check == "screenshot": + if not ms.check.description: + raise ValueError("Screenshot milestone has no description") from openadapt_evals.vlm_evaluator import vlm_judge - success, _ = vlm_judge(screenshot, ms.check.description or "") + success, _ = vlm_judge(screenshot, ms.check.description) if success: passed += 1 elif ms.check.check == "command": - result = self._run_vm_command(ms.check.run or "", server_url) - if self._check_match(result, ms.check.expect or "", ms.check.match): + if not isinstance(ms.check.run, str) or not ms.check.run.strip(): + raise ValueError("Command milestone has no command") + if ms.check.expect is None: + raise ValueError("Command milestone has no expected result") + result = self._run_vm_command(ms.check.run, server_url) + if self._check_match(result, ms.check.expect, ms.check.match): passed += 1 else: - logger.warning("Milestone check type '%s' not yet supported", ms.check.check) + raise ValueError( + f"Unsupported milestone check type: {ms.check.check!r}" + ) except Exception as exc: - logger.warning("Milestone '%s' evaluation failed: %s", ms.name, exc) + if isinstance(exc, EvaluationUnavailableError): + raise + raise EvaluationUnavailableError( + f"Milestone {ms.name!r} could not be evaluated: {exc}", + error_type="evaluation", + ) from exc logger.info("Milestones: %d/%d passed", passed, total) return passed, total @staticmethod def _run_vm_command(command: str, server_url: str) -> str: - """Execute a shell command on the VM and return stdout. + """Execute a shell command and return stdout from a proved receipt. WAA's /execute_windows endpoint runs Python code via exec(). Shell commands (PowerShell, cmd) must be wrapped in subprocess. """ + import json + import requests - # Wrap shell command in Python subprocess so /execute_windows can exec() it - escaped = command.replace("\\", "\\\\").replace("'", "\\'") + if not isinstance(command, str) or not command.strip(): + raise EvaluationUnavailableError( + "VM command must be a non-empty string", + error_type="evaluation", + ) + + # Print one structured receipt. Plain stdout cannot distinguish an + # empty successful command from a failed command that produced no text. python_code = ( - "import subprocess; " - f"r = subprocess.run('{escaped}', shell=True, capture_output=True, text=True); " - "print(r.stdout.strip())" + "import json, subprocess; " + f"r = subprocess.run({json.dumps(command)}, shell=True, " + "capture_output=True, text=True); " + "print(json.dumps({'returncode': r.returncode, " + "'stdout': r.stdout, 'stderr': r.stderr}))" ) - resp = requests.post( - f"{server_url}/execute_windows", - json={"command": python_code}, - timeout=30, - ) - if resp.status_code == 200: - return resp.json().get("output", "").strip() - return "" + try: + resp = requests.post( + f"{server_url}/execute_windows", + json={"command": python_code}, + timeout=30, + ) + except requests.RequestException as exc: + raise EvaluationUnavailableError( + f"VM command request failed: {exc}", + error_type="infrastructure", + ) from exc + if resp.status_code != 200: + raise EvaluationUnavailableError( + f"VM command returned HTTP {resp.status_code}", + error_type="infrastructure", + ) + try: + payload = parse_strict_json_response(resp, context="VM command") + except ReceiptValidationError as exc: + raise EvaluationUnavailableError( + "VM command response was not valid JSON", + error_type="infrastructure", + ) from exc + if not isinstance(payload, dict): + raise EvaluationUnavailableError( + "VM command response was not an object", + error_type="infrastructure", + ) + for success_field in ("success", "ok"): + if success_field in payload and payload[success_field] is not True: + raise EvaluationUnavailableError( + f"VM command endpoint returned {success_field}=" + f"{payload[success_field]!r}", + error_type="infrastructure", + ) + if "failed" in payload and payload["failed"] is not False: + raise EvaluationUnavailableError( + f"VM command endpoint returned failed={payload['failed']!r}", + error_type="infrastructure", + ) + if ( + "delivery_state" in payload + and payload["delivery_state"] != "delivered" + ): + raise EvaluationUnavailableError( + "VM command endpoint did not confirm delivery: " + f"{payload['delivery_state']!r}", + error_type="infrastructure", + ) + for response_field in ("error", "stderr"): + value = payload.get(response_field) + if value not in (None, ""): + raise EvaluationUnavailableError( + f"VM command endpoint returned {response_field}: {value}", + error_type="infrastructure", + ) + outer_returncode = payload.get("returncode") + if outer_returncode is not None and ( + isinstance(outer_returncode, bool) + or not isinstance(outer_returncode, int) + or outer_returncode != 0 + ): + raise EvaluationUnavailableError( + f"VM command endpoint returned status {outer_returncode!r}", + error_type="infrastructure", + ) + + output = payload.get("output") + if not isinstance(output, str): + raise EvaluationUnavailableError( + "VM command response did not contain a string receipt", + error_type="infrastructure", + ) + def reject_duplicate_fields( + pairs: list[tuple[str, Any]], + ) -> dict[str, Any]: + parsed: dict[str, Any] = {} + for key, value in pairs: + if key in parsed: + raise ValueError(f"duplicate field {key!r}") + parsed[key] = value + return parsed + + try: + receipt = json.loads(output, object_pairs_hook=reject_duplicate_fields) + except (json.JSONDecodeError, ValueError) as exc: + raise EvaluationUnavailableError( + "VM command response did not contain a command receipt", + error_type="evaluation", + ) from exc + if not isinstance(receipt, dict) or set(receipt) != { + "returncode", + "stdout", + "stderr", + }: + raise EvaluationUnavailableError( + "VM command receipt was malformed", + error_type="evaluation", + ) + returncode = receipt["returncode"] + stdout = receipt["stdout"] + stderr = receipt["stderr"] + if ( + isinstance(returncode, bool) + or not isinstance(returncode, int) + or returncode != 0 + ): + raise EvaluationUnavailableError( + f"VM command failed with return code {returncode!r}", + error_type="evaluation", + ) + if not isinstance(stdout, str) or not isinstance(stderr, str): + raise EvaluationUnavailableError( + "VM command receipt output fields were malformed", + error_type="evaluation", + ) + if stderr.strip(): + raise EvaluationUnavailableError( + f"VM command wrote to stderr: {stderr.strip()}", + error_type="evaluation", + ) + return stdout.strip() def evaluate_checks_local( self, screenshot: bytes, server_url: str, @@ -452,47 +596,78 @@ def evaluate_checks_local( 1.0 if checks pass (respecting ``combine`` mode), else 0.0. """ if not self.checks: - return 0.0 + raise EvaluationUnavailableError( + "No local evaluation checks are configured", + error_type="evaluation", + ) results: list[bool] = [] for check in self.checks: try: if check.check == "screenshot": + if not check.description: + raise ValueError("Screenshot check has no description") from openadapt_evals.vlm_evaluator import vlm_judge - success, _ = vlm_judge(screenshot, check.description or "") + success, _ = vlm_judge(screenshot, check.description) results.append(success) elif check.check == "command": - result = self._run_vm_command(check.run or "", server_url) + if not isinstance(check.run, str) or not check.run.strip(): + raise ValueError("Command check has no command") + if check.expect is None: + raise ValueError("Command check has no expected result") + result = self._run_vm_command(check.run, server_url) results.append( - self._check_match(result, check.expect or "", check.match) + self._check_match(result, check.expect, check.match) ) else: - results.append(False) + raise ValueError( + f"Unsupported local check type: {check.check!r}" + ) except Exception as exc: - logger.warning("evaluate_checks_local: %s", exc) - results.append(False) + if isinstance(exc, EvaluationUnavailableError): + raise + raise EvaluationUnavailableError( + f"Local {check.check!r} check could not be evaluated: {exc}", + error_type="evaluation", + ) from exc if not results: - return 0.0 + raise EvaluationUnavailableError( + "Local evaluation produced no check results", + error_type="evaluation", + ) if self.combine == "or": return 1.0 if any(results) else 0.0 - return 1.0 if all(results) else 0.0 + if self.combine == "and": + return 1.0 if all(results) else 0.0 + raise EvaluationUnavailableError( + f"Unsupported local check combination: {self.combine!r}", + error_type="evaluation", + ) @staticmethod def _check_match(actual: str, expected: str, match_type: str) -> bool: """Check if actual matches expected using the specified method.""" + if not isinstance(actual, str) or not isinstance(expected, str): + raise ValueError("Check values must be strings") if match_type == "exact": return actual.strip() == expected.strip() elif match_type == "contains": + if not expected.strip(): + raise ValueError("Contains checks require a non-empty expected value") return expected.strip() in actual elif match_type == "regex": import re + if not expected.strip(): + raise ValueError("Regex checks require a non-empty expected pattern") return bool(re.search(expected, actual)) elif match_type == "fuzzy": import difflib + if not expected.strip(): + raise ValueError("Fuzzy checks require a non-empty expected value") return difflib.SequenceMatcher(None, actual, expected).ratio() >= 0.8 - return False + raise ValueError(f"Unsupported match type: {match_type!r}") def evaluate_milestones_screenshot( @@ -501,11 +676,11 @@ def evaluate_milestones_screenshot( *, model: str = "gpt-4.1-mini", ) -> float: - """Evaluate milestones using VLM screenshot checks only. + """Evaluate a screenshot-only milestone contract using a VLM. - Returns milestones_passed / total as a float [0, 1]. - Skips non-screenshot milestones (command, file checks). - No WAA server needed -- runs entirely client-side. + Every required milestone must be measurable from the screenshot. A caller + must use :meth:`TaskConfig.evaluate_milestones` for mixed screenshot and + command contracts. Args: task_config: A TaskConfig with milestones defined. @@ -513,26 +688,53 @@ def evaluate_milestones_screenshot( model: VLM model name for screenshot evaluation. Returns: - Fraction of screenshot milestones passed (0.0 to 1.0). - Returns 0.0 if there are no screenshot milestones. + Fraction of milestones passed (0.0 to 1.0). + + Raises: + EvaluationUnavailableError: If the contract is empty or any required + milestone cannot be measured from the screenshot. """ - screenshot_milestones = [ - m for m in task_config.milestones if m.check.check == "screenshot" + if not task_config.milestones: + raise EvaluationUnavailableError( + "No milestone evaluation contract is configured", + error_type="evaluation", + ) + unsupported = [ + milestone.name + for milestone in task_config.milestones + if milestone.check.check != "screenshot" ] - if not screenshot_milestones: - return 0.0 + if unsupported: + raise EvaluationUnavailableError( + "Screenshot-only evaluation cannot measure required milestones: " + + ", ".join(repr(name) for name in unsupported), + error_type="evaluation", + ) from openadapt_evals.vlm_evaluator import vlm_judge passed = 0 - for milestone in screenshot_milestones: - success, _confidence = vlm_judge( - screenshot, milestone.check.description or "", model=model - ) + for milestone in task_config.milestones: + if not milestone.check.description: + raise EvaluationUnavailableError( + f"Screenshot milestone {milestone.name!r} has no description", + error_type="evaluation", + ) + try: + success, _confidence = vlm_judge( + screenshot, milestone.check.description, model=model + ) + except Exception as exc: + if isinstance(exc, EvaluationUnavailableError): + raise + raise EvaluationUnavailableError( + f"Milestone {milestone.name!r} could not be evaluated: {exc}", + error_type="evaluation", + ) from exc if success: passed += 1 - return passed / len(screenshot_milestones) + return passed / len(task_config.milestones) # --------------------------------------------------------------------------- diff --git a/openadapt_evals/telemetry.py b/openadapt_evals/telemetry.py index 43a86fa..93a888a 100644 --- a/openadapt_evals/telemetry.py +++ b/openadapt_evals/telemetry.py @@ -58,6 +58,9 @@ def track_agent_run_completed( entrypoint: str | None = None, mode: str | None = None, num_tasks: int | None = None, + attempt_count: int | None = None, + outcome_count: int | None = None, + error_count: int | None = None, success_count: int | None = None, avg_steps: float | None = None, return_code: int | None = None, @@ -70,6 +73,9 @@ def track_agent_run_completed( "entrypoint": entrypoint, "mode": mode, "num_tasks": num_tasks, + "attempt_count": attempt_count, + "outcome_count": outcome_count, + "error_count": error_count, "success_count": success_count, "avg_steps": avg_steps, "return_code": return_code, diff --git a/openadapt_evals/training/standalone/config.py b/openadapt_evals/training/standalone/config.py index a6edd81..2737b19 100644 --- a/openadapt_evals/training/standalone/config.py +++ b/openadapt_evals/training/standalone/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from dataclasses import dataclass, field @@ -62,3 +63,24 @@ class TrainingConfig: use_unsloth: bool = False # Weave project for LLM tracing (empty = disabled) weave_project: str = "" + + def validate_for_training(self) -> None: + """Reject configurations that cannot perform a measured training step.""" + positive_integer_fields = ( + "num_training_steps", + "num_rollouts_per_step", + "max_steps_per_episode", + "save_every_steps", + ) + for name in positive_integer_fields: + value = getattr(self, name) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + + if ( + isinstance(self.learning_rate, bool) + or not isinstance(self.learning_rate, (int, float)) + or not math.isfinite(float(self.learning_rate)) + or self.learning_rate <= 0 + ): + raise ValueError("learning_rate must be finite and positive") diff --git a/openadapt_evals/training/standalone/prompt.py b/openadapt_evals/training/standalone/prompt.py index c130137..ec79499 100644 --- a/openadapt_evals/training/standalone/prompt.py +++ b/openadapt_evals/training/standalone/prompt.py @@ -6,12 +6,18 @@ from __future__ import annotations -import json as _json import logging -import re +import math from dataclasses import dataclass from typing import Any +from openadapt_evals.action_envelope import ( + parse_single_dsl_action, + parse_single_json_object, + require_exact_fields, +) +from openadapt_evals.errors import ActionParseError + logger = logging.getLogger(__name__) DEFAULT_SCREEN_SIZE: tuple[int, int] = (1920, 1080) @@ -50,8 +56,23 @@ class SimpleAction: key: str | None = None +def _normalized_point_to_pixels( + x: float, + y: float, + screen_size: tuple[int, int], +) -> tuple[int, int]: + """Map inclusive normalized coordinates to valid viewport indices.""" + width, height = screen_size + if width <= 0 or height <= 0: + raise ActionParseError("Screen dimensions must be positive") + return min(int(x * width), width - 1), min(int(y * height), height - 1) + + def build_agent_messages( - instruction: str, *, include_image: bool = False, action_history: str = "", + instruction: str, + *, + include_image: bool = False, + action_history: str = "", ) -> list[dict]: """Build chat messages matching the SFT prompt format.""" history_text = f"{action_history}\n" if action_history else "" @@ -75,78 +96,136 @@ def build_agent_messages( def parse_vlm_output_to_action( - text: str, screen_size: tuple[int, int] = DEFAULT_SCREEN_SIZE, + text: str, + screen_size: tuple[int, int] = DEFAULT_SCREEN_SIZE, ) -> SimpleAction: """Parse VLM output to SimpleAction. Supports Thought/Action, bare DSL, and JSON.""" text = text.strip() width, height = screen_size logger.debug("Parsing VLM output (%d chars): %.200s", len(text), text) - # Extract from "Action: ..." format - action_match = re.search(r"Action:\s*(.+)", text, re.IGNORECASE) - if action_match: - text = action_match.group(1).strip() - # JSON: {"action_type": "click", "coordinate": [x, y]} - json_match = re.search(r'\{[^}]*"action_type"[^}]*\}', text) - if json_match: - try: - d = _json.loads(json_match.group()) - atype = d.get("action_type", "").lower() - coord = d.get("coordinate", d.get("coords", [])) - if atype == "click" and len(coord) >= 2: + data = parse_single_json_object(text) + if data is not None: + atype = data.get("action_type", "") + if not isinstance(atype, str): + raise ActionParseError("JSON action_type must be a string") + atype = atype.lower() + if atype == "click": + coordinate_fields = {field for field in ("coordinate", "coords") if field in data} + if len(coordinate_fields) != 1: + raise ActionParseError("JSON click requires exactly one coordinate field") + field = next(iter(coordinate_fields)) + _require_json_fields(data, {"action_type", field}) + coord = data[field] + if not isinstance(coord, (list, tuple)) or len(coord) != 2: + raise ActionParseError("JSON click requires exactly two coordinates") + if any(isinstance(value, bool) for value in coord): + raise ActionParseError("JSON click coordinates must be numbers, not booleans") + try: xv, yv = float(coord[0]), float(coord[1]) - if xv <= 1.0 and yv <= 1.0: - xv, yv = xv * width, yv * height - return SimpleAction(type="click", x=int(xv), y=int(yv)) - if atype == "type": - return SimpleAction(type="type", text=d.get("text", "")) - if atype in ("done", "wait"): - return SimpleAction(type=atype) - except Exception: - pass - - # CLICK(x=..., y=...) - m = re.search(r"CLICK\(x=(-?[\d.]+),\s*y=(-?[\d.]+)\)", text, re.IGNORECASE) - if m: + except (TypeError, ValueError) as exc: + raise ActionParseError("Malformed JSON click coordinates") from exc + if not math.isfinite(xv) or not math.isfinite(yv): + raise ActionParseError("JSON click coordinates must be finite") + normalized = 0.0 <= xv <= 1.0 and 0.0 <= yv <= 1.0 + pixels = 0.0 <= xv < width and 0.0 <= yv < height + if normalized: + x_px, y_px = _normalized_point_to_pixels(xv, yv, screen_size) + return SimpleAction(type="click", x=x_px, y=y_px) + if not all(isinstance(value, int) and not isinstance(value, bool) for value in coord): + raise ActionParseError( + "Pixel-space JSON click coordinates must be explicit integers" + ) + if not pixels: + raise ActionParseError("JSON click coordinates are outside the viewport") + return SimpleAction(type="click", x=coord[0], y=coord[1]) + if atype == "type": + _require_json_fields(data, {"action_type", "text"}) + if not isinstance(data.get("text"), str): + raise ActionParseError("JSON type requires string text") + return SimpleAction(type="type", text=data["text"]) + if atype in ("done", "wait"): + _require_json_fields(data, {"action_type"}) + return SimpleAction(type=atype) + raise ActionParseError(f"Unsupported JSON action type: {atype!r}") + + command, arguments = parse_single_dsl_action( + text, + allowed_commands={"CLICK", "TYPE", "WAIT", "DONE"}, + ) + if command == "CLICK": + require_exact_fields(arguments, {"x", "y"}, command) try: - xf = max(0.0, min(1.0, float(m.group(1)))) - yf = max(0.0, min(1.0, float(m.group(2)))) - return SimpleAction(type="click", x=int(xf * width), y=int(yf * height)) + xf = float(arguments["x"]) + yf = float(arguments["y"]) + if not all(math.isfinite(v) and 0.0 <= v <= 1.0 for v in (xf, yf)): + raise ActionParseError("CLICK coordinates must be between 0 and 1") + x_px, y_px = _normalized_point_to_pixels(xf, yf, screen_size) + return SimpleAction(type="click", x=x_px, y=y_px) except (ValueError, TypeError, OverflowError): - # The regex [\d.]+ can match literal "..." from model output - # (e.g. CLICK(x=..., y=...)), causing float("...") to raise - # ValueError. Fall through to DONE rather than crashing. - logger.warning("Malformed CLICK coords: x=%s y=%s — returning DONE", m.group(1), m.group(2)) - return SimpleAction(type="done") - - # TYPE(text="...") - m = re.search(r"""TYPE\(text=["']([^"'\\]*(?:\\.[^"'\\]*)*)["']\)""", text, re.IGNORECASE) - if m: - t = m.group(1).replace("\\\\", "\\").replace('\\"', '"').replace("\\'", "'") - return SimpleAction(type="type", text=t) - - if re.search(r"\bWAIT\s*\(\s*\)", text, re.IGNORECASE): + raise ActionParseError( + f"Malformed CLICK coordinates: " + f"x={arguments['x']!r}, y={arguments['y']!r}" + ) + if command == "TYPE": + require_exact_fields(arguments, {"text"}, command) + value = arguments["text"] + value = value.replace("\\\\", "\\").replace('\\"', '"').replace("\\'", "'") + return SimpleAction(type="type", text=value) + if command == "WAIT": + require_exact_fields(arguments, set(), command) return SimpleAction(type="wait") - if re.search(r"\bDONE\s*\(\s*\)", text, re.IGNORECASE): + if command == "DONE": + require_exact_fields(arguments, set(), command) return SimpleAction(type="done") - logger.warning("Could not parse VLM output: %s. Defaulting to DONE.", text) - return SimpleAction(type="done") + raise ActionParseError(f"Unsupported action command: {command!r}") + + +def _require_json_fields(data: dict[str, Any], required: set[str]) -> None: + """Reject missing and unrelated standalone JSON action fields.""" + fields = set(data) - {"reasoning"} + missing = required - fields + extra = fields - required + if missing: + raise ActionParseError( + f"JSON action requires {', '.join(sorted(missing))}" + ) + if extra: + raise ActionParseError( + f"JSON action has unsupported fields: {', '.join(sorted(extra))}" + ) def format_action_as_text( - action: SimpleAction, screen_size: tuple[int, int] = DEFAULT_SCREEN_SIZE, + action: SimpleAction, + screen_size: tuple[int, int] = DEFAULT_SCREEN_SIZE, ) -> str: """Convert SimpleAction to DSL text for log-prob computation.""" width, height = screen_size if action.type == "click": - xf = (action.x or 0) / width if width > 0 else 0.0 - yf = (action.y or 0) / height if height > 0 else 0.0 + if width <= 0 or height <= 0: + raise ActionParseError("Screen dimensions must be positive") + if action.x is None or action.y is None: + raise ActionParseError("Cannot format CLICK without x and y") + if not ( + math.isfinite(action.x) + and math.isfinite(action.y) + and 0 <= action.x < width + and 0 <= action.y < height + ): + raise ActionParseError("Cannot format CLICK outside the viewport") + xf = action.x / width + yf = action.y / height return f"CLICK(x={xf:.2f}, y={yf:.2f})" if action.type == "type": - escaped = (action.text or "").replace("\\", "\\\\").replace('"', '\\"') + if not isinstance(action.text, str): + raise ActionParseError("Cannot format TYPE without explicit string text") + escaped = action.text.replace("\\", "\\\\").replace('"', '\\"') return f'TYPE(text="{escaped}")' if action.type == "wait": return "WAIT()" - return "DONE()" + if action.type == "done": + return "DONE()" + raise ActionParseError(f"Cannot format unsupported action type: {action.type!r}") diff --git a/openadapt_evals/training/standalone/reward.py b/openadapt_evals/training/standalone/reward.py index 742efcf..015f64c 100644 --- a/openadapt_evals/training/standalone/reward.py +++ b/openadapt_evals/training/standalone/reward.py @@ -2,9 +2,16 @@ from __future__ import annotations +import io import logging +import math +from numbers import Real from typing import Any +from PIL import Image + +from openadapt_evals.errors import RolloutEvaluationError, RolloutLossError + logger = logging.getLogger(__name__) @@ -12,24 +19,87 @@ def compute_group_advantages(rewards: list[float]) -> list[float]: """GRPO group-relative advantages: (r - mean) / (std + eps).""" n = len(rewards) if n == 0: - return [] - mean = sum(rewards) / n - variance = sum((r - mean) ** 2 for r in rewards) / n + raise RolloutLossError("Cannot compute advantages without rewards") + measured_rewards: list[float] = [] + for index, reward in enumerate(rewards): + if isinstance(reward, bool) or not isinstance(reward, Real): + raise RolloutLossError(f"Reward {index} must be numeric") + measured = float(reward) + if not math.isfinite(measured) or not 0.0 <= measured <= 1.0: + raise RolloutLossError( + f"Reward {index} must be finite and within [0, 1]" + ) + measured_rewards.append(measured) + + mean = sum(measured_rewards) / n + variance = sum((reward - mean) ** 2 for reward in measured_rewards) / n std = variance**0.5 if std < 1e-8: return [0.0] * n - return [(r - mean) / (std + 1e-8) for r in rewards] + advantages = [ + (reward - mean) / (std + 1e-8) for reward in measured_rewards + ] + if any(not math.isfinite(advantage) for advantage in advantages): + raise RolloutLossError("Group advantage calculation produced a non-finite value") + return advantages def evaluate_milestones_screenshot( task_config: Any, screenshot: bytes, *, model: str = "gpt-4.1-mini", ) -> float: - """VLM screenshot-only milestone evaluation. Returns passed/total [0,1].""" - milestones = getattr(task_config, "milestones", []) - sm = [m for m in milestones if m.check.check == "screenshot"] - if not sm: - return 0.0 + """Measure screenshot milestones or raise when no score was produced.""" + milestones = list(getattr(task_config, "milestones", []) or []) + if not milestones: + raise RolloutEvaluationError("No screenshot milestone contract is configured") + unsupported = [ + milestone.name + for milestone in milestones + if milestone.check.check != "screenshot" + ] + if unsupported: + raise RolloutEvaluationError( + "Screenshot reward cannot measure required milestones: " + + ", ".join(repr(name) for name in unsupported) + ) + if not isinstance(screenshot, bytes) or not screenshot: + raise RolloutEvaluationError("Milestone evaluation requires screenshot bytes") + try: + with Image.open(io.BytesIO(screenshot)) as image: + image.load() + except Exception as exc: + raise RolloutEvaluationError( + "Milestone evaluation requires a decodable screenshot" + ) from exc from openadapt_evals.vlm_evaluator import vlm_judge - passed = sum(1 for m in sm if vlm_judge(screenshot, m.check.description or "", model=model)[0]) - return passed / len(sm) + passed = 0 + for milestone in milestones: + description = milestone.check.description + if not description: + raise RolloutEvaluationError( + f"Screenshot milestone {milestone.name!r} has no description" + ) + try: + success, confidence = vlm_judge(screenshot, description, model=model) + except Exception as exc: + raise RolloutEvaluationError( + f"Screenshot milestone {milestone.name!r} could not be evaluated" + ) from exc + if not isinstance(success, bool): + raise RolloutEvaluationError( + f"Screenshot milestone {milestone.name!r} returned no YES/NO verdict" + ) + if isinstance(confidence, bool) or not isinstance(confidence, Real): + raise RolloutEvaluationError( + f"Screenshot milestone {milestone.name!r} returned invalid confidence" + ) + measured_confidence = float(confidence) + if not math.isfinite(measured_confidence) or not 0.0 <= measured_confidence <= 1.0: + raise RolloutEvaluationError( + f"Screenshot milestone {milestone.name!r} returned invalid confidence" + ) + passed += int(success) + reward = passed / len(milestones) + if not math.isfinite(reward) or not 0.0 <= reward <= 1.0: + raise RolloutEvaluationError("Milestone reward was not a finite unit value") + return reward diff --git a/openadapt_evals/training/standalone/trainer.py b/openadapt_evals/training/standalone/trainer.py index 341c7d3..ea40b16 100644 --- a/openadapt_evals/training/standalone/trainer.py +++ b/openadapt_evals/training/standalone/trainer.py @@ -14,13 +14,16 @@ import argparse import io import logging +import math import re import time +from numbers import Real from pathlib import Path from typing import Any from PIL import Image +from openadapt_evals.errors import RolloutInfrastructureError, RolloutLossError from openadapt_evals.telemetry import ( track_checkpoint_saved, track_rollout_collected, @@ -46,9 +49,31 @@ def policy_gradient_loss(current_logps, old_logps, advantages, epsilon=0.2): """PPO-clipped policy gradient. Single-epoch: reduces to REINFORCE.""" import torch + + if not isinstance(epsilon, Real) or not math.isfinite(float(epsilon)): + raise RolloutLossError("Policy loss epsilon must be finite") + if not 0.0 <= float(epsilon) < 1.0: + raise RolloutLossError("Policy loss epsilon must be within [0, 1)") + for name, value in ( + ("current log probabilities", current_logps), + ("old log probabilities", old_logps), + ("advantages", advantages), + ): + try: + finite = bool(torch.isfinite(value).all().item()) + except Exception as exc: + raise RolloutLossError(f"Policy loss {name} are not numeric tensors") from exc + if not finite: + raise RolloutLossError(f"Policy loss {name} must be finite") + ratio = torch.exp(current_logps - old_logps) + if not bool(torch.isfinite(ratio).all().item()): + raise RolloutLossError("Policy loss ratio must be finite") clipped = torch.clamp(ratio, 1 - epsilon, 1 + epsilon) - return -torch.min(ratio * advantages, clipped * advantages).mean() + loss = -torch.min(ratio * advantages, clipped * advantages).mean() + if not bool(torch.isfinite(loss).all().item()): + raise RolloutLossError("Policy loss must be finite") + return loss class GRPOTrainer: @@ -200,10 +225,9 @@ def _collect_rollout(self, task_id: str, instruction: str) -> Rollout: try: screenshot = self._env.screenshot() except Exception as e: - logger.warning( - "Screenshot failed at step %d after retries: %s", step_idx, e, - ) - break + raise RolloutInfrastructureError( + f"Screenshot failed at step {step_idx} after retries" + ) from e recent.append(screenshot) if self._env.is_stuck(recent, window=self._config.stuck_window): logger.info("Stuck at step %d", step_idx) @@ -220,9 +244,10 @@ def _collect_rollout(self, task_id: str, instruction: str) -> Rollout: try: screenshot = self._env.screenshot() image = Image.open(io.BytesIO(screenshot)) - except Exception: - logger.error("Screenshot retry also failed, skipping step") - break + except Exception as exc: + raise RolloutInfrastructureError( + f"Screenshot remained unreadable at step {step_idx}" + ) from exc if image.mode != "RGB": image = image.convert("RGB") # .convert() drops .format; restore it for outlines.Image @@ -301,12 +326,11 @@ def _collect_rollout(self, task_id: str, instruction: str) -> Rollout: # Fresh screenshot for evaluation tc = self._task_configs.get(task_id) - if tc and getattr(tc, "milestones", None): - try: - rollout.reward = evaluate_milestones_screenshot( - tc, self._env.screenshot(), model=self._config.eval_model) - except Exception as e: - logger.warning("Milestone eval failed: %s", e) + if not tc: + raise RolloutInfrastructureError(f"Task configuration {task_id!r} is missing") + rollout.reward = evaluate_milestones_screenshot( + tc, self._env.screenshot(), model=self._config.eval_model + ) return rollout def _collect_group(self, task_id: str) -> list[Rollout]: @@ -320,12 +344,9 @@ def _collect_group(self, task_id: str) -> list[Rollout]: # to a full group of rollouts (avoids wasting time on a dead server). probe = self._env.probe() if not probe.get("screenshot_ok"): - logger.error( - "Pre-rollout health check FAILED for task %s: %s — " - "skipping group (returning empty rollouts)", - task_id, probe, + raise RolloutInfrastructureError( + f"Pre-rollout health check failed for task {task_id}: {probe}" ) - return [] tc = self._task_configs.get(task_id) instruction = getattr(tc, "name", "") or task_id if tc else task_id @@ -350,18 +371,35 @@ def _collect_group(self, task_id: str) -> list[Rollout]: def _compute_rollout_loss(self, rollout: Rollout, advantage: float, scale: float) -> float: """Compute GRPO loss for one rollout. Per-step backward to avoid OOM.""" + if isinstance(advantage, bool) or not isinstance(advantage, Real): + raise RolloutLossError("Rollout advantage must be numeric") + if not math.isfinite(float(advantage)): + raise RolloutLossError("Rollout advantage must be finite") + if isinstance(scale, bool) or not isinstance(scale, Real): + raise RolloutLossError("Rollout loss scale must be numeric") + if not math.isfinite(float(scale)) or float(scale) <= 0.0: + raise RolloutLossError("Rollout loss scale must be finite and positive") + if not rollout.steps: + raise RolloutLossError("Cannot compute loss for an empty rollout") + if any(not step.screenshot for step in rollout.steps): + raise RolloutLossError("Every rollout step requires screenshot evidence") + + images = [] + for index, step in enumerate(rollout.steps): + try: + image = Image.open(io.BytesIO(step.screenshot)) + image.load() + except Exception as exc: + raise RolloutLossError( + f"Rollout step {index} has an unreadable screenshot" + ) from exc + images.append(image) + import torch device = next(self._model.parameters()).device - valid = [s for s in rollout.steps if s.screenshot] - if not valid: - return 0.0 - total, n = 0.0, len(valid) + total, n = 0.0, len(rollout.steps) - for step in valid: - try: - image = Image.open(io.BytesIO(step.screenshot)) - except Exception: - continue + for index, (step, image) in enumerate(zip(rollout.steps, images)): if image.mode != "RGB": image = image.convert("RGB") image.format = "PNG" @@ -399,7 +437,7 @@ def _compute_rollout_loss(self, rollout: Rollout, advantage: float, scale: float action_ids = inner_tok(action_text, add_special_tokens=False, return_tensors="pt")["input_ids"] n_action = action_ids.shape[1] if n_action <= 0: - continue + raise RolloutLossError(f"Rollout step {index} has no action tokens") full_text = prompt_text + action_text full_inputs = self._processor( @@ -428,6 +466,10 @@ def _compute_rollout_loss(self, rollout: Rollout, advantage: float, scale: float full_inputs = {k: v.to(device) for k, v in full_inputs.items()} outputs = self._model(**full_inputs) + if not bool(torch.isfinite(outputs.logits).all().item()): + raise RolloutLossError( + f"Rollout step {index} produced non-finite model logits" + ) # Action logits are the last n_action positions in the output seq_len = outputs.logits.shape[1] @@ -439,16 +481,33 @@ def _compute_rollout_loss(self, rollout: Rollout, advantage: float, scale: float slp = tlp.sum() adv = torch.tensor(advantage, device=device, dtype=slp.dtype) loss = policy_gradient_loss(slp.unsqueeze(0), slp.detach().unsqueeze(0), adv.unsqueeze(0)) - (loss * scale / n).backward() - total += loss.detach().item() - return total / max(n, 1) + scaled_loss = loss * scale / n + if not bool(torch.isfinite(scaled_loss).all().item()): + raise RolloutLossError( + f"Rollout step {index} produced a non-finite scaled loss" + ) + scaled_loss.backward() + loss_value = float(loss.detach().item()) + if not math.isfinite(loss_value): + raise RolloutLossError( + f"Rollout step {index} produced a non-finite loss" + ) + total += loss_value + average_loss = total / n + if not math.isfinite(average_loss): + raise RolloutLossError("Rollout average loss must be finite") + return average_loss def _training_step(self, rollouts: list[Rollout]) -> dict[str, float]: """Single GRPO gradient step.""" - import torch + if not rollouts: + raise RolloutLossError("Cannot train from an empty rollout group") + rewards = [r.reward for r in rollouts] advantages = compute_group_advantages(rewards) - reward_mean = sum(rewards) / len(rewards) if rewards else 0.0 + reward_mean = sum(float(reward) for reward in rewards) / len(rewards) + if not math.isfinite(reward_mean): + raise RolloutLossError("Mean reward must be finite") valid = [(r, a) for r, a in zip(rollouts, advantages) if abs(a) >= 1e-8] if not valid: return {"reward_mean": reward_mean, "loss": 0.0, "skipped": True} @@ -458,25 +517,65 @@ def _training_step(self, rollouts: list[Rollout]) -> dict[str, float]: losses = [] for r, a in valid: loss = self._compute_rollout_loss(r, a, 1.0 / n) + if isinstance(loss, bool) or not isinstance(loss, Real): + raise RolloutLossError("Rollout loss must be numeric") + if not math.isfinite(float(loss)): + raise RolloutLossError("Rollout loss must be finite") losses.append(loss) - grad_norm = torch.nn.utils.clip_grad_norm_( - [p for p in self._model.parameters() if p.requires_grad], - max_norm=self._config.max_grad_norm, - ) + + avg_loss = sum(losses) / n + abs_loss = sum(abs(loss) for loss in losses) / n + if not math.isfinite(avg_loss) or not math.isfinite(abs_loss): + raise RolloutLossError("Training loss metrics must be finite") + + import torch + + trainable_parameters = [ + parameter + for parameter in self._model.parameters() + if parameter.requires_grad + ] + if not trainable_parameters: + raise RolloutLossError("The model has no trainable parameters") + gradient_parameters = [ + parameter for parameter in trainable_parameters if parameter.grad is not None + ] + if not gradient_parameters: + raise RolloutLossError("Training produced no gradients") + for parameter in gradient_parameters: + if not bool(torch.isfinite(parameter.grad).all().item()): + raise RolloutLossError("Training produced a non-finite gradient") + max_grad_norm = self._config.max_grad_norm + if ( + isinstance(max_grad_norm, bool) + or not isinstance(max_grad_norm, Real) + or not math.isfinite(float(max_grad_norm)) + or float(max_grad_norm) <= 0.0 + ): + raise RolloutLossError("Maximum gradient norm must be finite and positive") + try: + grad_norm = torch.nn.utils.clip_grad_norm_( + trainable_parameters, + max_norm=max_grad_norm, + error_if_nonfinite=True, + ) + except Exception as exc: + raise RolloutLossError( + "Gradient norm could not be measured as finite" + ) from exc gn = grad_norm.item() if hasattr(grad_norm, "item") else float(grad_norm) + if not math.isfinite(gn): + raise RolloutLossError("Gradient norm must be finite") if gn > 10 * self._config.max_grad_norm: logger.warning( "grad_norm=%.1f is %.0fx the clip threshold (%.1f). " "Gradients are dominated by clipping, not learning signal. " "Consider lowering learning_rate (current: %.1e).", - gn, gn / self._config.max_grad_norm, - self._config.max_grad_norm, self._config.learning_rate, + gn, gn / max_grad_norm, + max_grad_norm, self._config.learning_rate, ) self._optimizer.step() - avg_loss = sum(losses) / max(n, 1) - abs_loss = sum(abs(loss) for loss in losses) / max(n, 1) - return { "reward_mean": reward_mean, "loss": avg_loss, @@ -497,6 +596,8 @@ def _save_checkpoint(self, step: int) -> str: def train(self) -> str: """Run GRPO training loop. Returns path to final checkpoint.""" + self._config.validate_for_training() + import torch logger.info( @@ -544,19 +645,16 @@ def train(self) -> str: Path(self._config.output_dir).mkdir(parents=True, exist_ok=True) t0 = time.time() last_reward_mean: float | None = None + optimizer_updates = 0 for step in range(self._config.num_training_steps): ts = time.time() task_id = self._config.task_ids[step % len(self._config.task_ids)] self._model.eval() rollouts = self._collect_group(task_id) self._model.train() - if not rollouts: - logger.warning( - "Step %d/%d: no rollouts collected (server may be down), skipping.", - step + 1, self._config.num_training_steps, - ) - continue m = self._training_step(rollouts) + if not m.get("skipped", False): + optimizer_updates += 1 m.update({"step": step, "task_id": task_id, "elapsed": time.time() - t0, "step_time": time.time() - ts}) last_reward_mean = m.get("reward_mean") logger.info( @@ -584,13 +682,21 @@ def train(self) -> str: if self._on_step_complete is not None: self._on_step_complete(step, rollouts, m) - if (step + 1) % self._config.save_every_steps == 0: + if ( + optimizer_updates > 0 + and (step + 1) % self._config.save_every_steps == 0 + ): self._save_checkpoint(step + 1) try: track_checkpoint_saved(step=step + 1) except Exception: pass + if optimizer_updates == 0: + raise RolloutLossError( + "Training completed no optimizer updates; refusing a trained checkpoint" + ) + self._save_checkpoint(self._config.num_training_steps) try: track_checkpoint_saved(step=self._config.num_training_steps) diff --git a/openadapt_evals/training/trl_rollout.py b/openadapt_evals/training/trl_rollout.py index 6b3ec00..8671913 100644 --- a/openadapt_evals/training/trl_rollout.py +++ b/openadapt_evals/training/trl_rollout.py @@ -67,16 +67,21 @@ import hashlib import io -import json import logging -import re +import math import time from typing import Any, Callable, Optional from pydantic import BaseModel +from openadapt_evals.action_envelope import ( + parse_single_dsl_action, + parse_single_json_object, + require_exact_fields, +) from openadapt_evals.adapters.base import BenchmarkAction from openadapt_evals.adapters.rl_env import ResetConfig, RLEnvironment +from openadapt_evals.errors import ActionParseError, RolloutInfrastructureError # Re-exported on purpose, not merely imported: this module must use the SAME # system prompt object as the standalone trainer, and @@ -88,6 +93,73 @@ logger = logging.getLogger(__name__) + +def _required_fraction(data: dict, key: str, action_type: str) -> float: + """Return one required finite normalized coordinate.""" + if key not in data: + raise ActionParseError(f"{action_type} requires {key}") + try: + value = float(data[key]) + except (TypeError, ValueError) as exc: + raise ActionParseError( + f"{action_type} has invalid {key}: {data[key]!r}" + ) from exc + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + raise ActionParseError(f"{action_type} {key} must be between 0 and 1") + return value + + +def _parse_json_action(data: dict) -> BenchmarkAction: + """Validate one JSON action without inventing missing arguments.""" + action_type = data.get("type") + if action_type == "click": + _require_json_fields(data, {"type", "x", "y"}) + return BenchmarkAction( + type=action_type, + x=_required_fraction(data, "x", action_type), + y=_required_fraction(data, "y", action_type), + ) + if action_type == "scroll": + _require_json_fields(data, {"type", "x", "y", "direction"}) + direction = data.get("direction") + if direction not in ("up", "down"): + raise ActionParseError("scroll requires direction='up' or direction='down'") + return BenchmarkAction( + type="scroll", + x=_required_fraction(data, "x", action_type), + y=_required_fraction(data, "y", action_type), + scroll_direction=direction, + ) + if action_type == "type": + _require_json_fields(data, {"type", "text"}) + if "text" not in data or not isinstance(data["text"], str): + raise ActionParseError("type requires string text") + return BenchmarkAction(type="type", text=data["text"]) + if action_type == "key": + _require_json_fields(data, {"type", "key"}) + if not isinstance(data.get("key"), str) or not data["key"]: + raise ActionParseError("key requires non-empty string key") + return BenchmarkAction(type="key", key=data["key"]) + if action_type in ("done", "noop"): + _require_json_fields(data, {"type"}) + return BenchmarkAction(type=action_type) + raise ActionParseError(f"Unsupported JSON action type: {action_type!r}") + + +def _require_json_fields(data: dict, required: set[str]) -> None: + """Reject missing or unrelated fields in one JSON action object.""" + fields = set(data) - {"reasoning"} + missing = required - fields + extra = fields - required + if missing: + raise ActionParseError( + f"JSON action requires {', '.join(sorted(missing))}" + ) + if extra: + raise ActionParseError( + f"JSON action has unsupported fields: {', '.join(sorted(extra))}" + ) + # --------------------------------------------------------------------------- # Constrained decoding regex -- ported from standalone trainer # --------------------------------------------------------------------------- @@ -181,85 +253,39 @@ def parse_action_json(text: str) -> BenchmarkAction: Returns: BenchmarkAction parsed from the text. """ - # --- Try JSON first --- - stripped = text.strip() - stripped = re.sub(r"```json\s*", "", stripped) - stripped = re.sub(r"```\s*$", "", stripped) - - match = re.search(r"\{[^{}]*\}", stripped) - if match: - try: - data = json.loads(match.group()) - action_type = data.get("type", "done") - if action_type not in ("click", "type", "key", "scroll", "done", "noop"): - action_type = "done" - return BenchmarkAction( - type=action_type, - x=data.get("x"), - y=data.get("y"), - text=data.get("text"), - key=data.get("key"), - ) - except json.JSONDecodeError: - pass # Fall through to DSL parsing + data = parse_single_json_object(text) + if data is not None: + return _parse_json_action(data) # --- DSL fallback (Thought/Action format from standalone trainer) --- # This handles output from constrained decoding and existing checkpoints. # Extract fractional coordinates directly from DSL rather than using # parse_vlm_output_to_action (which converts to pixels). The TRL path # needs fractional coords for pixel_action(x_frac=, y_frac=). - action_line = text - action_match = re.search(r"Action:\s*(.+)", text, re.IGNORECASE) - if action_match: - action_line = action_match.group(1).strip() - - click_m = re.search(r"CLICK\(x=(-?[\d.]+),\s*y=(-?[\d.]+)\)", action_line, re.IGNORECASE) - if click_m: - try: - x = max(0.0, min(1.0, float(click_m.group(1)))) - y = max(0.0, min(1.0, float(click_m.group(2)))) - return BenchmarkAction(type="click", x=x, y=y) - except (ValueError, TypeError): - pass - - type_m = re.search(r"""TYPE\(text=["']([^"'\\]*(?:\\.[^"'\\]*)*)["']\)""", action_line, re.IGNORECASE) - if type_m: - t = type_m.group(1).replace("\\\\", "\\").replace('\\"', '"').replace("\\'", "'") - return BenchmarkAction(type="type", text=t) - - if re.search(r"\bWAIT\s*\(\s*\)", action_line, re.IGNORECASE): + command, arguments = parse_single_dsl_action( + text, + allowed_commands={"CLICK", "TYPE", "WAIT", "DONE"}, + ) + if command == "CLICK": + require_exact_fields(arguments, {"x", "y"}, command) + return BenchmarkAction( + type="click", + x=_required_fraction(arguments, "x", command), + y=_required_fraction(arguments, "y", command), + ) + if command == "TYPE": + require_exact_fields(arguments, {"text"}, command) + value = arguments["text"] + value = value.replace("\\\\", "\\").replace('\\"', '"').replace("\\'", "'") + return BenchmarkAction(type="type", text=value) + if command == "WAIT": + require_exact_fields(arguments, set(), command) return BenchmarkAction(type="wait") - if re.search(r"\bDONE\s*\(\s*\)", action_line, re.IGNORECASE): + if command == "DONE": + require_exact_fields(arguments, set(), command) return BenchmarkAction(type="done") - logger.warning("Could not parse VLM output (no JSON or DSL): %s", text[:200]) - return BenchmarkAction(type="done") - - -def _empty_rollout_result( - prompts: list[str], - num_generations: int, -) -> dict[str, list]: - """Return a zero-reward rollout result with the correct dict shape. - - Used when the WAA server is unreachable or unhealthy so that TRL receives - a consistent output structure (empty token lists, zero rewards) instead of - crashing. - - Args: - prompts: List of prompt strings from the trainer. - num_generations: Number of generations per prompt. - - Returns: - Dict with prompt_ids, completion_ids, logprobs, env_reward -- all zeroed. - """ - total = len(prompts) * num_generations - return { - "prompt_ids": [[] for _ in range(total)], - "completion_ids": [[] for _ in range(total)], - "logprobs": [[] for _ in range(total)], - "env_reward": [0.0] * total, - } + raise ActionParseError(f"Unsupported action command: {command!r}") def _run_episode( @@ -331,7 +357,9 @@ def _run_episode( break # Handle fractional coordinates - if action.x is not None and action.y is not None: + if action.type in ("click", "double_click", "right_click"): + if action.x is None or action.y is None: + raise ActionParseError(f"{action.type} requires x and y") if 0 <= action.x <= 1 and 0 <= action.y <= 1: step_result = env.pixel_action( x_frac=action.x, y_frac=action.y, @@ -342,8 +370,6 @@ def _run_episode( x=int(action.x), y=int(action.y), action_type=action.type, text=action.text, key=action.key, ) - elif action.type in ("type", "key"): - step_result = env.step(action) else: step_result = env.step(action) @@ -396,8 +422,8 @@ def make_waa_rollout_func( Ported from the standalone trainer's WAADirect.is_stuck(). on_before_collect: ``(task_id, env) -> None`` callback fired before each episode begins. Useful for health checks, logging, or - pre-rollout setup. A raised exception is caught and logged as - a warning (does not abort the episode). + pre-rollout setup. A raised exception aborts collection so stale + state cannot be evaluated as the requested task. on_rollout_complete: ``(rollout, index) -> None`` callback fired after each episode completes. ``rollout`` is a dict with keys ``prompt``, ``task_id``, ``reward``, ``gen_idx``. A raised @@ -406,12 +432,39 @@ def make_waa_rollout_func( Returns: A callable suitable for GRPOTrainer(rollout_func=...). """ - # Index task configs by name for lookup + positive_integer_options = { + "max_steps": max_steps, + "max_new_tokens": max_new_tokens, + "screenshot_retries": screenshot_retries, + } + for name, value in positive_integer_options.items(): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + if ( + isinstance(stuck_threshold, bool) + or not isinstance(stuck_threshold, int) + or stuck_threshold < 0 + ): + raise ValueError("stuck_threshold must be a non-negative integer") + if ( + isinstance(temperature, bool) + or not isinstance(temperature, (int, float)) + or not math.isfinite(float(temperature)) + or temperature <= 0 + ): + raise ValueError("temperature must be finite and positive") + + # Index task configs by exact prompt identity for lookup. config_map: dict[str, Any] = {} if task_configs: for tc in task_configs: - config_map[tc.name] = tc - config_map[tc.id] = tc + for key in (tc.name, tc.id): + if not isinstance(key, str) or not key: + raise ValueError("Task config names and IDs must be non-empty strings") + existing = config_map.get(key) + if existing is not None and existing is not tc: + raise ValueError(f"Duplicate task prompt identity: {key!r}") + config_map[key] = tc # Outlines generator is created lazily on first rollout call # (needs the trainer's model and processor which aren't available yet). @@ -456,6 +509,14 @@ def rollout_func(prompts: list[str], trainer: Any) -> dict[str, list]: device = next(model.parameters()).device num_generations = getattr(trainer.args, "num_generations", 8) + if ( + isinstance(num_generations, bool) + or not isinstance(num_generations, int) + or num_generations <= 0 + ): + raise RolloutInfrastructureError( + "trainer.args.num_generations must be a positive integer" + ) # --- Pre-rollout health check (P0) --- _mod = getattr(type(adapter), "__module__", "") or "" @@ -465,23 +526,18 @@ def rollout_func(prompts: list[str], trainer: Any) -> dict[str, list]: try: health_obs = adapter.observe() screenshot = getattr(health_obs, "screenshot", None) - if screenshot is not None and isinstance(screenshot, bytes) \ - and len(screenshot) < 100: - logger.warning( - "WAA server health check failed (screenshot=%d bytes) " - "-- returning zero rewards for %d prompts", - len(screenshot), - len(prompts), + if not isinstance(screenshot, bytes) or len(screenshot) < 100: + size = len(screenshot) if isinstance(screenshot, bytes) else None + raise RolloutInfrastructureError( + "WAA server health check returned an invalid screenshot " + f"({size} bytes)" ) - return _empty_rollout_result(prompts, num_generations) except Exception as exc: - logger.warning( - "WAA server unreachable: %s -- returning zero rewards for " - "%d prompts", - exc, - len(prompts), - ) - return _empty_rollout_result(prompts, num_generations) + if isinstance(exc, RolloutInfrastructureError): + raise + raise RolloutInfrastructureError( + f"WAA server health check failed: {exc}" + ) from exc # Lazy-init Outlines generator on first call if constrained_decoding and not _outlines_state["attempted"]: @@ -519,12 +575,10 @@ def generate_fn(screenshot_bytes: bytes, instruction: str): ) time.sleep(screenshot_retry_delay) else: - logger.error( - "Screenshot corrupt after %d attempts, " - "returning DONE action", - screenshot_retries, - ) - return "done", [], [] + raise RolloutInfrastructureError( + "Screenshot remained unreadable after " + f"{screenshot_retries} attempts" + ) from exc # Use the SAME message construction as the standalone trainer. # This includes the "Goal:" prefix, format guidance, and the @@ -706,23 +760,30 @@ def generate_fn(screenshot_bytes: bytes, instruction: str): return text, completion_ids, logprobs for prompt in prompts: + if not isinstance(prompt, str) or not prompt: + raise RolloutInfrastructureError( + "Every rollout prompt must be a non-empty task identity" + ) tc = config_map.get(prompt) + if tc is None: + raise RolloutInfrastructureError( + f"Rollout prompt {prompt!r} has no exact TaskConfig match" + ) for gen_idx in range(num_generations): env = RLEnvironment(adapter, task_config=tc) - task_id = tc.id if tc else "default" + task_id = tc.id # --- on_before_collect callback --- if on_before_collect is not None: try: on_before_collect(task_id, env) except Exception as exc: - logger.warning( - "on_before_collect callback raised for " - "task_id=%s gen=%d: %s", - task_id, gen_idx, exc, - ) + raise RolloutInfrastructureError( + "on_before_collect failed for " + f"task_id={task_id} gen={gen_idx}: {exc}" + ) from exc try: p_ids, c_ids, lps, reward = _run_episode( @@ -733,7 +794,7 @@ def generate_fn(screenshot_bytes: bytes, instruction: str): "Rollout failed for prompt=%s gen=%d: %s", prompt[:50], gen_idx, exc, ) - p_ids, c_ids, lps, reward = [], [], [], 0.0 + raise # --- on_rollout_complete callback --- if on_rollout_complete is not None: diff --git a/openadapt_evals/training/trl_wrapper.py b/openadapt_evals/training/trl_wrapper.py index e129a4f..b07b703 100644 --- a/openadapt_evals/training/trl_wrapper.py +++ b/openadapt_evals/training/trl_wrapper.py @@ -70,6 +70,8 @@ def __init__( def train(self) -> str: """Run GRPO training via TRL. Returns path to final checkpoint.""" + self._config.validate_for_training() + from datasets import Dataset from trl import GRPOConfig from trl import GRPOTrainer as _TRLTrainer @@ -96,11 +98,14 @@ def train(self) -> str: len(filtered), len(task_configs) + len(filtered) - len(filtered), ) else: - logger.warning( + raise ValueError( "task_ids=%s matched no tasks from task_dir=%s. " - "Available: %s. Using all tasks.", - self._config.task_ids, self._config.task_dir, - [tc.id for tc in task_configs], + "Available: %s" + % ( + self._config.task_ids, + self._config.task_dir, + [tc.id for tc in task_configs], + ) ) if not task_configs: diff --git a/openadapt_evals/vlm_evaluator.py b/openadapt_evals/vlm_evaluator.py index fa9ddf0..a2c1f3b 100644 --- a/openadapt_evals/vlm_evaluator.py +++ b/openadapt_evals/vlm_evaluator.py @@ -16,8 +16,14 @@ from __future__ import annotations +import io import json import logging +import math + +from PIL import Image + +from openadapt_evals.errors import RolloutEvaluationError logger = logging.getLogger(__name__) @@ -65,42 +71,83 @@ def vlm_judge( Returns: Tuple of (success: bool, confidence: float). + + Raises: + RolloutEvaluationError: If the screenshot or model response cannot be + measured under the judge contract. """ from openadapt_evals.vlm import vlm_call - prompt = _JUDGE_PROMPT.format(description=description) - response = vlm_call( - prompt, - images=[screenshot], - model=model, - provider=provider, - max_tokens=256, - temperature=0.1, - cost_label="vlm_judge", - ) + if not isinstance(screenshot, bytes) or not screenshot: + raise RolloutEvaluationError("VLM judging requires screenshot bytes") + try: + with Image.open(io.BytesIO(screenshot)) as image: + image.load() + except Exception as exc: + raise RolloutEvaluationError( + "VLM judging requires a decodable screenshot" + ) from exc + if not isinstance(description, str) or not description.strip(): + raise RolloutEvaluationError("VLM judging requires a condition description") + prompt = _JUDGE_PROMPT.format(description=description) try: - import re - - match = re.search(r"\{[^}]+\}", response, re.DOTALL) - if match: - data = json.loads(match.group()) - else: - data = json.loads(response) - - verdict = str(data.get("verdict", "NO")).upper().startswith("Y") - confidence = float(data.get("confidence", 0.5)) - explanation = data.get("explanation", "") - logger.info( - "VLM judge: %s (confidence=%.2f) — %s", - "PASS" if verdict else "FAIL", - confidence, - explanation[:80], + response = vlm_call( + prompt, + images=[screenshot], + model=model, + provider=provider, + max_tokens=256, + temperature=0.1, + cost_label="vlm_judge", ) - return verdict, confidence + except Exception as exc: + raise RolloutEvaluationError("VLM judge call failed") from exc - except (json.JSONDecodeError, ValueError, KeyError): - # Fallback: check if response starts with YES - verdict = response.strip().upper().startswith("YES") - logger.warning("VLM judge JSON parse failed, fallback: %s", verdict) - return verdict, 0.5 + if not isinstance(response, str): + raise RolloutEvaluationError("VLM judge returned a non-text response") + + def reject_duplicate_keys(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + try: + data = json.loads(response.strip(), object_pairs_hook=reject_duplicate_keys) + except (json.JSONDecodeError, TypeError, ValueError) as exc: + raise RolloutEvaluationError( + "VLM judge did not return the required JSON object" + ) from exc + if not isinstance(data, dict): + raise RolloutEvaluationError("VLM judge response must be a JSON object") + + verdict_text = data.get("verdict") + if verdict_text not in {"YES", "NO"}: + raise RolloutEvaluationError( + "VLM judge verdict must be exactly 'YES' or 'NO'" + ) + confidence_value = data.get("confidence") + if isinstance(confidence_value, bool) or not isinstance( + confidence_value, (int, float) + ): + raise RolloutEvaluationError("VLM judge confidence must be numeric") + confidence = float(confidence_value) + if not math.isfinite(confidence) or not 0.0 <= confidence <= 1.0: + raise RolloutEvaluationError( + "VLM judge confidence must be finite and within [0, 1]" + ) + explanation = data.get("explanation", "") + if not isinstance(explanation, str): + raise RolloutEvaluationError("VLM judge explanation must be text") + + verdict = verdict_text == "YES" + logger.info( + "VLM judge: %s (confidence=%.2f) — %s", + "PASS" if verdict else "FAIL", + confidence, + explanation[:80], + ) + return verdict, confidence diff --git a/tests/test_action_envelopes.py b/tests/test_action_envelopes.py new file mode 100644 index 0000000..f3dbefc --- /dev/null +++ b/tests/test_action_envelopes.py @@ -0,0 +1,71 @@ +"""Strict one-action envelope regression tests.""" + +from __future__ import annotations + +from collections.abc import Callable + +import pytest + +from openadapt_evals.adapters.verl_env import _parse_action_str +from openadapt_evals.errors import ActionParseError +from openadapt_evals.training.standalone.prompt import parse_vlm_output_to_action +from openadapt_evals.training.trl_rollout import parse_action_json + +_DSL_PARSERS: tuple[Callable[[str], object], ...] = ( + _parse_action_str, + parse_action_json, + parse_vlm_output_to_action, +) +_JSON_PARSERS: tuple[Callable[[str], object], ...] = ( + parse_action_json, + parse_vlm_output_to_action, +) + + +@pytest.mark.parametrize("parser", _DSL_PARSERS) +@pytest.mark.parametrize( + "output", + [ + "CLICK(x=0.2, y=0.3)\nDONE()", + "Action: CLICK(x=0.2, y=0.3)\nAction: DONE()", + "CLICK(x=0.2, y=0.3) trailing command", + "prefix CLICK(x=0.2, y=0.3)", + "Thought: choose target\nAction: CLICK(x=0.2, y=0.3)\nextra", + ], +) +def test_dsl_parser_rejects_multiple_or_extra_commands( + parser: Callable[[str], object], output: str +) -> None: + with pytest.raises(ActionParseError): + parser(output) + + +@pytest.mark.parametrize("parser", _DSL_PARSERS) +def test_dsl_parser_rejects_duplicate_keyword_fields( + parser: Callable[[str], object], +) -> None: + with pytest.raises(ActionParseError, match="Duplicate action field"): + parser("CLICK(x=0.1, x=0.2, y=0.3)") + + +@pytest.mark.parametrize("parser", _JSON_PARSERS) +def test_json_parser_rejects_two_objects(parser: Callable[[str], object]) -> None: + with pytest.raises(ActionParseError): + parser( + '{"type":"done","action_type":"done"} ' + '{"type":"done","action_type":"done"}' + ) + + +@pytest.mark.parametrize("parser", _JSON_PARSERS) +def test_json_parser_rejects_duplicate_fields(parser: Callable[[str], object]) -> None: + with pytest.raises(ActionParseError, match="Duplicate JSON action field"): + parser('{"type":"done","type":"done"}') + + +@pytest.mark.parametrize("parser", _DSL_PARSERS) +def test_optional_thought_with_one_action_remains_valid( + parser: Callable[[str], object], +) -> None: + action = parser("Thought: choose the exact target\nAction: CLICK(x=0.2, y=0.3)") + assert getattr(action, "type") == "click" diff --git a/tests/test_agent_failure_contract.py b/tests/test_agent_failure_contract.py index b7071ae..94303d1 100644 --- a/tests/test_agent_failure_contract.py +++ b/tests/test_agent_failure_contract.py @@ -23,6 +23,11 @@ def test_parse_failures_are_error_actions(): assert parse_qwen_action("not an action").type == "error" assert parse_smol_action("not an action").type == "error" + agent = ApiAgent.__new__(ApiAgent) + action = agent._parse_computer_action("computer.unknown()", BenchmarkObservation()) + assert action.type == "error" + assert action.raw_action["error_type"] == "agent" + def test_api_agent_keeps_terminal_decisions_distinct(): agent = ApiAgent.__new__(ApiAgent) diff --git a/tests/test_api_agent_parsing.py b/tests/test_api_agent_parsing.py index 6019415..60ce9c1 100644 --- a/tests/test_api_agent_parsing.py +++ b/tests/test_api_agent_parsing.py @@ -182,6 +182,44 @@ def test_parse_terminal_wait(self, mock_agent): assert result["status"] == "terminal" assert result["action"] == "WAIT" + @pytest.mark.parametrize( + "decision", ["NOT DONE", "DONE AND FAIL", "FAIL: NOT DONE", "MAYBE"] + ) + def test_ambiguous_terminal_decision_is_rejected(self, mock_agent, decision): + result = mock_agent._parse_api_response( + f"```decision\n{decision}\n```", 1920, 1200, {} + ) + + assert result["status"] == "failed" + + def test_terminal_decision_cannot_compete_with_action(self, mock_agent): + result = mock_agent._parse_api_response( + "```decision\nDONE\n```\n```python\ncomputer.click(1, 2)\n```", + 1920, + 1200, + {}, + ) + + assert result["status"] == "failed" + + @pytest.mark.parametrize( + "response", + [ + "computer.click(1, 2) computer.press('enter')", + 'computer.click(1, 2) {"type":"press","key":"enter"}', + "```python\ncomputer.click(1, 2)\n```\n" + '{"type":"press","key":"enter"}', + "```decision\nCONTINUE\n```\n```python\ncomputer.click(1, 2)\n```\n" + "```json\n{\"type\":\"press\",\"key\":\"enter\"}\n```", + "```\ncomputer.click(1, 2)\n" + '{"type":"press","key":"enter"}\n```', + ], + ) + def test_competing_action_envelopes_are_rejected(self, mock_agent, response): + result = mock_agent._parse_api_response(response, 1920, 1200, {}) + + assert result["status"] == "failed" + def test_parse_failure(self, mock_agent): """Test handling of unparseable response.""" response = """I'm not sure what to do here. Let me think about it.""" diff --git a/tests/test_areal_workflow.py b/tests/test_areal_workflow.py index 12660e5..e89c2fb 100644 --- a/tests/test_areal_workflow.py +++ b/tests/test_areal_workflow.py @@ -22,6 +22,7 @@ BenchmarkTask, ) from openadapt_evals.adapters.rl_env import RLEnvironment +from openadapt_evals.errors import ActionParseError from openadapt_evals.task_config import Milestone, TaskCheck, TaskConfig from openadapt_evals.training.areal_workflow import ( SYSTEM_PROMPT, @@ -59,14 +60,20 @@ def mock_step(action): call_count["n"] += 1 done = call_count["n"] >= steps_to_done return ( - BenchmarkObservation(screenshot=fake_png, raw_observation={}), + BenchmarkObservation( + screenshot=fake_png, + viewport=(1920, 1200), + raw_observation={}, + ), done, {"step": call_count["n"]}, ) adapter.step.side_effect = mock_step adapter.reset.return_value = BenchmarkObservation( - screenshot=fake_png, raw_observation={} + screenshot=fake_png, + viewport=(1920, 1200), + raw_observation={}, ) adapter.load_task.return_value = BenchmarkTask( task_id="test-task", instruction="Test instruction", domain="desktop" @@ -301,8 +308,7 @@ def test_dense_reward_with_milestones(self): # 1/2 milestones = 0.5, binary = 0.0, max = 0.5 assert reward == pytest.approx(0.5) - def test_unparseable_llm_output_treated_as_done(self): - """Garbage LLM output should parse as 'done' and end episode.""" + def test_unparseable_llm_output_is_not_task_completion(self): adapter = _make_mock_adapter(steps_to_done=10) env = RLEnvironment(adapter, default_task_id="test-task") @@ -313,20 +319,19 @@ def test_unparseable_llm_output_treated_as_done(self): with patch.object(wf, "_create_env", return_value=env): with patch("openadapt_evals.training.areal_workflow.AsyncOpenAI", return_value=mock_client): - reward = asyncio.run( - wf.run( - data={ - "task_id": "test-task", - "instruction": "Do something", - "max_steps": 5, - }, - base_url="http://fake:8000/v1", - api_key="fake-key", + with pytest.raises(ActionParseError): + asyncio.run( + wf.run( + data={ + "task_id": "test-task", + "instruction": "Do something", + "max_steps": 5, + }, + base_url="http://fake:8000/v1", + api_key="fake-key", + ) ) - ) - assert isinstance(reward, float) - # Unparseable -> done -> no env steps assert adapter.step.call_count == 0 def test_type_action_no_coordinates(self): diff --git a/tests/test_benchmark_result_contract.py b/tests/test_benchmark_result_contract.py new file mode 100644 index 0000000..cc5d621 --- /dev/null +++ b/tests/test_benchmark_result_contract.py @@ -0,0 +1,230 @@ +"""Fail-closed tests for benchmark results from adapter and file boundaries.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +from openadapt_evals.adapters import ( + BenchmarkAction, + BenchmarkResult, + WAAMockAdapter, + normalize_benchmark_result, +) +from openadapt_evals.agents import ScriptedAgent +from openadapt_evals.benchmarks.runner import ( + EvaluationConfig, + _failed_task_result, + _run_single_task, + compute_metrics, +) +from openadapt_evals.integrations.wandb_logger import load_results_from_summary + + +@pytest.mark.parametrize( + "result", + [ + BenchmarkResult(task_id="t", success=1, score=1.0), + BenchmarkResult(task_id="t", success=True, score=float("nan")), + BenchmarkResult(task_id="t", success=True, score=1.1), + BenchmarkResult(task_id="t", success=True, score=1.0, error_type="other"), + BenchmarkResult(task_id="t", success=True, score=0.0), + BenchmarkResult(task_id="t", success=False, score=1.0), + ], +) +def test_malformed_result_becomes_unscored_evaluation_failure( + result: BenchmarkResult, +) -> None: + normalized = normalize_benchmark_result(result) + + assert normalized.success is False + assert normalized.score == 0.0 + assert normalized.error_type == "evaluation" + assert normalized.reason and normalized.reason.startswith("Malformed") + + +def test_partial_success_score_remains_valid() -> None: + result = BenchmarkResult(task_id="t", success=True, score=0.75) + + assert normalize_benchmark_result(result) is result + + +def test_valid_unavailable_category_is_preserved_while_success_is_cleared() -> None: + result = BenchmarkResult( + task_id="t", + success=True, + score=1.0, + error_type="infrastructure", + ) + + normalized = normalize_benchmark_result(result) + + assert normalized.success is False + assert normalized.score == 0.0 + assert normalized.error_type == "infrastructure" + + +def test_done_gate_rejects_malformed_adapter_result() -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = MagicMock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=True, + score=1.0, + error_type="invented", + ) + ) + + result = _run_single_task( + ScriptedAgent([BenchmarkAction(type="done")]), + adapter, + task, + EvaluationConfig( + done_gate=True, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "evaluation" + assert "unknown error_type" in (result.error or "") + + +def test_final_result_from_non_waa_adapter_is_validated() -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = MagicMock( + return_value=BenchmarkResult( + task_id=task.task_id, + success="yes", + score=1.0, + ) + ) + + result = _run_single_task( + ScriptedAgent([BenchmarkAction(type="done")]), + adapter, + task, + EvaluationConfig( + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "evaluation" + assert "success must be a bool" in (result.error or "") + + +def test_aggregation_excludes_malformed_direct_result() -> None: + metrics = compute_metrics( + [BenchmarkResult(task_id="t", success=True, score=1.0, error_type="bad")] + ) + + assert metrics["num_tasks"] == 1 + assert metrics["num_outcome_tasks"] == 0 + assert metrics["num_evaluation_failures"] == 1 + assert metrics["success_count"] == 0 + + +def test_unknown_exception_error_type_becomes_evaluation_failure() -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + + class PluginError(RuntimeError): + error_type = "plugin" + + error = PluginError("bad plugin") + + result = _failed_task_result(task, error) + + assert result.error_type == "evaluation" + + +def test_summary_loader_preserves_valid_diagnostics(tmp_path) -> None: + path = tmp_path / "summary.json" + path.write_text( + json.dumps( + { + "tasks": [ + { + "task_id": "t", + "success": False, + "score": 0.0, + "num_steps": 3, + "total_time_seconds": 1.5, + "error": "offline", + "reason": "evaluator offline", + "error_type": "infrastructure", + } + ] + } + ) + ) + + [result] = load_results_from_summary(path) + + assert result.error_type == "infrastructure" + assert result.error == "offline" + assert result.reason == "evaluator offline" + assert result.num_steps == 3 + assert result.total_time_seconds == 1.5 + + +def test_summary_loader_does_not_invent_missing_error_classification( + tmp_path, +) -> None: + path = tmp_path / "summary.json" + path.write_text( + json.dumps({"tasks": [{"task_id": "t", "success": False, "score": 0.0}]}) + ) + + [result] = load_results_from_summary(path) + + assert result.error_type == "evaluation" + assert "unknown error_type" in (result.reason or "") + + +@pytest.mark.parametrize( + "updates", + [ + {"score": None}, + {"score": float("nan")}, + {"success": "false"}, + {"error_type": "other"}, + {"num_steps": -1}, + {"total_time_seconds": float("inf")}, + ], +) +def test_summary_loader_marks_malformed_rows_unscored(tmp_path, updates) -> None: + row = { + "task_id": "t", + "success": False, + "score": 0.0, + "num_steps": 0, + "total_time_seconds": 0.0, + "error_type": None, + } + row.update(updates) + path = tmp_path / "summary.json" + path.write_text(json.dumps({"tasks": [row]})) + + [result] = load_results_from_summary(path) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == "evaluation" + + +def test_summary_loader_rejects_invalid_document_shape(tmp_path) -> None: + path = tmp_path / "summary.json" + path.write_text(json.dumps({"tasks": {}})) + + with pytest.raises(ValueError, match="tasks list"): + load_results_from_summary(path) diff --git a/tests/test_demo_controller.py b/tests/test_demo_controller.py index 67f22d0..af5d008 100644 --- a/tests/test_demo_controller.py +++ b/tests/test_demo_controller.py @@ -8,6 +8,8 @@ from unittest.mock import MagicMock, patch +import pytest + from openadapt_evals.adapters.base import ( BenchmarkAction, BenchmarkObservation, @@ -20,6 +22,7 @@ PlanStep, run_with_controller, ) +from openadapt_evals.errors import RolloutInfrastructureError from openadapt_evals.plan_verify import VerificationResult # --------------------------------------------------------------------------- @@ -678,8 +681,175 @@ def test_agent_error_returns_failure(self, mock_verify_step, mock_verify_goal): task = _make_task() result = controller.execute(task, max_steps=30) + assert result.success is False + assert result.error_type == "agent" + mock_adapter.evaluate.assert_not_called() + + @patch("openadapt_evals.demo_controller.verify_step") + def test_agent_exception_is_agent_failure(self, mock_verify_step): + controller, mock_agent, mock_adapter, _, _ = self._make_controller() + mock_agent.act.side_effect = RuntimeError("provider rejected output") + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.error_type == "agent" + assert "provider rejected output" in (result.error or "") + mock_adapter.step.assert_not_called() + mock_adapter.evaluate.assert_not_called() + + def test_agent_reset_exception_is_agent_failure(self): + controller, mock_agent, mock_adapter, _, _ = self._make_controller() + mock_agent.reset.side_effect = RuntimeError("agent reset failed") + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.error_type == "agent" + mock_adapter.reset.assert_not_called() + mock_adapter.evaluate.assert_not_called() + + def test_environment_reset_exception_is_infrastructure_failure(self): + controller, _, mock_adapter, _, _ = self._make_controller() + mock_adapter.reset.side_effect = RuntimeError("desktop reset failed") + + result = controller.execute(_make_task(), max_steps=1) + assert result.success is False assert result.error_type == "infrastructure" + assert "desktop reset failed" in (result.error or "") + mock_adapter.evaluate.assert_not_called() + + @patch("openadapt_evals.demo_controller.verify_step") + def test_environment_exception_is_infrastructure_failure(self, mock_verify_step): + controller, _, mock_adapter, _, _ = self._make_controller() + mock_adapter.step.side_effect = RuntimeError("desktop transport offline") + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.error_type == "infrastructure" + assert "desktop transport offline" in (result.error or "") + mock_adapter.evaluate.assert_not_called() + + @pytest.mark.parametrize( + ("error", "expected_type"), + [ + (RuntimeError("oracle failed"), "evaluation"), + (RolloutInfrastructureError("evaluator host offline"), "infrastructure"), + ], + ) + @patch("openadapt_evals.demo_controller.verify_step") + def test_evaluator_exception_keeps_stable_type( + self, mock_verify_step, error, expected_type + ): + controller, _, mock_adapter, _, _ = self._make_controller() + mock_verify_step.return_value = _make_unclear() + mock_adapter.evaluate.side_effect = error + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == expected_type + assert str(error) in (result.error or "") + + @pytest.mark.parametrize( + "malformed", + [ + BenchmarkResult( + task_id="test-task-001", + success="false", # type: ignore[arg-type] + score=0.0, + ), + BenchmarkResult( + task_id="test-task-001", + success=False, + score=float("nan"), + ), + BenchmarkResult( + task_id="test-task-001", + success=True, + score=1.1, + ), + BenchmarkResult( + task_id="wrong-task", + success=True, + score=1.0, + ), + BenchmarkResult( + task_id="test-task-001", + success=True, + score=0.0, + ), + BenchmarkResult( + task_id="test-task-001", + success=False, + score=1.0, + ), + ], + ) + @patch("openadapt_evals.demo_controller.verify_step") + def test_malformed_adapter_result_is_unscored_evaluation_failure( + self, mock_verify_step, malformed + ): + controller, _, _, _, _ = self._make_controller(eval_result=malformed) + mock_verify_step.return_value = _make_unclear() + + result = controller.execute(_make_task(), max_steps=1) + + assert result.task_id == "test-task-001" + assert result.success is False + assert result.score == 0.0 + assert result.error_type == "evaluation" + assert "Malformed demo controller adapter result" in (result.error or "") + + @pytest.mark.parametrize( + ("error", "expected_type"), + [ + (RuntimeError("verifier failed"), "evaluation"), + (RolloutInfrastructureError("verifier host offline"), "infrastructure"), + ], + ) + @patch("openadapt_evals.demo_controller.verify_step") + def test_step_verifier_exception_keeps_stable_type( + self, mock_verify_step, error, expected_type + ): + controller, _, mock_adapter, _, _ = self._make_controller() + mock_verify_step.side_effect = error + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == expected_type + assert str(error) in (result.error or "") + mock_adapter.evaluate.assert_not_called() + + @pytest.mark.parametrize( + ("error", "expected_type"), + [ + (RuntimeError("goal verifier failed"), "evaluation"), + (RolloutInfrastructureError("goal verifier offline"), "infrastructure"), + ], + ) + @patch("openadapt_evals.demo_controller.verify_goal_completion") + def test_goal_verifier_exception_keeps_stable_type( + self, mock_verify_goal, error, expected_type + ): + controller, mock_agent, mock_adapter, _, _ = self._make_controller() + for step in controller.plan_state.steps: + step.status = "done" + mock_verify_goal.side_effect = error + + result = controller.execute(_make_task(), max_steps=1) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == expected_type + assert str(error) in (result.error or "") + mock_agent.act.assert_not_called() + mock_adapter.evaluate.assert_not_called() @patch("openadapt_evals.demo_controller.verify_goal_completion") @patch("openadapt_evals.demo_controller.verify_step") diff --git a/tests/test_demo_executor_e2e.py b/tests/test_demo_executor_e2e.py index 8663580..b1f7e7b 100644 --- a/tests/test_demo_executor_e2e.py +++ b/tests/test_demo_executor_e2e.py @@ -17,6 +17,7 @@ from openadapt_evals.adapters.rl_env import ResetConfig, RolloutStep from openadapt_evals.agents.demo_executor import DemoExecutor from openadapt_evals.demo_library import Demo, DemoStep +from openadapt_evals.errors import ActionExecutionError # --------------------------------------------------------------------------- # Helpers @@ -440,26 +441,21 @@ class TestEdgeCases: @patch("time.sleep", return_value=None) def test_empty_demo(self, _mock_sleep: MagicMock) -> None: - """An empty demo should still reset, evaluate, and return.""" env = MockEnv() executor = DemoExecutor(step_delay=0) demo = _make_demo([]) task_config = _TaskConfig() - score, screenshots = executor.run(env, demo, task_config) + with pytest.raises(ActionExecutionError, match="no executable steps"): + executor.run(env, demo, task_config) assert len(env.actions) == 0 - # With no milestones, falls through to evaluate() - assert score == pytest.approx(0.75) - # Only the reset screenshot - assert len(screenshots) == 1 @patch("time.sleep", return_value=None) - def test_key_step_with_no_value_skipped( + def test_key_step_with_no_value_fails( self, _mock_sleep: MagicMock ) -> None: - """A key step with no action_value should be skipped (no crash).""" env = MockEnv() executor = DemoExecutor(step_delay=0) @@ -477,17 +473,15 @@ def test_key_step_with_no_value_skipped( ]) task_config = _TaskConfig() - score, screenshots = executor.run(env, demo, task_config) + with pytest.raises(ActionExecutionError, match="did not produce an action"): + executor.run(env, demo, task_config) - # Only the second step should execute - assert len(env.actions) == 1 - assert env.actions[0]["key"] == "Return" + assert len(env.actions) == 0 @patch("time.sleep", return_value=None) - def test_unknown_action_type_skipped( + def test_unknown_action_type_fails( self, _mock_sleep: MagicMock ) -> None: - """Unknown action types should be skipped without raising.""" env = MockEnv() executor = DemoExecutor(step_delay=0) @@ -505,10 +499,10 @@ def test_unknown_action_type_skipped( ]) task_config = _TaskConfig() - score, screenshots = executor.run(env, demo, task_config) + with pytest.raises(ActionExecutionError, match="did not produce an action"): + executor.run(env, demo, task_config) - assert len(env.actions) == 1 - assert env.actions[0]["key"] == "Escape" + assert len(env.actions) == 0 @patch("time.sleep", return_value=None) def test_milestones_checked_per_step( diff --git a/tests/test_demo_executor_http_grounder.py b/tests/test_demo_executor_http_grounder.py index 7a8c2fc..b77e98e 100644 --- a/tests/test_demo_executor_http_grounder.py +++ b/tests/test_demo_executor_http_grounder.py @@ -230,17 +230,15 @@ def test_bbox_center_click( assert abs(action.x - 0.2) < 0.01 assert abs(action.y - 0.3) < 0.01 - def test_http_error_returns_center_fallback( + def test_http_error_does_not_invent_center_click( self, executor_with_endpoint, fake_observation ): - """HTTP errors should return a fallback click at center.""" with patch("requests.post", side_effect=Exception("Connection refused")): action = executor_with_endpoint._ground_click_http( fake_observation, "target" ) - assert action.type == "click" - assert action.x == 0.5 - assert action.y == 0.5 + assert action.type == "error" + assert "Connection refused" in action.raw_action["grounder_error"] def test_no_screenshot(self, executor_with_endpoint): """Should still work without a screenshot (text-only grounding).""" diff --git a/tests/test_dense_rewards.py b/tests/test_dense_rewards.py index 0bbb7fc..271b90b 100644 --- a/tests/test_dense_rewards.py +++ b/tests/test_dense_rewards.py @@ -4,11 +4,14 @@ from unittest.mock import MagicMock, patch +import pytest + from openadapt_evals.adapters.base import ( BenchmarkAction, BenchmarkObservation, BenchmarkResult, BenchmarkTask, + EvaluationUnavailableError, ) from openadapt_evals.adapters.rl_env import ResetConfig, RLEnvironment from openadapt_evals.task_config import Milestone, TaskCheck, TaskConfig @@ -286,3 +289,56 @@ def test_local_not_called_when_binary_succeeds(self): mock_local.assert_not_called() assert score >= 0.75 + + def test_measured_milestone_can_replace_unavailable_binary_evaluator(self): + adapter = _make_adapter() + adapter.evaluate.return_value = BenchmarkResult( + task_id="test-001", + success=False, + score=0.0, + error_type="infrastructure", + ) + check = TaskCheck(check="command", run="echo ok", expect="ok") + task_config = _make_task_config( + milestones=[Milestone(name="Saved", check=check)] + ) + env = RLEnvironment(adapter, task_config=task_config) + env.reset(config=ResetConfig(task_id="test-001")) + + with patch.object(TaskConfig, "_run_vm_command", return_value="ok"): + assert env.evaluate_dense() == 1.0 + + def test_unmeasured_milestone_does_not_hide_evaluator_failure(self): + adapter = _make_adapter() + adapter.evaluate.return_value = BenchmarkResult( + task_id="test-001", + success=False, + score=0.0, + error_type="infrastructure", + ) + check = TaskCheck(check="command", run="echo ok", expect="ok") + task_config = _make_task_config( + milestones=[Milestone(name="Saved", check=check)] + ) + env = RLEnvironment(adapter, task_config=task_config) + env.reset(config=ResetConfig(task_id="test-001")) + + with ( + patch.object(TaskConfig, "_run_vm_command", side_effect=OSError("offline")), + pytest.raises(EvaluationUnavailableError), + ): + env.evaluate_dense() + + def test_local_check_exception_is_not_a_measured_zero(self): + check = TaskCheck(check="screenshot", description="Saved result") + task_config = _make_task_config() + task_config.checks = [check] + + with ( + patch( + "openadapt_evals.vlm_evaluator.vlm_judge", + side_effect=RuntimeError("judge offline"), + ), + pytest.raises(EvaluationUnavailableError), + ): + task_config.evaluate_checks_local(b"image", "http://localhost:5001") diff --git a/tests/test_evaluate_endpoint.py b/tests/test_evaluate_endpoint.py index 27e75a2..3d09b80 100644 --- a/tests/test_evaluate_endpoint.py +++ b/tests/test_evaluate_endpoint.py @@ -2,12 +2,16 @@ from unittest.mock import MagicMock, patch +import pytest + from openadapt_evals.server.evaluate_endpoint import ( + EvaluationNotRunError, MockEnv, StandaloneMetrics, _truncate_value, create_standalone_evaluator, evaluate_task_state, + get_expected_value, ) @@ -144,6 +148,38 @@ def test_evaluator_load_failure(self): assert result["success"] is False assert "Failed to load" in result["reason"] + def test_missing_expected_contract_cannot_match_missing_actual(self): + getters = MagicMock() + getters.get_thing.return_value = None + metrics = MagicMock() + metrics.exact_match.return_value = 1.0 + + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + result = evaluate_task_state( + {"evaluator": {"result": {"type": "thing"}}} + ) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + metrics.exact_match.assert_not_called() + + +def test_expected_getter_failure_is_not_a_null_expected_value(): + getters = MagicMock() + getters.get_reference.side_effect = RuntimeError("offline") + + with pytest.raises(EvaluationNotRunError) as excinfo: + get_expected_value( + {"expected": {"type": "reference"}}, + env=MagicMock(), + getters=getters, + ) + + assert excinfo.value.error_type == "infrastructure" + class TestCreateStandaloneEvaluator: """Tests for create_standalone_evaluator function.""" @@ -176,7 +212,10 @@ def test_basic_evaluation(self): with patch("requests.post") as mock_post: mock_response = MagicMock() mock_response.status_code = 200 - mock_response.json.return_value = {"output": "test"} + mock_response.json.return_value = { + "returncode": 0, + "output": "test", + } mock_post.return_value = mock_response result = evaluate(config) diff --git a/tests/test_evaluator_boundaries.py b/tests/test_evaluator_boundaries.py new file mode 100644 index 0000000..bad398f --- /dev/null +++ b/tests/test_evaluator_boundaries.py @@ -0,0 +1,431 @@ +"""Fail-closed evaluator boundary regression tests.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from openadapt_evals.server.evaluate_endpoint import ( + EvaluationNotRunError, + StandaloneMetrics, + create_standalone_evaluator, + evaluate_task_state, + run_metric, +) + + +def _task_with_postconfig(postconfig: list) -> dict: + return { + "evaluator": { + "result": {"type": "thing"}, + "expected": {"value": "saved"}, + "func": "exact_match", + "postconfig": postconfig, + } + } + + +@pytest.mark.parametrize( + ("postconfig", "status", "error_type"), + [ + ([{"type": "unknown"}], 200, "evaluation"), + ([{"type": "activate_window", "name": "App"}], 500, "infrastructure"), + ([{"type": "execute", "command": "save"}], 503, "infrastructure"), + ], +) +def test_required_postconfig_failure_is_unscored( + postconfig: list, status: int, error_type: str +) -> None: + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metric = MagicMock(return_value=1.0) + metrics = SimpleNamespace(exact_match=metric) + response = MagicMock(status_code=status) + + with ( + patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ), + patch("requests.post", return_value=response), + ): + result = evaluate_task_state(_task_with_postconfig(postconfig)) + + assert result["scored"] is False + assert result["error_type"] == error_type + metric.assert_not_called() + + +def test_postconfig_request_exception_is_unscored_infrastructure() -> None: + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metric = MagicMock(return_value=1.0) + metrics = SimpleNamespace(exact_match=metric) + + with ( + patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ), + patch("requests.post", side_effect=requests.ConnectionError("offline")), + ): + result = evaluate_task_state( + _task_with_postconfig([{"type": "execute", "command": "save"}]) + ) + + assert result["scored"] is False + assert result["error_type"] == "infrastructure" + metric.assert_not_called() + + +@pytest.mark.parametrize( + "receipt", + [ + {"success": False}, + {"success": True, "stderr": "warning"}, + {"delivery_state": "uncertain"}, + {"delivery_state": "invalid"}, + {}, + ], +) +def test_postconfig_http_200_requires_success_receipt(receipt: object) -> None: + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metric = MagicMock(return_value=1.0) + metrics = SimpleNamespace(exact_match=metric) + response = MagicMock(status_code=200) + response.json.return_value = receipt + + with ( + patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ), + patch("requests.post", return_value=response), + ): + result = evaluate_task_state( + _task_with_postconfig([{"type": "execute", "command": "save"}]) + ) + + assert result["scored"] is False + assert result["error_type"] == "infrastructure" + metric.assert_not_called() + + +def test_postconfig_success_receipt_allows_evaluation() -> None: + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metric = MagicMock(return_value=1.0) + metrics = SimpleNamespace(exact_match=metric) + response = MagicMock(status_code=200) + response.json.return_value = {"success": True, "delivery_state": "delivered"} + + with ( + patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ), + patch("requests.post", return_value=response), + ): + result = evaluate_task_state( + _task_with_postconfig([{"type": "execute", "command": "save"}]) + ) + + assert result["scored"] is True + assert result["success"] is True + + +@pytest.mark.parametrize("score", [True, float("nan"), float("inf"), -0.1, 1.1]) +def test_metric_rejects_invalid_scores(score: object) -> None: + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: score) + + with pytest.raises(EvaluationNotRunError, match="invalid score"): + run_metric("exact_match", "saved", "saved", metrics=metrics) + + +def _standalone_config(**overrides: object) -> dict: + evaluator = { + "result": {"type": "vm_command_line", "command": "read"}, + "expected": {"value": "saved"}, + "func": "exact_match", + } + evaluator.update(overrides) + return {"evaluator": evaluator} + + +@pytest.mark.parametrize( + "result_spec", + [None, {}, {"type": "vm_command_line"}, {"type": "vm_file"}], +) +def test_missing_or_incomplete_result_contract_is_unscored( + result_spec: object, +) -> None: + task = { + "evaluator": { + "result": result_spec, + "expected": {"value": ""}, + "func": "exact_match", + } + } + getters = SimpleNamespace( + get_vm_command_line=lambda _env, _spec: "", + get_vm_file=lambda _env, _spec: "", + ) + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + result = evaluate_task_state(task) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +@pytest.mark.parametrize( + "result_spec", + [None, {}, {"type": "vm_command_line"}, {"type": "vm_file"}], +) +def test_standalone_missing_or_incomplete_result_is_unscored( + result_spec: object, +) -> None: + evaluate = create_standalone_evaluator() + result = evaluate( + { + "evaluator": { + "result": result_spec, + "expected": {"value": ""}, + "func": "exact_match", + } + } + ) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_whitespace_only_command_contract_is_unscored_in_both_evaluators() -> None: + task = { + "evaluator": { + "result": {"type": "vm_command_line", "command": " "}, + "expected": {"value": ""}, + "func": "exact_match", + } + } + getters = SimpleNamespace(get_vm_command_line=lambda _env, _spec: "") + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + main_result = evaluate_task_state(task) + standalone_result = create_standalone_evaluator()(task) + + assert main_result["scored"] is False + assert standalone_result["scored"] is False + + +def test_whitespace_only_postconfig_command_is_unscored() -> None: + task = _task_with_postconfig([{"type": "execute", "command": " "}]) + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + result = evaluate_task_state(task) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_standalone_http_200_failure_cannot_verify_empty_output() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=200) + response.json.return_value = { + "success": False, + "error": "failed", + "output": "", + } + config = _standalone_config(expected={"value": ""}) + + with patch("requests.post", return_value=response): + result = evaluate(config) + + assert result["scored"] is False + assert result["error_type"] == "infrastructure" + + +def test_standalone_passes_declared_metric_options() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=200) + response.json.return_value = {"returncode": 0, "output": "saved"} + metric = MagicMock(return_value=1.0) + + with ( + patch("requests.post", return_value=response), + patch.object(StandaloneMetrics, "exact_match", metric), + ): + result = evaluate(_standalone_config(options={"strict": True})) + + assert result["scored"] is True + metric.assert_called_once_with("saved", "saved", strict=True) + + +def test_standalone_refuses_declared_postconfig_it_cannot_execute() -> None: + evaluate = create_standalone_evaluator() + result = evaluate( + _standalone_config(postconfig=[{"type": "execute", "command": "prepare"}]) + ) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +@pytest.mark.parametrize("postconfig", [None, {}, ""]) +def test_main_evaluator_rejects_malformed_falsy_postconfig( + postconfig: object, +) -> None: + task = _task_with_postconfig([]) + task["evaluator"]["postconfig"] = postconfig + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + result = evaluate_task_state(task) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +@pytest.mark.parametrize("infeasible", ["false", 1, {}]) +def test_malformed_infeasible_marker_cannot_turn_fail_into_success( + infeasible: object, +) -> None: + task = { + "evaluator": {"infeasible": infeasible}, + "agent_last_action": "FAIL", + } + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(MagicMock(), MagicMock()), + ): + result = evaluate_task_state(task) + + assert result["success"] is False + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_infeasible_marker_requires_string_last_action() -> None: + task = { + "evaluator": {"infeasible": True}, + "agent_last_action": 1, + } + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(MagicMock(), MagicMock()), + ): + result = evaluate_task_state(task) + + assert result["success"] is False + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_standalone_unknown_metric_is_not_substituted() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=200) + response.json.return_value = { + "success": True, + "delivery_state": "delivered", + "output": "saved", + } + with patch("requests.post", return_value=response): + result = evaluate(_standalone_config(func="not_a_metric")) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_standalone_getter_exception_is_not_a_measured_zero() -> None: + evaluate = create_standalone_evaluator() + with patch( + "openadapt_evals.server.evaluate_endpoint.StandaloneGetters.get_vm_command_line", + side_effect=RuntimeError("offline"), + ): + result = evaluate(_standalone_config()) + + assert result["scored"] is False + assert result["error_type"] == "infrastructure" + + +def test_standalone_non_2xx_getter_is_unscored_infrastructure() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=503) + with patch("requests.post", return_value=response): + result = evaluate(_standalone_config()) + + assert result["scored"] is False + assert result["error_type"] == "infrastructure" + + +@pytest.mark.parametrize( + "overrides", + [ + {"func": 1}, + {"func": ["exact_match", ""]}, + {"options": []}, + {"conj": "maybe"}, + ], +) +def test_evaluator_rejects_malformed_metric_contract(overrides: dict) -> None: + task = _task_with_postconfig([]) + task["evaluator"].update(overrides) + getters = SimpleNamespace(get_thing=lambda _env, _spec: "saved") + metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + + with patch( + "openadapt_evals.server.evaluate_endpoint._load_waa_evaluators", + return_value=(getters, metrics), + ): + result = evaluate_task_state(task) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_standalone_metric_exception_is_not_a_measured_zero() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=200) + response.json.return_value = { + "success": True, + "delivery_state": "delivered", + "output": "saved", + } + with ( + patch("requests.post", return_value=response), + patch.object(StandaloneMetrics, "exact_match", side_effect=RuntimeError("bad")), + ): + result = evaluate(_standalone_config()) + + assert result["scored"] is False + assert result["error_type"] == "evaluation" + + +def test_standalone_success_emits_canonical_scored_contract() -> None: + evaluate = create_standalone_evaluator() + response = MagicMock(status_code=200) + response.json.return_value = { + "success": True, + "delivery_state": "delivered", + "output": "saved", + } + with patch("requests.post", return_value=response): + result = evaluate(_standalone_config()) + + assert result["success"] is True + assert result["score"] == 1.0 + assert result["scored"] is True + assert result["error_type"] is None diff --git a/tests/test_evaluator_client_failure_visibility.py b/tests/test_evaluator_client_failure_visibility.py index 8458bf1..d0b13c4 100644 --- a/tests/test_evaluator_client_failure_visibility.py +++ b/tests/test_evaluator_client_failure_visibility.py @@ -13,6 +13,8 @@ import builtins from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest import requests @@ -141,6 +143,200 @@ def test_vm_reported_command_error_is_not_an_empty_measurement( client._require_output({"error": "WinError 5", "output": ""}, "file_content") +@pytest.mark.parametrize("receipt", [{}, None, {"output": None}, {"output": 1}]) +def test_missing_or_non_string_getter_output_is_not_measured( + tmp_path: Path, receipt: object +) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + + with pytest.raises(EvaluationError): + client._require_output(receipt, "file_content") + + +def test_explicit_empty_getter_output_remains_valid(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + + assert client._require_output( + {"returncode": 0, "output": ""}, "file_content" + ) == "" + + +def test_delivery_alone_does_not_prove_command_completion(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + + with pytest.raises(EvaluationError): + client._require_output( + {"delivery_state": "delivered", "output": ""}, + "file_content", + ) + + +@pytest.mark.parametrize( + "receipt", + [ + {"success": False, "output": "saved"}, + {"returncode": 0, "stderr": "warning", "output": "saved"}, + {"delivery_state": "uncertain", "output": "saved"}, + {"delivery_state": "invalid", "output": "saved"}, + {"output": "saved"}, + ], +) +def test_failed_or_unproved_getter_receipt_is_not_measured( + tmp_path: Path, receipt: dict +) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + + with pytest.raises(EvaluationError): + client._require_output(receipt, "file_content") + + +def test_empty_process_name_cannot_match_every_output(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + env = MagicMock() + + with pytest.raises(EvaluationError): + client._fallback_getter(env, "process_running", {"process": ""}) + env.execute.assert_not_called() + + +def test_loaded_getter_requires_known_result_parameters(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + client._getters = SimpleNamespace(get_file_content=lambda _env, _spec: "") + client._metrics = SimpleNamespace(exact_match=lambda _actual, _expected: 1.0) + + result = client.evaluate({ + "evaluator": { + "result": {"type": "file_content"}, + "expected": {"value": ""}, + "func": "exact_match", + } + }) + + assert result.success is False + assert result.scored is False + assert result.error_type == "evaluation" + + +def test_metric_options_are_applied_and_invalid_conjunction_is_refused( + tmp_path: Path, +) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + metric = MagicMock(side_effect=lambda _a, _e, strict=False: 0.0 if strict else 1.0) + client._metrics = SimpleNamespace(exact_match=metric) + + assert client._run_metric( + {"func": "exact_match", "options": {"strict": True}}, "a", "a" + ) == 0.0 + metric.assert_called_once_with("a", "a", strict=True) + + with pytest.raises(EvaluationError, match="conj='and'"): + client._run_metric({"func": "exact_match", "conj": "invalid"}, "a", "a") + + +def test_failed_http_receipt_is_classified_as_infrastructure(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + response = MagicMock(status_code=200) + response.raise_for_status.return_value = None + response.json.return_value = {"success": False, "output": ""} + + with patch("requests.post", return_value=response): + result = client.evaluate({ + "evaluator": { + "result": {"type": "file_content", "path": "C:/out.txt"}, + "expected": {"value": ""}, + "func": "exact_match", + } + }) + + assert result.success is False + assert result.scored is False + assert result.error_type == "infrastructure" + + +def test_duplicate_outer_receipt_keys_cannot_score_exact_empty(tmp_path: Path) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + response = requests.Response() + response.status_code = 200 + response._content = b'{"returncode":1,"returncode":0,"output":""}' + + with patch("requests.post", return_value=response): + result = client.evaluate({ + "evaluator": { + "result": {"type": "file_content", "path": "C:/out.txt"}, + "expected": {"value": ""}, + "func": "exact_match", + } + }) + + assert result.success is False + assert result.scored is False + assert result.error_type == "infrastructure" + + +@pytest.mark.parametrize(("value", "rendered"), [("", ""), (0, "0"), (False, "False")]) +def test_result_serialization_preserves_falsy_evidence( + value: object, + rendered: str, +) -> None: + payload = EvaluationResult(True, 1.0, actual=value, expected=value).to_dict() + + assert payload["actual"] == rendered + assert payload["expected"] == rendered + + +@pytest.mark.parametrize("expected", [None, {}, {"type": "literal"}]) +def test_missing_or_malformed_expected_contract_is_unscored( + tmp_path: Path, expected: object +) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + config = {"result": {"type": "file_content", "path": "C:/out.txt"}} + if expected is not None: + config["expected"] = expected + + with pytest.raises(EvaluationError): + client._get_expected_value(config) + + def test_unknown_metric_name_is_not_silently_scored_with_exact_match( tmp_path: Path, ) -> None: @@ -167,6 +363,22 @@ def test_known_metric_still_scores_normally(tmp_path: Path) -> None: assert client._fallback_metric("exact_match", "a", "b") == 0.0 +@pytest.mark.parametrize("score", [float("nan"), float("inf"), -0.1, 1.1, True]) +def test_invalid_metric_score_is_not_a_measurement( + tmp_path: Path, score: object +) -> None: + client = EvaluatorClient( + vm_ip="10.0.0.1", + waa_evaluators_path=tmp_path, + require_waa_evaluators=False, + ) + client._metrics = MagicMock() + client._metrics.exact_match.return_value = score + + with pytest.raises(EvaluationError): + client._run_metric({"func": "exact_match"}, "a", "a") + + def test_missing_evaluator_config_is_flagged_not_scored(tmp_path: Path) -> None: """A task with no evaluator spec was never measured.""" client = EvaluatorClient( diff --git a/tests/test_false_success_sweep.py b/tests/test_false_success_sweep.py index 1c16055..6978f06 100644 --- a/tests/test_false_success_sweep.py +++ b/tests/test_false_success_sweep.py @@ -84,6 +84,41 @@ def test_genuine_zero_is_still_a_measurement(self) -> None: assert env.evaluate() == 0.0 + @pytest.mark.parametrize("score", [float("nan"), float("inf"), -0.1, 1.1]) + def test_invalid_score_is_not_a_reward(self, score: float) -> None: + adapter = _adapter( + BenchmarkResult( + task_id="t1", success=False, score=score, error_type=None + ) + ) + env = RLEnvironment(adapter) + env.reset(ResetConfig(task_id="t1")) + env.step(BenchmarkAction(type="click", x=1, y=1)) + env.trajectory[-1].reward = -1.0 + + with pytest.raises(EvaluationUnavailableError, match="score"): + env.evaluate() + + assert env.trajectory[-1].reward == -1.0 + + def test_any_error_type_is_not_flattened_to_a_score(self) -> None: + adapter = _adapter( + BenchmarkResult( + task_id="t1", + success=False, + score=0.0, + error_type="agent", + reason="agent action could not be evaluated", + ) + ) + env = RLEnvironment(adapter) + env.reset(ResetConfig(task_id="t1")) + + with pytest.raises(EvaluationUnavailableError) as excinfo: + env.evaluate() + + assert excinfo.value.error_type == "agent" + def test_evaluate_result_exposes_the_tristate(self) -> None: adapter = _adapter( BenchmarkResult( @@ -652,3 +687,67 @@ def test_live_adapter_does_not_accept_an_unscored_200_as_a_result() -> None: task, {"success": False, "score": 0.0, "scored": True, "error_type": None}, ) assert measured.error_type is None + + +@pytest.mark.parametrize( + "body", + [ + None, + [], + {}, + {"success": 1, "score": 1.0}, + {"success": True, "score": True}, + {"success": False, "score": float("nan")}, + {"success": False, "score": float("inf")}, + {"success": False, "score": -0.1}, + {"success": True, "score": 1.1}, + {"success": True, "score": 0.9}, + {"success": False, "score": 1.0}, + {"success": False, "score": 0.0, "scored": "false"}, + {"success": False, "score": 0.0, "error_type": "agent"}, + { + "success": False, + "score": 0.0, + "scored": True, + "error_type": "infrastructure", + }, + { + "success": False, + "score": 0.5, + "scored": False, + "error_type": "evaluation", + }, + ], +) +def test_live_adapter_rejects_malformed_evaluate_responses(body: object) -> None: + from openadapt_evals.adapters.waa.live import WAALiveAdapter + + adapter = WAALiveAdapter.__new__(WAALiveAdapter) + adapter._step_count = 2 + + result = adapter._result_from_evaluate_response(_task(), body) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == "evaluation" + assert result.reason and result.reason.startswith("Malformed evaluation response:") + + +def test_live_adapter_accepts_canonical_partial_and_complete_measurements() -> None: + from openadapt_evals.adapters.waa.live import WAALiveAdapter + + adapter = WAALiveAdapter.__new__(WAALiveAdapter) + adapter._step_count = 2 + task = _task() + + partial = adapter._result_from_evaluate_response( + task, + {"success": False, "score": 0.5, "scored": True, "error_type": None}, + ) + complete = adapter._result_from_evaluate_response( + task, + {"success": True, "score": 1.0}, + ) + + assert (partial.success, partial.score, partial.error_type) == (False, 0.5, None) + assert (complete.success, complete.score, complete.error_type) == (True, 1.0, None) diff --git a/tests/test_local_adapter.py b/tests/test_local_adapter.py index ff12817..0dccbd3 100644 --- a/tests/test_local_adapter.py +++ b/tests/test_local_adapter.py @@ -2,6 +2,8 @@ from __future__ import annotations +import sys +from types import SimpleNamespace from unittest.mock import patch import pytest @@ -20,6 +22,13 @@ BenchmarkTask, ) from openadapt_evals.adapters.local import LocalAdapter +from openadapt_evals.adapters.local.adapter import _get_input_monitor_geometry +from openadapt_evals.errors import ( + ActionDeliveredObservationError, + ActionDeliveryState, + ActionDeliveryUncertainError, + ActionExecutionError, +) # --------------------------------------------------------------------------- # Fixtures @@ -62,6 +71,11 @@ def _make_mock_observe(adapter): """Patch adapter.observe to return a fake observation without screen capture.""" def _fake_observe(): + adapter._record_capture_geometry( + {"left": 0, "top": 0, "width": 1920, "height": 1080}, + (1920, 1080), + {"left": 0.0, "top": 0.0, "width": 1920.0, "height": 1080.0}, + ) return BenchmarkObservation( screenshot=MINIMAL_PNG, viewport=(1920, 1080), @@ -130,6 +144,42 @@ def test_reset_clears_step_count(self, adapter, sample_task): adapter.reset(sample_task) assert adapter._step_count == 0 + @pytest.mark.parametrize( + "action", + [ + BenchmarkAction(type="click", x=1, y=1), + BenchmarkAction(type="type", text="sensitive"), + BenchmarkAction(type="key", key="enter"), + ], + ) + def test_failed_reset_invalidates_task_and_geometry( + self, + adapter, + sample_task, + action, + ): + _make_mock_observe(adapter) + adapter.reset(sample_task) + + with ( + patch.object(adapter, "observe", side_effect=RuntimeError("capture failed")), + pytest.raises(RuntimeError, match="capture failed"), + ): + adapter.reset(sample_task) + + with ( + patch.object(adapter, "_execute_action") as execute, + pytest.raises(ActionExecutionError, match=r"reset\(\)"), + ): + adapter.step(action) + + execute.assert_not_called() + assert adapter._current_task is None + assert adapter._last_viewport is None + assert adapter._capture_origin is None + assert adapter._capture_scale is None + assert adapter._capture_pixel_size is None + # --------------------------------------------------------------------------- # step() @@ -198,15 +248,103 @@ def test_step_error_action_sets_done_true(self, adapter, sample_task): _, done, _ = adapter.step(BenchmarkAction(type="error")) assert done is True - def test_step_handles_execution_error(self, adapter, sample_task): + def test_step_does_not_report_execution_error_as_completion(self, adapter, sample_task): _make_mock_observe(adapter) adapter.reset(sample_task) with patch.object(adapter, "_execute_action", side_effect=RuntimeError("perm")): - obs, done, info = adapter.step(BenchmarkAction(type="click", x=1, y=1)) + with pytest.raises(ActionDeliveryUncertainError, match="perm") as raised: + adapter.step(BenchmarkAction(type="click", x=1, y=1)) - assert done is True - assert "error" in info + assert adapter._step_count == 0 + assert raised.value.delivery_state is ActionDeliveryState.UNCERTAIN + assert raised.value.retry_safe is False + + @pytest.mark.parametrize( + ("action", "message"), + [ + (BenchmarkAction(type="unknown"), "Unsupported local action"), + (BenchmarkAction(type="click", x=None, y=2), "requires x"), + (BenchmarkAction(type="click", x=1, y=None), "requires y"), + (BenchmarkAction(type="type", text=None), "requires text"), + (BenchmarkAction(type="key", key=None), "requires key"), + ( + BenchmarkAction(type="scroll", scroll_direction=None), + "requires direction", + ), + ( + BenchmarkAction(type="scroll", scroll_direction="diagonal"), + "requires direction", + ), + ( + BenchmarkAction(type="drag", x=1, y=2, end_x=3, end_y=None), + "requires end_y", + ), + ], + ) + def test_invalid_action_is_rejected_before_dispatch( + self, adapter, sample_task, action, message + ): + _make_mock_observe(adapter) + adapter.reset(sample_task) + + with ( + patch.object(adapter, "_execute_action") as execute, + pytest.raises(ActionExecutionError, match=message), + ): + adapter.step(action) + + execute.assert_not_called() + assert adapter._step_count == 0 + + def test_explicit_empty_text_remains_valid(self, adapter, sample_task): + _make_mock_observe(adapter) + adapter.reset(sample_task) + + with patch.object(adapter, "_execute_action") as execute: + adapter.step(BenchmarkAction(type="type", text="")) + + execute.assert_called_once() + assert adapter._step_count == 1 + + @pytest.mark.parametrize( + "action", + [ + BenchmarkAction(type="click", x=-1, y=20), + BenchmarkAction(type="click", x=1920, y=20), + BenchmarkAction(type="drag", x=1, y=2, end_x=30, end_y=1080), + ], + ) + def test_out_of_viewport_pointer_action_is_rejected( + self, adapter, sample_task, action + ): + _make_mock_observe(adapter) + adapter._last_viewport = (1920, 1080) + adapter.reset(sample_task) + + with pytest.raises(ActionExecutionError, match="viewport|non-negative") as raised: + adapter.step(action) + + assert raised.value.delivery_state is ActionDeliveryState.NOT_DELIVERED + assert raised.value.retry_safe is True + assert adapter._step_count == 0 + + def test_observation_failure_preserves_confirmed_delivery( + self, adapter, sample_task + ): + _make_mock_observe(adapter) + adapter.reset(sample_task) + + with ( + patch.object(adapter, "_execute_action"), + patch.object(adapter, "observe", side_effect=RuntimeError("capture failed")), + pytest.raises(ActionDeliveredObservationError) as raised, + ): + adapter.step(BenchmarkAction(type="click", x=1, y=1)) + + assert raised.value.delivery_state is ActionDeliveryState.DELIVERED + assert raised.value.retry_safe is False + assert adapter._step_count == 1 # --------------------------------------------------------------------------- @@ -224,6 +362,7 @@ def test_evaluate_score_is_zero(self, adapter, sample_task): result = adapter.evaluate(sample_task) assert result.score == 0.0 assert result.success is False + assert result.error_type == "evaluation" # --------------------------------------------------------------------------- @@ -258,14 +397,53 @@ def test_context_manager(self): class TestLocalAdapterScaling: def test_to_logical_scale_1(self, adapter): - adapter._scale = 1.0 + adapter._record_capture_geometry( + {"left": 0, "top": 0, "width": 1920, "height": 1080}, + (1920, 1080), + {"left": 0.0, "top": 0.0, "width": 1920.0, "height": 1080.0}, + ) assert adapter._to_logical(100, 200) == (100.0, 200.0) - def test_to_logical_scale_2(self, adapter): - adapter._scale = 2.0 + def test_to_logical_uses_retina_secondary_monitor_input_geometry(self, adapter): + adapter._record_capture_geometry( + {"left": -2560, "top": 240, "width": 2560, "height": 1440}, + (2560, 1440), + {"left": -1280.0, "top": 120.0, "width": 1280.0, "height": 720.0}, + ) lx, ly = adapter._to_logical(200, 400) - assert lx == pytest.approx(100.0) - assert ly == pytest.approx(200.0) + assert lx == pytest.approx(-1180.0) + assert ly == pytest.approx(320.0) + + def test_to_logical_rejects_missing_geometry(self, adapter): + with pytest.raises(ActionExecutionError, match="capture geometry"): + adapter._to_logical(100, 200) + + def test_to_logical_rejects_viewport_geometry_mismatch(self, adapter): + adapter._record_capture_geometry( + {"left": 0, "top": 0, "width": 1280, "height": 720}, + (2560, 1440), + {"left": 0.0, "top": 0.0, "width": 1280.0, "height": 720.0}, + ) + adapter._last_viewport = (1280, 720) + + with pytest.raises(ActionExecutionError, match="matching capture geometry"): + adapter._to_logical(100, 200) + + def test_rotated_macos_display_refuses_unmodeled_transform(self): + quartz = SimpleNamespace( + kCGErrorSuccess=0, + CGGetActiveDisplayList=lambda *_args: (0, [42], 1), + CGDisplayRotation=lambda _display_id: 90.0, + ) + with ( + patch("platform.system", return_value="Darwin"), + patch.dict(sys.modules, {"Quartz": quartz}), + pytest.raises(ActionExecutionError, match="bind the selected macOS display"), + ): + _get_input_monitor_geometry( + 1, + {"left": 0, "top": 0, "width": 1440, "height": 2560}, + ) # --------------------------------------------------------------------------- diff --git a/tests/test_metrics.py b/tests/test_metrics.py index da0912a..ebff34a 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -1,9 +1,12 @@ """Tests for shared evaluation metrics module.""" +import pytest + from openadapt_evals.evaluation.metrics import ( boolean, contains, exact_match, + file_exists, fuzzy_match, get_metric, ) @@ -23,6 +26,10 @@ def test_numbers(self): assert exact_match(42, 42) == 1.0 assert exact_match(42, 43) == 0.0 + def test_missing_values_are_not_a_match(self): + with pytest.raises(ValueError): + exact_match(None, None) + class TestFuzzyMatch: def test_exact_match_high_score(self): @@ -48,6 +55,10 @@ def test_negative(self): def test_case_insensitive(self): assert contains("Hello World", "WORLD") == 1.0 + def test_empty_expectation_is_not_automatic_success(self): + with pytest.raises(ValueError): + contains("anything", "") + class TestBoolean: def test_both_truthy(self): @@ -59,6 +70,21 @@ def test_both_falsy(self): def test_mismatch(self): assert boolean(1, False) == 0.0 + def test_boolean_strings_are_parsed_not_treated_as_truthy(self): + assert boolean("false", "true") == 0.0 + + def test_ambiguous_boolean_is_rejected(self): + with pytest.raises(ValueError): + boolean("yes", True) + + +class TestFileExists: + def test_requires_remote_boolean_evidence(self): + assert file_exists(True, True) == 1.0 + assert file_exists(False, True) == 0.0 + with pytest.raises(ValueError): + file_exists("/tmp/local-path", "/tmp/local-path") + class TestGetMetric: def test_known_metric(self): diff --git a/tests/test_milestone_screenshot_eval.py b/tests/test_milestone_screenshot_eval.py index db528a4..6f74af1 100644 --- a/tests/test_milestone_screenshot_eval.py +++ b/tests/test_milestone_screenshot_eval.py @@ -4,6 +4,9 @@ from unittest.mock import patch +import pytest + +from openadapt_evals.adapters.base import EvaluationUnavailableError from openadapt_evals.task_config import ( Milestone, TaskCheck, @@ -26,19 +29,20 @@ def _make_task_config(milestones): class TestEvaluateMilestonesScreenshot: - def test_no_milestones_returns_zero(self): + def test_no_milestones_is_unmeasured(self): task = _make_task_config([]) - assert evaluate_milestones_screenshot(task, b"fake-png") == 0.0 + with pytest.raises(EvaluationUnavailableError, match="No milestone"): + evaluate_milestones_screenshot(task, b"fake-png") - def test_no_screenshot_milestones_returns_zero(self): - """Non-screenshot milestones are skipped entirely.""" + def test_non_screenshot_milestone_is_not_silently_skipped(self): task = _make_task_config([ Milestone( name="Command check", check=TaskCheck(check="command", run="echo 1", expect="1"), ), ]) - assert evaluate_milestones_screenshot(task, b"fake-png") == 0.0 + with pytest.raises(EvaluationUnavailableError, match="cannot measure"): + evaluate_milestones_screenshot(task, b"fake-png") @patch("openadapt_evals.vlm_evaluator.vlm_judge") def test_all_pass(self, mock_vlm): @@ -74,8 +78,7 @@ def test_partial_pass(self, mock_vlm): assert score == 0.5 @patch("openadapt_evals.vlm_evaluator.vlm_judge") - def test_skips_non_screenshot_milestones(self, mock_vlm): - """Mixed milestones: only screenshot ones are evaluated.""" + def test_mixed_contract_is_not_partially_measured(self, mock_vlm): mock_vlm.return_value = (True, 0.9) task = _make_task_config([ Milestone( @@ -87,9 +90,9 @@ def test_skips_non_screenshot_milestones(self, mock_vlm): check=TaskCheck(check="screenshot", description="Something visible"), ), ]) - score = evaluate_milestones_screenshot(task, b"fake-png") - assert score == 1.0 # 1/1 screenshot milestone passed - mock_vlm.assert_called_once() + with pytest.raises(EvaluationUnavailableError, match="cannot measure"): + evaluate_milestones_screenshot(task, b"fake-png") + mock_vlm.assert_not_called() @patch("openadapt_evals.vlm_evaluator.vlm_judge") def test_custom_model_passed_through(self, mock_vlm): diff --git a/tests/test_openenv.py b/tests/test_openenv.py index 9e4b44e..e7e2a7c 100644 --- a/tests/test_openenv.py +++ b/tests/test_openenv.py @@ -19,6 +19,7 @@ def _make_mock_adapter(): adapter = MagicMock() _fake_obs = BenchmarkObservation( screenshot=b"\x89PNG\r\n\x1a\n" + b"\x00" * 100, + viewport=(1280, 720), raw_observation={}, ) adapter.observe.return_value = _fake_obs diff --git a/tests/test_pixel_action.py b/tests/test_pixel_action.py index 9943550..994f8ca 100644 --- a/tests/test_pixel_action.py +++ b/tests/test_pixel_action.py @@ -4,17 +4,38 @@ them via _send_command(), bypassing the element-based _translate_action path. """ -from unittest.mock import patch - -from openadapt_evals.adapters.base import BenchmarkAction, BenchmarkObservation -from openadapt_evals.adapters.waa.live import WAALiveAdapter, WAALiveConfig +import base64 +import subprocess +from unittest.mock import MagicMock, patch + +import pytest + +from openadapt_evals.adapters.base import ( + BenchmarkAction, + BenchmarkObservation, + BenchmarkTask, +) +from openadapt_evals.adapters.waa.live import ( + AdapterInfrastructureError, + SetupReadinessError, + WAALiveAdapter, + WAALiveConfig, +) +from openadapt_evals.errors import ( + ActionDeliveredObservationError, + ActionDeliveryState, + ActionDeliveryUncertainError, + ActionExecutionError, +) def _make_adapter(**config_kwargs) -> WAALiveAdapter: """Create a WAALiveAdapter without connecting to a server.""" adapter = WAALiveAdapter.__new__(WAALiveAdapter) adapter.config = WAALiveConfig(**config_kwargs) - adapter._current_task = None + adapter._current_task = BenchmarkTask( + task_id="test-task", instruction="test", domain="desktop" + ) adapter._step_count = 0 adapter._current_a11y = None adapter._current_rects = {} @@ -44,9 +65,7 @@ def test_right_click(self): def test_type_action_clicks_then_types(self): adapter = _make_adapter() - cmd = adapter._build_pixel_command( - action_type="type", x=100, y=200, text="hello" - ) + cmd = adapter._build_pixel_command(action_type="type", x=100, y=200, text="hello") assert "pyautogui.click(100, 200)" in cmd assert "time.sleep(0.2)" in cmd assert "pyautogui.write('hello'" in cmd @@ -76,30 +95,32 @@ def test_error_returns_none(self): cmd = adapter._build_pixel_command(action_type="error") assert cmd is None - def test_unknown_action_returns_none(self): + def test_unknown_action_is_rejected(self): adapter = _make_adapter() - cmd = adapter._build_pixel_command(action_type="unknown_action") - assert cmd is None + with pytest.raises(ActionExecutionError, match="Unsupported WAA pixel"): + adapter._build_pixel_command(action_type="unknown_action") - def test_clamps_to_safe_margin(self): - """Coordinates at screen edges should be clamped to avoid fail-safe.""" + def test_fail_safe_corner_is_rejected_without_target_shift(self): adapter = _make_adapter() - # Top-left corner - cmd = adapter._build_pixel_command(action_type="click", x=0, y=0) - assert cmd == "import pyautogui; pyautogui.click(5, 5)" + with pytest.raises(ActionExecutionError, match="fail-safe"): + adapter._build_pixel_command(action_type="click", x=0, y=0) - def test_clamps_bottom_right(self): - """Bottom-right corner should be clamped to screen_size - margin.""" + cmd = adapter._build_pixel_command(action_type="click", x=0, y=500) + assert cmd == "import pyautogui; pyautogui.click(0, 500)" + + @pytest.mark.parametrize( + ("x", "y"), + [(-1, 20), (20, -1), (1920, 20), (20, 1200), (2000, 1300)], + ) + def test_out_of_viewport_coordinates_are_rejected(self, x, y): adapter = _make_adapter() - cmd = adapter._build_pixel_command(action_type="click", x=2000, y=1300) - # screen is 1920x1200, margin=5 => max is (1915, 1195) - assert cmd == "import pyautogui; pyautogui.click(1915, 1195)" + with pytest.raises(ActionExecutionError, match="outside"): + adapter._build_pixel_command(action_type="click", x=x, y=y) - def test_none_coords_default_to_clamped_zero(self): + def test_none_coords_are_rejected(self): adapter = _make_adapter() - cmd = adapter._build_pixel_command(action_type="click", x=None, y=None) - # None -> 0 -> clamped to margin (5) - assert cmd == "import pyautogui; pyautogui.click(5, 5)" + with pytest.raises(ActionExecutionError, match="requires x and y"): + adapter._build_pixel_command(action_type="click", x=None, y=None) def test_float_coords_are_truncated(self): """Float pixel values (not fracs) should be cast to int.""" @@ -121,7 +142,11 @@ def test_generated_commands_are_valid_python(self): ] for action_type, x, y, text, key in cases: cmd = adapter._build_pixel_command( - action_type=action_type, x=x, y=y, text=text, key=key, + action_type=action_type, + x=x, + y=y, + text=text, + key=key, ) if cmd is not None: compile(cmd, f"<{action_type}>", "exec") @@ -132,9 +157,7 @@ class TestPixelActionBypassesTranslateAction: @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_pixel_action_does_not_call_translate_action( - self, mock_send, mock_obs - ): + def test_pixel_action_does_not_call_translate_action(self, mock_send, mock_obs): """pixel_action should never call _translate_action.""" mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() @@ -145,23 +168,17 @@ def test_pixel_action_does_not_call_translate_action( @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_pixel_action_calls_send_command_directly( - self, mock_send, mock_obs - ): + def test_pixel_action_calls_send_command_directly(self, mock_send, mock_obs): """pixel_action should call _send_command with the direct command.""" mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() adapter.pixel_action(x=500, y=300, action_type="click") - mock_send.assert_called_once_with( - "import pyautogui; pyautogui.click(500, 300)" - ) + mock_send.assert_called_once_with("import pyautogui; pyautogui.click(500, 300)") @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_pixel_action_returns_pixel_direct_flag( - self, mock_send, mock_obs - ): + def test_pixel_action_returns_pixel_direct_flag(self, mock_send, mock_obs): """info dict should contain pixel_direct=True.""" mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() @@ -171,9 +188,7 @@ def test_pixel_action_returns_pixel_direct_flag( @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_pixel_action_increments_step_count( - self, mock_send, mock_obs - ): + def test_pixel_action_increments_step_count(self, mock_send, mock_obs): """pixel_action should increment _step_count.""" mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() @@ -187,9 +202,7 @@ def test_pixel_action_increments_step_count( @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_pixel_action_records_action_history( - self, mock_send, mock_obs - ): + def test_pixel_action_records_action_history(self, mock_send, mock_obs): """pixel_action should append to _actions list.""" mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() @@ -212,6 +225,22 @@ def test_pixel_action_done_returns_true(self, mock_send, mock_obs): # _send_command should NOT be called for done actions mock_send.assert_not_called() + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_pixel_observation_failure_preserves_confirmed_delivery( + self, mock_send, mock_obs + ): + adapter = _make_adapter() + mock_obs.side_effect = RuntimeError("capture failed") + + with pytest.raises(ActionDeliveredObservationError) as raised: + adapter.pixel_action(x=500, y=300, action_type="click") + + mock_send.assert_called_once() + assert raised.value.delivery_state is ActionDeliveryState.DELIVERED + assert raised.value.retry_safe is False + assert adapter._step_count == 1 + class TestPixelActionFracConversion: """Tests that fractional coordinates are converted to absolute pixels.""" @@ -224,9 +253,66 @@ def test_frac_to_pixel_conversion(self, mock_send, mock_obs): adapter.pixel_action(x_frac=0.5, y_frac=0.5) # 0.5 * 1920 = 960, 0.5 * 1200 = 600 - mock_send.assert_called_once_with( - "import pyautogui; pyautogui.click(960, 600)" - ) + mock_send.assert_called_once_with("import pyautogui; pyautogui.click(960, 600)") + + def test_screen_size_refuses_configured_fallback_before_measurement(self): + adapter = _make_adapter(screen_width=1920, screen_height=1200) + adapter._actual_screen_size = None + + with pytest.raises(ActionExecutionError, match="exact measured viewport"): + _ = adapter.screen_size + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_pointer_action_before_reset_is_rejected(self, mock_send, mock_obs): + adapter = _make_adapter() + adapter._current_task = None + + with pytest.raises(ActionExecutionError, match="loaded task"): + adapter.pixel_action(x=500, y=300) + + mock_send.assert_not_called() + mock_obs.assert_not_called() + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_keyboard_action_before_reset_is_rejected(self, mock_send, mock_obs): + adapter = _make_adapter() + adapter._current_task = None + + with pytest.raises(ActionExecutionError, match="loaded task"): + adapter.pixel_action(action_type="key", key="enter") + + mock_send.assert_not_called() + mock_obs.assert_not_called() + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_pointer_action_before_measured_viewport_is_rejected( + self, mock_send, mock_obs + ): + adapter = _make_adapter() + adapter._actual_screen_size = None + + with pytest.raises(ActionExecutionError, match="exact measured viewport"): + adapter.pixel_action(x=500, y=300) + + mock_send.assert_not_called() + mock_obs.assert_not_called() + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_absolute_pixel_uses_measured_not_configured_viewport( + self, mock_send, mock_obs + ): + adapter = _make_adapter(screen_width=1920, screen_height=1200) + adapter._actual_screen_size = (800, 600) + + with pytest.raises(ActionExecutionError, match="800x600 viewport"): + adapter.pixel_action(x=900, y=300) + + mock_send.assert_not_called() + mock_obs.assert_not_called() @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") @@ -237,22 +323,40 @@ def test_frac_overrides_absolute(self, mock_send, mock_obs): adapter.pixel_action(x=100, y=100, x_frac=0.5, y_frac=0.5) # Fracs override: 0.5 * 1920 = 960, 0.5 * 1200 = 600 - mock_send.assert_called_once_with( - "import pyautogui; pyautogui.click(960, 600)" - ) + mock_send.assert_called_once_with("import pyautogui; pyautogui.click(960, 600)") @patch.object(WAALiveAdapter, "_get_observation") @patch.object(WAALiveAdapter, "_send_command") - def test_corner_frac_clamped(self, mock_send, mock_obs): - """Fraction at (0.0, 0.0) should be clamped to safe margin.""" + def test_normalized_corner_is_not_shifted_to_another_target( + self, mock_send, mock_obs + ): mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) adapter = _make_adapter() - adapter.pixel_action(x_frac=0.0, y_frac=0.0) - # 0.0 * 1920 = 0, 0.0 * 1200 = 0 -> clamped to (5, 5) - mock_send.assert_called_once_with( - "import pyautogui; pyautogui.click(5, 5)" - ) + with pytest.raises(ActionExecutionError, match="fail-safe"): + adapter.pixel_action(x_frac=1.0, y_frac=1.0) + mock_send.assert_not_called() + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_invalid_fraction_is_rejected(self, mock_send, mock_obs): + mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) + adapter = _make_adapter() + + with pytest.raises(ValueError, match="between 0 and 1"): + adapter.pixel_action(x_frac=1.1, y_frac=0.5) + + mock_send.assert_not_called() + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_corner_fraction_is_rejected(self, mock_send, mock_obs): + mock_obs.return_value = BenchmarkObservation(viewport=(1920, 1200)) + adapter = _make_adapter() + + with pytest.raises(ActionExecutionError, match="fail-safe"): + adapter.pixel_action(x_frac=0.0, y_frac=0.0) + mock_send.assert_not_called() class TestSendCommandRefactor: @@ -266,9 +370,7 @@ def test_step_delegates_to_send_command(self, mock_send, mock_obs): adapter = _make_adapter() adapter._current_rects = {"btn1": [400, 100, 500, 140]} - action = BenchmarkAction( - type="click", target_node_id="btn1" - ) + action = BenchmarkAction(type="click", target_node_id="btn1") adapter.step(action) mock_send.assert_called_once() # The command should be an element-grounded click via _translate_action @@ -305,3 +407,501 @@ def test_step_returns_command_in_info(self, mock_send, mock_obs): _, _, info = adapter.step(BenchmarkAction(type="wait")) assert info["command"] == "import time; time.sleep(1)" + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_delivery_failure_does_not_advance_or_observe(self, mock_send, mock_obs): + adapter = _make_adapter() + mock_send.side_effect = ActionExecutionError("not delivered") + + with pytest.raises(ActionExecutionError, match="not delivered"): + adapter.step(BenchmarkAction(type="click", x=500, y=300)) + + assert adapter._step_count == 0 + assert adapter._actions == [] + mock_obs.assert_not_called() + + @pytest.mark.parametrize( + "action", + [ + BenchmarkAction(type="click", x=None, y=300), + BenchmarkAction(type="click", x=500, y=None), + ], + ) + def test_click_requires_both_coordinates(self, action): + adapter = _make_adapter() + + with pytest.raises(ActionExecutionError, match="requires x and y"): + adapter.step(action) + + assert adapter._step_count == 0 + assert adapter._actions == [] + + def test_stale_element_without_fallback_is_rejected(self): + adapter = _make_adapter() + + with pytest.raises(ActionExecutionError, match="stale"): + adapter.step(BenchmarkAction(type="click", target_node_id="gone")) + + assert adapter._step_count == 0 + assert adapter._actions == [] + + def test_stale_element_does_not_trust_recorded_coordinates(self): + adapter = _make_adapter() + + with pytest.raises(ActionExecutionError, match="not authorized"): + adapter.step( + BenchmarkAction( + type="click", target_node_id="gone", x=500, y=300 + ) + ) + + assert adapter._step_count == 0 + assert adapter._actions == [] + + def test_unknown_action_is_rejected_before_rollout_advances(self): + adapter = _make_adapter() + + with pytest.raises(ActionExecutionError, match="Unsupported WAA action"): + adapter.step(BenchmarkAction(type="teleport")) + + assert adapter._step_count == 0 + assert adapter._actions == [] + + @patch.object(WAALiveAdapter, "_get_observation") + @patch.object(WAALiveAdapter, "_send_command") + def test_observation_failure_preserves_confirmed_delivery( + self, mock_send, mock_obs + ): + adapter = _make_adapter() + mock_obs.side_effect = RuntimeError("capture failed") + + with pytest.raises(ActionDeliveredObservationError) as raised: + adapter.step(BenchmarkAction(type="click", x=500, y=300)) + + mock_send.assert_called_once() + assert raised.value.delivery_state is ActionDeliveryState.DELIVERED + assert raised.value.retry_safe is False + assert adapter._step_count == 1 + assert len(adapter._actions) == 1 + + +def _response(status: int, *, text: str = "", payload: object | None = None): + response = MagicMock() + response.status_code = status + response.text = text + response.json.return_value = {} if payload is None else payload + return response + + +class TestSendCommandDeliveryReceipts: + def test_http_500_is_not_success(self): + adapter = _make_adapter() + + with ( + patch("requests.post", return_value=_response(500, text="failed")), + pytest.raises(ActionDeliveryUncertainError, match="HTTP 500") as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.UNCERTAIN + assert raised.value.retry_safe is False + + def test_http_error_with_explicit_non_delivery_is_retry_safe(self): + adapter = _make_adapter() + response = _response( + 409, + text="rejected before dispatch", + payload={"delivery_state": "not_delivered"}, + ) + + with ( + patch("requests.post", return_value=response), + pytest.raises(ActionExecutionError) as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.NOT_DELIVERED + assert raised.value.retry_safe is True + + def test_stderr_is_not_success(self): + adapter = _make_adapter() + response = _response(200, payload={"stderr": "permission denied"}) + + with ( + patch("requests.post", return_value=response), + pytest.raises(ActionExecutionError, match="stderr: permission denied"), + ): + adapter._send_command("do_work()") + + def test_connection_error_is_uncertain_not_success(self): + adapter = _make_adapter() + + with ( + patch("requests.post", side_effect=ConnectionError("offline")), + pytest.raises( + ActionDeliveryUncertainError, match="outcome is uncertain" + ) as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.UNCERTAIN + assert raised.value.retry_safe is False + + def test_failed_failsafe_recovery_is_not_success(self): + adapter = _make_adapter() + response = _response( + 500, + text="pyautogui.FailSafeException", + payload={"message": "pyautogui.FailSafeException"}, + ) + + with ( + patch("requests.post", return_value=response), + patch.object(adapter, "_recover_failsafe", return_value=False), + pytest.raises(ActionExecutionError, match="recovery failed"), + ): + adapter._send_command("do_work()") + + def test_failed_recovery_preserves_confirmed_non_delivery(self): + adapter = _make_adapter() + response = _response( + 500, + text="pyautogui.FailSafeException", + payload={ + "message": "pyautogui.FailSafeException", + "delivery_state": "not_delivered", + }, + ) + + with ( + patch("requests.post", return_value=response), + patch.object(adapter, "_recover_failsafe", return_value=False), + pytest.raises(ActionExecutionError) as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.NOT_DELIVERED + assert raised.value.retry_safe is True + + def test_failed_retry_is_not_success(self): + adapter = _make_adapter() + initial = _response( + 500, + text="pyautogui.FailSafeException", + payload={ + "message": "pyautogui.FailSafeException", + "delivery_state": "not_delivered", + }, + ) + retry = _response(500, text="still failed") + + with ( + patch("requests.post", side_effect=[initial, retry]), + patch.object(adapter, "_recover_failsafe", return_value=True), + pytest.raises(ActionExecutionError, match="retry was not confirmed"), + ): + adapter._send_command("do_work()") + + @pytest.mark.parametrize( + "retry", + [ + _response( + 409, + text="not dispatched", + payload={"delivery_state": "not_delivered"}, + ), + _response( + 200, + payload={ + "stderr": "not dispatched", + "delivery_state": "not_delivered", + }, + ), + ], + ) + def test_retry_preserves_confirmed_non_delivery(self, retry): + adapter = _make_adapter() + initial = _response( + 500, + text="pyautogui.FailSafeException", + payload={ + "message": "pyautogui.FailSafeException", + "delivery_state": "not_delivered", + }, + ) + + with ( + patch("requests.post", side_effect=[initial, retry]), + patch.object(adapter, "_recover_failsafe", return_value=True), + pytest.raises(ActionExecutionError) as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.NOT_DELIVERED + assert raised.value.retry_safe is True + + def test_partial_failsafe_delivery_is_not_blindly_retried(self): + adapter = _make_adapter() + response = _response( + 500, + text="pyautogui.FailSafeException", + payload={"message": "pyautogui.FailSafeException"}, + ) + command = "pyautogui.click(100, 100); pyautogui.moveTo(0, 0)" + + with ( + patch("requests.post", return_value=response) as post, + patch.object(adapter, "_recover_failsafe", return_value=True), + pytest.raises( + ActionDeliveryUncertainError, match="refusing a blind retry" + ) as raised, + ): + adapter._send_command(command) + + assert post.call_count == 1 + assert raised.value.delivery_state is ActionDeliveryState.UNCERTAIN + assert raised.value.retry_safe is False + + def test_explicit_non_delivery_receipt_allows_one_retry(self): + adapter = _make_adapter() + initial = _response( + 500, + text="pyautogui.FailSafeException", + payload={ + "message": "pyautogui.FailSafeException", + "delivery_state": "not_delivered", + }, + ) + retry = _response(200, payload={"stdout": "ok"}) + + with ( + patch("requests.post", side_effect=[initial, retry]) as post, + patch.object(adapter, "_recover_failsafe", return_value=True), + ): + adapter._send_command("do_work()") + + assert post.call_count == 2 + + @pytest.mark.parametrize( + "payload", + [ + {"message": "pyautogui.FailSafeException"}, + {"stdout": "pyautogui.FailSafeException"}, + ], + ) + def test_retry_failsafe_in_any_diagnostic_is_not_success(self, payload): + adapter = _make_adapter() + initial = _response( + 500, + text="pyautogui.FailSafeException", + payload={ + "message": "pyautogui.FailSafeException", + "delivery_state": "not_delivered", + }, + ) + retry = _response(200, payload=payload) + + with ( + patch("requests.post", side_effect=[initial, retry]) as post, + patch.object(adapter, "_recover_failsafe", return_value=True), + pytest.raises( + ActionDeliveryUncertainError, + match="retry triggered a fail-safe", + ) as raised, + ): + adapter._send_command("do_work()") + + assert post.call_count == 2 + assert raised.value.delivery_state is ActionDeliveryState.UNCERTAIN + assert raised.value.retry_safe is False + + @pytest.mark.parametrize( + "payload", + [ + {"success": False}, + {"success": "true"}, + {"error": "command rejected"}, + {"delivery_state": "uncertain"}, + {"delivery_state": "delivered-ish"}, + ], + ) + def test_explicit_failed_http_200_receipt_is_not_success(self, payload): + adapter = _make_adapter() + + with ( + patch("requests.post", return_value=_response(200, payload=payload)), + pytest.raises( + ActionDeliveryUncertainError, + match="receipt did not confirm execution", + ), + ): + adapter._send_command("do_work()") + + def test_explicit_non_delivery_http_200_receipt_is_retry_safe_failure(self): + adapter = _make_adapter() + response = _response(200, payload={"delivery_state": "not_delivered"}) + + with ( + patch("requests.post", return_value=response), + pytest.raises(ActionExecutionError) as raised, + ): + adapter._send_command("do_work()") + + assert raised.value.delivery_state is ActionDeliveryState.NOT_DELIVERED + assert raised.value.retry_safe is True + + def test_recovery_rejects_stderr_and_restores_failsafe_in_script(self): + adapter = _make_adapter() + response = _response(200, payload={"stderr": "move failed"}) + + with patch("requests.post", return_value=response) as post: + assert adapter._recover_failsafe() is False + + command = post.call_args.kwargs["json"]["command"] + encoded = command.split("base64.b64decode('", 1)[1].split("')", 1)[0] + script = base64.b64decode(encoded).decode() + assert "finally:" in script + assert "pyautogui.FAILSAFE = previous" in script + + @pytest.mark.parametrize( + "payload", + [ + {"success": False}, + {"error": "recovery rejected"}, + {"delivery_state": "uncertain"}, + {"delivery_state": "delivered-ish"}, + ], + ) + def test_recovery_rejects_explicit_failed_receipt(self, payload): + adapter = _make_adapter() + + with patch( + "requests.post", + return_value=_response(200, payload=payload), + ): + assert adapter._recover_failsafe() is False + + +class TestObservationFailure: + def test_reset_failure_invalidates_previous_task_evidence(self): + adapter = _make_adapter() + adapter._current_a11y = {"name": "old task"} + adapter._current_rects = {"old": [10, 10, 20, 20]} + adapter._current_screenshot = b"old" + + with ( + patch.object(adapter, "check_connection", return_value=False), + pytest.raises(RuntimeError, match="Cannot connect"), + ): + adapter.reset( + BenchmarkTask( + task_id="new-task", + instruction="new", + domain="desktop", + ) + ) + + assert adapter._current_task is None + assert adapter._current_a11y is None + assert adapter._current_rects == {} + assert adapter._current_screenshot is None + assert adapter._actual_screen_size is None + + @pytest.mark.parametrize( + "action", + [ + BenchmarkAction(type="key", key="enter"), + BenchmarkAction(type="type", text="sensitive value"), + ], + ) + def test_failed_setup_leaves_non_pointer_actions_disabled(self, action): + adapter = _make_adapter(lightweight=True) + task = BenchmarkTask( + task_id="new-task", + instruction="new", + domain="desktop", + raw_config={"config": [{"type": "execute", "parameters": {}}]}, + ) + + with ( + patch.object(adapter, "check_connection", return_value=True), + patch.object( + adapter, + "_run_task_setup", + side_effect=SetupReadinessError("setup failed"), + ), + pytest.raises(SetupReadinessError, match="setup failed"), + ): + adapter.reset(task) + + with pytest.raises(ActionExecutionError, match="loaded task"): + adapter.step(action) + + @pytest.mark.parametrize( + "payload", + [ + {"stderr": "setup failed"}, + {"success": False}, + {"error": "setup rejected"}, + {"delivery_state": "uncertain"}, + {"delivery_state": "delivered-ish"}, + {"returncode": 7}, + ], + ) + def test_setup_http_200_failed_receipt_is_not_ok(self, payload): + adapter = _make_adapter(lightweight=True) + raw_config = { + "config": [{"type": "execute", "parameters": {}}], + } + + with ( + patch.object(adapter, "_config_entry_to_command", return_value="setup()"), + patch("requests.post", return_value=_response(200, payload=payload)), + pytest.raises(SetupReadinessError, match="failing steps"), + ): + adapter._run_task_setup(raw_config) + + assert adapter._last_setup_results[0]["status"] == "error" + + def test_execute_setup_command_raises_on_nonzero_shell_exit(self): + command = WAALiveAdapter._config_entry_to_command( + { + "type": "execute", + "parameters": {"command": "exit 7"}, + } + ) + + assert command is not None + with pytest.raises(subprocess.CalledProcessError) as raised: + exec(command, {}) + + assert raised.value.returncode == 7 + + def test_partial_observation_does_not_reuse_old_accessibility_geometry(self): + adapter = _make_adapter() + adapter._current_a11y = {"name": "old task"} + adapter._current_rects = {"old": [10, 10, 20, 20]} + screenshot = _response(200) + screenshot.content = b"fresh-but-not-an-image" + missing_a11y = _response(503, text="unavailable") + + with patch("requests.get", side_effect=[screenshot, missing_a11y]): + observation = adapter._get_observation() + + assert observation.screenshot == b"fresh-but-not-an-image" + assert observation.accessibility_tree is None + assert adapter._current_a11y is None + assert adapter._current_rects == {} + assert adapter._current_screenshot == b"fresh-but-not-an-image" + assert adapter._actual_screen_size is None + + def test_missing_screenshot_and_accessibility_is_not_normal_observation(self): + adapter = _make_adapter() + missing = _response(503, text="unavailable") + + with ( + patch("requests.get", side_effect=[missing, missing]), + pytest.raises(AdapterInfrastructureError, match="no screenshot"), + ): + adapter._get_observation() diff --git a/tests/test_planner_grounder_agent.py b/tests/test_planner_grounder_agent.py index 8fb2f6b..e1c43b6 100644 --- a/tests/test_planner_grounder_agent.py +++ b/tests/test_planner_grounder_agent.py @@ -381,6 +381,42 @@ def test_http_grounder_calls_endpoint(self, mock_post, observation, task): call_url = mock_post.call_args[0][0] assert "/v1/chat/completions" in call_url + @pytest.mark.parametrize( + "raw", + [ + '{"type":"done","x":0.5,"y":0.5}', + '{"type":"click","x":0.1,"x":0.9,"y":0.5}', + '{"type":"click","x":0.5,"y":0.5,"key":"enter"}', + '{"type":"click","x":0.5,"y":0.5}{"type":"done"}', + '{"type":"click","x":NaN,"y":0.5}', + '{"type":"click","x":500,"y":0.5}', + "[900, 100, 100, 200]", + "[500, 0.5]", + "[-1, 100]", + "[1001, 500]", + ], + ) + def test_grounder_rejects_ambiguous_or_invalid_coordinates(self, raw): + action = PlannerGrounderAgent._parse_bbox_to_action(raw) + + assert action.type == "error" + assert action.raw_action["parse_error"] + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ('{"type":"click","x":0.25,"y":1.0}', (0.25, 1.0)), + ("[250, 200, 750, 800]", (0.5, 0.5)), + ("[0.25, 1.0]", (0.25, 1.0)), + ("[250, 750]", (0.25, 0.75)), + ], + ) + def test_grounder_accepts_one_coherent_coordinate_space(self, raw, expected): + action = PlannerGrounderAgent._parse_bbox_to_action(raw) + + assert action.type == "click" + assert (action.x, action.y) == expected + # -- Tests: _action_to_planner_output helper ---------------------------------- @@ -704,6 +740,31 @@ def test_structured_click_calls_grounder( assert action.x == 0.6 assert action.y == 0.4 + @pytest.mark.parametrize("action_type", ["click", "double_click"]) + @patch("openadapt_evals.vlm.vlm_call") + @patch("openadapt_evals.vlm.extract_json") + def test_structured_pointer_action_requires_a_target( + self, mock_extract, mock_vlm, action_type, observation, task + ): + mock_vlm.return_value = "{}" + mock_extract.return_value = { + "decision": "COMMAND", + "action_type": action_type, + "target_description": "", + } + grounder = MagicMock() + agent = PlannerGrounderAgent( + planner="claude-sonnet-4-20250514", + grounder=grounder, + planner_provider="anthropic", + ) + + action = agent.act(observation, task) + + assert action.type == "error" + assert action.raw_action["parse_error"] == "missing_target_description" + grounder.act.assert_not_called() + @patch("openadapt_evals.vlm.vlm_call") @patch("openadapt_evals.vlm.extract_json") def test_backward_compat_instruction_field( @@ -729,6 +790,78 @@ def test_backward_compat_instruction_field( assert action.type == "click" assert action.x == 0.5 + @pytest.mark.parametrize("decision", ["MAYBE", "", 1]) + @patch("openadapt_evals.vlm.vlm_call") + @patch("openadapt_evals.vlm.extract_json") + def test_unknown_decision_cannot_act( + self, mock_extract, mock_vlm, decision, observation, task + ): + mock_vlm.return_value = "{}" + mock_extract.return_value = { + "decision": decision, + "action_type": "click", + "target_description": "delete account", + } + grounder = MagicMock() + agent = PlannerGrounderAgent( + planner="claude-sonnet-4-20250514", + grounder=grounder, + planner_provider="anthropic", + ) + + action = agent.act(observation, task) + + assert action.type == "error" + assert action.raw_action["parse_error"] == "invalid_decision" + grounder.act.assert_not_called() + + @patch("openadapt_evals.vlm.vlm_call") + @patch("openadapt_evals.vlm.extract_json") + def test_unknown_structured_action_cannot_fall_back_to_click( + self, mock_extract, mock_vlm, observation, task + ): + mock_vlm.return_value = "{}" + mock_extract.return_value = { + "decision": "COMMAND", + "action_type": "delete", + "target_description": "delete account", + } + grounder = MagicMock() + agent = PlannerGrounderAgent( + planner="claude-sonnet-4-20250514", + grounder=grounder, + planner_provider="anthropic", + ) + + action = agent.act(observation, task) + + assert action.type == "error" + assert action.raw_action["parse_error"] == "invalid_action_type" + grounder.act.assert_not_called() + + @pytest.mark.parametrize("direction", ["", "left", None]) + @patch("openadapt_evals.vlm.vlm_call") + @patch("openadapt_evals.vlm.extract_json") + def test_structured_scroll_requires_explicit_supported_direction( + self, mock_extract, mock_vlm, direction, observation, task + ): + mock_vlm.return_value = "{}" + mock_extract.return_value = { + "decision": "COMMAND", + "action_type": "scroll", + "action_value": direction, + } + agent = PlannerGrounderAgent( + planner="claude-sonnet-4-20250514", + grounder=MagicMock(), + planner_provider="anthropic", + ) + + action = agent.act(observation, task) + + assert action.type == "error" + assert action.raw_action["parse_error"] == "invalid_scroll_direction" + # -- Tests: Type-then-Enter queuing ------------------------------------------ diff --git a/tests/test_result_artifact_reporting.py b/tests/test_result_artifact_reporting.py new file mode 100644 index 0000000..78af8b8 --- /dev/null +++ b/tests/test_result_artifact_reporting.py @@ -0,0 +1,211 @@ +"""Result artifact rows cannot manufacture measured benchmark success.""" + +from __future__ import annotations + +import json + +import pytest + +from openadapt_evals.adapters.base import ( + benchmark_result_is_scored, + normalize_benchmark_result_artifact, +) +from openadapt_evals.analysis.trace_analyzer import TraceAnalyzer +from openadapt_evals.benchmarks.pool_viewer import ( + get_domain_stats as get_pool_domain_stats, +) +from openadapt_evals.benchmarks.pool_viewer import parse_pool_logs +from openadapt_evals.benchmarks.viewer import ( + _get_domain_stats, + load_task_results, +) + + +@pytest.mark.parametrize( + "artifact", + [ + {"success": "false", "score": 0.0}, + {"success": False, "score": float("nan")}, + {"success": False, "score": 0.0, "error_type": "unknown"}, + ], +) +def test_strict_artifact_parser_marks_malformed_rows_unscored(artifact) -> None: + result = normalize_benchmark_result_artifact( + {"task_id": "bad", **artifact}, + expected_task_id="bad", + ) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == "evaluation" + assert benchmark_result_is_scored(result) is False + + +def test_trace_summary_excludes_unscored_rows_but_retains_attempts(tmp_path) -> None: + rows = [ + {"task_id": "pass", "success": True, "score": 1.0}, + { + "task_id": "agent-fail", + "success": False, + "score": 0.0, + "error_type": "agent", + }, + { + "task_id": "infra", + "success": False, + "score": 0.0, + "error_type": "infrastructure", + }, + {"task_id": "string-false", "success": "false", "score": 0.0}, + ] + artifact = tmp_path / "results.jsonl" + artifact.write_text("\n".join(json.dumps(row) for row in rows)) + + summary = TraceAnalyzer(artifact).summary() + + assert summary["total_episodes"] == 4 + assert summary["outcome_episodes"] == 2 + assert summary["unscored_episodes"] == 2 + assert summary["success_rate"] == 0.5 + assert summary["episodes_by_status"] == { + "passed": 1, + "failed": 1, + "infra_error": 1, + "evaluation_error": 1, + } + + +def _write_viewer_task(root, task_id: str, execution: object) -> None: + task_dir = root / "tasks" / task_id + task_dir.mkdir(parents=True) + (task_dir / "task.json").write_text( + json.dumps({"domain": "desktop", "instruction": task_id}) + ) + (task_dir / "execution.json").write_text(json.dumps(execution)) + + +def test_benchmark_viewer_normalizes_rows_before_domain_metrics(tmp_path) -> None: + _write_viewer_task(tmp_path, "pass", {"success": True, "score": 1.0}) + _write_viewer_task( + tmp_path, + "agent-fail", + {"success": False, "score": 0.0, "error_type": "agent"}, + ) + _write_viewer_task( + tmp_path, + "infra", + {"success": False, "score": 0.0, "error_type": "infrastructure"}, + ) + _write_viewer_task( + tmp_path, + "string-false", + {"success": "false", "score": 0.0}, + ) + + tasks = load_task_results(tmp_path) + stats = _get_domain_stats(tasks)["desktop"] + + assert all(type(task["execution"]["success"]) is bool for task in tasks) + assert stats == { + "total": 4, + "outcomes": 2, + "success": 1, + "fail": 1, + "unscored": 2, + } + + +def test_pool_domain_metrics_do_not_use_raw_truthiness() -> None: + tasks = [ + {"task_id": "pass", "domain": "desktop", "success": True, "score": 1.0}, + { + "task_id": "agent-fail", + "domain": "desktop", + "success": False, + "score": 0.0, + "error_type": "agent", + }, + { + "task_id": "infra", + "domain": "desktop", + "success": False, + "score": 0.0, + "error_type": "infrastructure", + }, + { + "task_id": "string-false", + "domain": "desktop", + "success": "false", + "score": 0.0, + }, + ] + + assert get_pool_domain_stats(tasks)["desktop"] == { + "total": 4, + "outcomes": 2, + "success": 1, + "fail": 1, + "unscored": 2, + } + + +def test_pool_log_without_result_is_retained_as_unscored_error(tmp_path) -> None: + log = tmp_path / "waa-pool-1.log" + log.write_text( + "[2026-07-28 12:00:00] [Domain]: desktop\n" + "[2026-07-28 12:00:01] [Example ID]: missing-result\n" + "[2026-07-28 12:00:02] Finished desktop/missing-result\n" + ) + + parsed = parse_pool_logs(tmp_path) + + assert len(parsed["tasks"]) == 1 + assert parsed["tasks"][0]["error_type"] == "evaluation" + assert parsed["tasks"][0]["success"] is False + assert parsed["workers"]["1"] == { + "tasks": 1, + "outcomes": 0, + "successes": 0, + "failures": 0, + "unscored": 1, + } + + +def test_partial_trajectory_score_is_measured_but_not_success(tmp_path) -> None: + trace_dir = tmp_path / "trajectory" + trace_dir.mkdir() + (trace_dir / "trajectories.jsonl").write_text( + json.dumps( + { + "episode_id": "partial", + "step_index": 0, + "episode_reward": 0.5, + "planner_output": {"action_type": "click"}, + } + ) + ) + + analyzer = TraceAnalyzer(trace_dir) + + assert analyzer.episodes[0].score == 0.5 + assert analyzer.episodes[0].success is False + assert analyzer.episodes[0].error_type is None + assert analyzer.summary()["success_rate"] == 0.0 + + +def test_partial_pool_score_is_measured_failure(tmp_path) -> None: + log = tmp_path / "waa-pool-2.log" + log.write_text( + "[2026-07-28 12:00:00] [Domain]: desktop\n" + "[2026-07-28 12:00:01] [Example ID]: partial\n" + "[2026-07-28 12:00:02] Result: 0.5\n" + "[2026-07-28 12:00:03] Finished desktop/partial\n" + ) + + parsed = parse_pool_logs(tmp_path) + + assert parsed["tasks"][0]["result"] == 0.5 + assert parsed["tasks"][0]["success"] is False + assert parsed["tasks"][0]["error_type"] is None + assert parsed["workers"]["2"]["outcomes"] == 1 + assert parsed["workers"]["2"]["failures"] == 1 diff --git a/tests/test_rl_env.py b/tests/test_rl_env.py index 4fa7834..85c13bf 100644 --- a/tests/test_rl_env.py +++ b/tests/test_rl_env.py @@ -5,13 +5,17 @@ from __future__ import annotations +from unittest.mock import MagicMock, patch + import pytest from openadapt_evals.adapters import ( BenchmarkAction, BenchmarkObservation, + BenchmarkTask, WAAMockAdapter, ) +from openadapt_evals.adapters.base import BenchmarkResult, EvaluationUnavailableError from openadapt_evals.adapters.rl_env import ( ResetConfig, RLEnvironment, @@ -48,6 +52,33 @@ def test_reset_with_config(self, mock_env: RLEnvironment) -> None: obs = mock_env.reset(ResetConfig(task_id=other_id)) assert isinstance(obs, BenchmarkObservation) + def test_failed_reset_invalidates_prior_episode_state(self) -> None: + adapter = MagicMock() + first_task = BenchmarkTask( + task_id="first", instruction="First", domain="desktop" + ) + second_task = BenchmarkTask( + task_id="second", instruction="Second", domain="desktop" + ) + observation = BenchmarkObservation(viewport=(640, 480)) + adapter.load_task.side_effect = [first_task, second_task] + adapter.reset.side_effect = [observation, RuntimeError("reset failed")] + adapter.step.return_value = (observation, False, {}) + env = RLEnvironment(adapter=adapter, default_task_id="first") + env.reset() + env.step(BenchmarkAction(type="wait")) + + with pytest.raises(RuntimeError, match="reset failed"): + env.reset(ResetConfig(task_id="second")) + + with pytest.raises(RuntimeError, match=r"reset\(\) before step"): + env.step(BenchmarkAction(type="wait")) + with pytest.raises(RuntimeError, match=r"reset\(\) before evaluate"): + env.evaluate_result() + with pytest.raises(RuntimeError, match="No measured viewport"): + _ = env.screen_size + assert env.trajectory == [] + # --------------------------------------------------------------------------- # step @@ -89,6 +120,104 @@ def test_evaluate_returns_score(self, mock_env: RLEnvironment) -> None: assert isinstance(score, float) assert 0.0 <= score <= 1.0 + def test_terminal_error_cannot_be_reclassified_as_success(self) -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["notepad"]) + task_id = adapter.list_tasks()[0].task_id + env = RLEnvironment(adapter=adapter, default_task_id=task_id) + env.reset() + env.step(BenchmarkAction(type="error")) + + with ( + patch.object( + adapter, + "evaluate", + return_value=BenchmarkResult(task_id=task_id, success=True, score=1.0), + ) as evaluate, + pytest.raises(EvaluationUnavailableError, match="terminal error"), + ): + env.evaluate() + + evaluate.assert_not_called() + assert env.trajectory[-1].reward == 0.0 + + def test_dense_evaluation_rejects_terminal_error_without_evaluator(self) -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["notepad"]) + task_id = adapter.list_tasks()[0].task_id + env = RLEnvironment(adapter=adapter, default_task_id=task_id) + env.reset() + env.step(BenchmarkAction(type="error")) + + with ( + patch.object( + adapter, + "evaluate", + return_value=BenchmarkResult(task_id=task_id, success=True, score=1.0), + ) as evaluate, + pytest.raises(EvaluationUnavailableError, match="terminal error"), + ): + env.evaluate_dense() + + evaluate.assert_not_called() + assert env.trajectory[-1].reward == 0.0 + + @pytest.mark.parametrize( + "result", + [ + BenchmarkResult(task_id="other", success=True, score=1.0), + BenchmarkResult(task_id="expected", success="false", score=1.0), + BenchmarkResult(task_id="expected", success=False, score=1.0), + ], + ) + def test_malformed_adapter_result_is_unscored(self, result) -> None: + adapter = MagicMock() + task = BenchmarkTask( + task_id="expected", instruction="Expected", domain="desktop" + ) + observation = BenchmarkObservation(viewport=(640, 480)) + adapter.load_task.return_value = task + adapter.reset.return_value = observation + adapter.step.return_value = (observation, True, {}) + adapter.evaluate.return_value = result + env = RLEnvironment(adapter=adapter, default_task_id=task.task_id) + env.reset() + env.step(BenchmarkAction(type="done")) + + normalized = env.evaluate_result() + + assert normalized.success is False + assert normalized.score == 0.0 + assert normalized.error_type == "evaluation" + assert env.trajectory[-1].reward == 0.0 + with pytest.raises(EvaluationUnavailableError, match="was not scored"): + env.evaluate() + + def test_per_step_malformed_result_is_unscored(self) -> None: + adapter = MagicMock() + task = BenchmarkTask( + task_id="expected", instruction="Expected", domain="desktop" + ) + observation = BenchmarkObservation(viewport=(640, 480)) + adapter.load_task.return_value = task + adapter.reset.return_value = observation + adapter.step.return_value = (observation, False, {}) + adapter.evaluate.return_value = BenchmarkResult( + task_id="other", + success=True, + score=1.0, + ) + env = RLEnvironment( + adapter=adapter, + default_task_id=task.task_id, + evaluate_every_step=True, + ) + env.reset() + + step = env.step(BenchmarkAction(type="wait")) + + assert "evaluation_score" not in step.info + assert step.info["evaluation_error_type"] == "evaluation" + assert step.reward == 0.0 + # --------------------------------------------------------------------------- # pixel_action @@ -117,6 +246,56 @@ def test_pixel_action_normalized_fracs(self, mock_env: RLEnvironment) -> None: assert result.action.x == int(0.5 * 1920) assert result.action.y == int(0.25 * 1200) + def test_delegated_fraction_uses_observed_viewport_before_dispatch(self) -> None: + adapter = MagicMock() + task = adapter.load_task.return_value + task.task_id = "measured-viewport" + task.instruction = "Click the measured target" + initial = BenchmarkObservation(viewport=(640, 480)) + adapter.reset.return_value = initial + adapter.pixel_action.return_value = (initial, False, {}) + env = RLEnvironment(adapter=adapter, default_task_id="measured-viewport") + env.reset() + + result = env.pixel_action(x_frac=0.5, y_frac=0.25) + + assert (result.action.x, result.action.y) == (320, 120) + assert adapter.pixel_action.call_args.kwargs["x"] == 320 + assert adapter.pixel_action.call_args.kwargs["y"] == 120 + assert adapter.pixel_action.call_args.kwargs["x_frac"] is None + assert adapter.pixel_action.call_args.kwargs["y_frac"] is None + + def test_delegated_error_action_is_terminal_and_unscored(self) -> None: + adapter = MagicMock() + task = adapter.load_task.return_value + task.task_id = "terminal-error" + task.instruction = "Return a parser error" + observation = BenchmarkObservation(viewport=(640, 480)) + adapter.reset.return_value = observation + adapter.pixel_action.return_value = (observation, False, {}) + env = RLEnvironment(adapter=adapter, default_task_id="terminal-error") + env.reset() + + result = env.pixel_action(action_type="error") + + assert result.done is True + assert env.done is True + assert result.reward == 0.0 + assert result.info["evaluation_error_type"] == "agent" + + def test_pixel_action_normalized_edge_stays_inside_viewport( + self, mock_env: RLEnvironment + ) -> None: + mock_env.reset() + result = mock_env.pixel_action(x_frac=1.0, y_frac=1.0) + + assert (result.action.x, result.action.y) == (1919, 1199) + + def test_pixel_action_rejects_invalid_fraction(self, mock_env: RLEnvironment) -> None: + mock_env.reset() + with pytest.raises(ValueError, match="between 0 and 1"): + mock_env.pixel_action(x_frac=1.1, y_frac=0.5) + def test_pixel_action_type_text(self, mock_env: RLEnvironment) -> None: """pixel_action with action_type='type' and text sends a type action.""" mock_env.reset() @@ -157,6 +336,23 @@ def test_observe_without_step(self, mock_env: RLEnvironment) -> None: class TestCollectRollout: + @pytest.mark.parametrize("max_steps", [0, -1, True, 1.5]) + def test_collect_rollout_requires_positive_integer_steps( + self, + mock_env: RLEnvironment, + max_steps, + ) -> None: + with ( + patch.object(mock_env, "reset") as reset, + pytest.raises(ValueError, match="positive integer"), + ): + mock_env.collect_rollout( + agent_fn=lambda _obs: BenchmarkAction(type="done"), + max_steps=max_steps, + ) + + reset.assert_not_called() + def test_collect_rollout(self, mock_env: RLEnvironment) -> None: """collect_rollout with a mock agent_fn produces the expected number of steps.""" mock_env.reset() @@ -224,6 +420,27 @@ def always_click(obs: BenchmarkObservation) -> BenchmarkAction: # Should terminate early because screenshots are identical assert len(rollout) < 15 + def test_terminal_error_rollout_stays_failed_without_evaluator(self) -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["notepad"]) + task_id = adapter.list_tasks()[0].task_id + env = RLEnvironment(adapter=adapter, default_task_id=task_id) + + with patch.object( + adapter, + "evaluate", + return_value=BenchmarkResult(task_id=task_id, success=True, score=1.0), + ) as evaluate: + rollout = env.collect_rollout( + agent_fn=lambda _obs: BenchmarkAction(type="error"), + max_steps=3, + ) + + evaluate.assert_not_called() + assert len(rollout) == 1 + assert rollout[-1].action.type == "error" + assert rollout[-1].reward == 0.0 + assert rollout[-1].info["evaluation_error_type"] == "agent" + # --------------------------------------------------------------------------- # screen_size @@ -242,3 +459,19 @@ def test_screen_size_property(self) -> None: width, height = env.screen_size assert width == 1920 assert height == 1200 + + def test_screen_size_refuses_without_measured_viewport(self) -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["notepad"]) + env = RLEnvironment(adapter=adapter) + + with pytest.raises(RuntimeError, match="No measured viewport"): + _ = env.screen_size + + @pytest.mark.parametrize("viewport", [None, (0, 720), (1920, -1)]) + def test_screen_size_refuses_invalid_measured_viewport(self, viewport) -> None: + adapter = WAAMockAdapter(num_tasks=1, domains=["notepad"]) + env = RLEnvironment(adapter=adapter) + env._last_obs = BenchmarkObservation(viewport=viewport) + + with pytest.raises((RuntimeError, ValueError), match="viewport"): + _ = env.screen_size diff --git a/tests/test_runner.py b/tests/test_runner.py index c5bb954..9386af6 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,5 +1,6 @@ """Tests for the evaluation runner.""" +import json from unittest.mock import Mock import pytest @@ -19,13 +20,18 @@ format_accessibility_tree, parse_action_response, ) +from openadapt_evals.benchmarks.data_collection import ExecutionTraceCollector from openadapt_evals.benchmarks.runner import ( EvaluationConfig, + _evaluate_parallel, + _evaluate_sequential, _run_single_task, compute_domain_metrics, compute_metrics, evaluate_agent_on_benchmark, ) +from openadapt_evals.errors import ActionParseError, RolloutInfrastructureError +from openadapt_evals.integrations.wandb_logger import WandbLogger class TestEvaluationConfig: @@ -110,6 +116,105 @@ def test_all_failures(self): assert metrics["success_rate"] == 0.0 assert metrics["avg_score"] == 0.1 + def test_unavailable_attempts_do_not_enter_outcome_rates(self): + results = [ + BenchmarkResult(task_id="pass", success=True, score=1.0), + BenchmarkResult(task_id="fail", success=False, score=0.0), + BenchmarkResult( + task_id="infra", + success=True, + score=1.0, + error_type="infrastructure", + ), + BenchmarkResult( + task_id="evaluation", + success=False, + score=0.0, + error_type="evaluation", + ), + BenchmarkResult( + task_id="agent", + success=False, + score=0.0, + error_type="agent", + ), + ] + + metrics = compute_metrics(results) + + assert metrics["num_tasks"] == 5 + assert metrics["num_outcome_tasks"] == 3 + assert metrics["error_count"] == 3 + assert metrics["success_rate"] == pytest.approx(1 / 3) + assert metrics["avg_score"] == pytest.approx(1 / 3) + assert metrics["success_count"] == 1 + assert metrics["fail_count"] == 2 + assert metrics["num_infrastructure_failures"] == 1 + assert metrics["num_evaluation_failures"] == 1 + assert metrics["num_agent_failures"] == 1 + + def test_agent_failures_reduce_the_headline_success_rate(self): + results = [BenchmarkResult(task_id="pass", success=True, score=1.0)] + results.extend( + BenchmarkResult( + task_id=f"agent-{index}", + success=False, + score=0.0, + error_type="agent", + ) + for index in range(99) + ) + + metrics = compute_metrics(results) + + assert metrics["num_outcome_tasks"] == 100 + assert metrics["success_rate"] == 0.01 + + def test_trace_summary_uses_the_same_outcome_denominator(self, tmp_path): + collector = ExecutionTraceCollector( + benchmark_name="test", + run_name="run", + output_dir=tmp_path, + capture_logs=False, + ) + collector.save_summary( + [ + BenchmarkResult(task_id="measured", success=True, score=1.0), + BenchmarkResult( + task_id="unavailable", + success=False, + score=0.0, + error_type="evaluation", + ), + ] + ) + + summary = json.loads((collector.run_dir / "summary.json").read_text()) + assert summary["num_tasks"] == 2 + assert summary["num_outcome_tasks"] == 1 + assert summary["error_count"] == 1 + assert summary["num_success"] == 1 + assert summary["success_rate"] == 1.0 + + def test_wandb_metrics_use_the_same_outcome_denominator(self): + results = [ + BenchmarkResult(task_id="measured", success=True, score=1.0), + BenchmarkResult( + task_id="unavailable", + success=False, + score=0.0, + error_type="infrastructure", + ), + ] + + metrics = WandbLogger._compute_metrics(object(), results) + + assert metrics["num_tasks"] == 2 + assert metrics["num_outcome_tasks"] == 1 + assert metrics["error_count"] == 1 + assert metrics["num_success"] == 1 + assert metrics["success_rate"] == 1.0 + class TestComputeDomainMetrics: """Tests for compute_domain_metrics function.""" @@ -556,8 +661,11 @@ def test_stops_on_done_action(self): # Should have only taken 0 steps (done before first step execution) assert result.num_steps == 0 - def test_agent_error_cannot_be_overwritten_by_successful_evaluation(self): - """A terminal agent error cannot be reported as task success.""" + @pytest.mark.parametrize("claimed_error_type", ["infrastructure", "evaluation"]) + def test_agent_error_claim_cannot_leave_outcome_denominator( + self, claimed_error_type + ): + """An agent cannot classify its own terminal failure as unavailable.""" adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) task = adapter.list_tasks()[0] adapter.evaluate = Mock( @@ -573,7 +681,7 @@ def test_agent_error_cannot_be_overwritten_by_successful_evaluation(self): type="error", raw_action={ "error": "provider offline", - "error_type": "infrastructure", + "error_type": claimed_error_type, }, ) ] @@ -590,7 +698,16 @@ def test_agent_error_cannot_be_overwritten_by_successful_evaluation(self): assert result.score == 0.0 assert result.error == "provider offline" assert result.reason == "provider offline" - assert result.error_type == "infrastructure" + assert result.error_type == "agent" + adapter.evaluate.assert_not_called() + + metrics = compute_metrics([result]) + assert metrics["num_tasks"] == 1 + assert metrics["num_outcome_tasks"] == 1 + assert metrics["success_count"] == 0 + assert metrics["fail_count"] == 1 + assert metrics["num_agent_failures"] == 1 + assert metrics["success_rate"] == 0.0 def test_agent_error_without_diagnostics_cannot_report_success(self): """The terminal action type alone is sufficient to refuse success.""" @@ -617,3 +734,287 @@ def test_agent_error_without_diagnostics_cannot_report_success(self): 0.0, "agent", ) + adapter.evaluate.assert_not_called() + + def test_agent_parse_exception_is_reported_as_agent_error(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + agent = Mock(spec=BenchmarkAgent) + agent.reset = Mock() + agent.act.side_effect = ActionParseError("invalid model output") + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "agent" + + def test_done_gate_evaluation_exception_is_not_accepted_as_done(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock(side_effect=RuntimeError("evaluator offline")) + agent = ScriptedAgent([BenchmarkAction(type="done")]) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + done_gate=True, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "evaluation" + assert "evaluator offline" in (result.error or "") + assert adapter.evaluate.call_count == 1 + + @pytest.mark.parametrize("error_type", ["infrastructure", "evaluation"]) + def test_done_gate_unscored_result_is_not_accepted_as_done(self, error_type): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + reason="result unavailable", + error_type=error_type, + ) + ) + agent = ScriptedAgent([BenchmarkAction(type="done")]) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + done_gate=True, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == error_type + assert result.error == "result unavailable" + + def test_done_gate_refresh_never_dispatches_synthetic_input(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + ) + ) + adapter.step = Mock(side_effect=AssertionError("must not dispatch input")) + adapter.observe = Mock(return_value=adapter.reset(task)) + agent = ScriptedAgent( + [BenchmarkAction(type="done"), BenchmarkAction(type="done")] + ) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + done_gate=True, + done_gate_max_overrides=1, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.error_type is None + assert result.success is False + adapter.observe.assert_called_once_with() + adapter.step.assert_not_called() + + def test_done_gate_stops_unscored_when_observation_refresh_is_missing(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + ) + ) + agent = Mock(spec=BenchmarkAgent) + agent.reset = Mock() + agent.act = Mock( + side_effect=[BenchmarkAction(type="done"), BenchmarkAction(type="done")] + ) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + done_gate=True, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "infrastructure" + assert "cannot provide a fresh observation" in (result.error or "") + assert agent.act.call_count == 1 + assert adapter.evaluate.call_count == 1 + + def test_done_gate_stops_unscored_when_observation_refresh_fails(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + ) + ) + adapter.observe = Mock(side_effect=RuntimeError("frame source offline")) + agent = Mock(spec=BenchmarkAgent) + agent.reset = Mock() + agent.act = Mock( + side_effect=[BenchmarkAction(type="done"), BenchmarkAction(type="done")] + ) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + done_gate=True, + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.success is False + assert result.error_type == "infrastructure" + assert "fresh observation failed: frame source offline" in (result.error or "") + assert agent.act.call_count == 1 + assert adapter.evaluate.call_count == 1 + + def test_final_evaluation_exception_is_classified_as_evaluation_error(self): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock(side_effect=RuntimeError("no evaluator response")) + agent = ScriptedAgent([BenchmarkAction(type="done")]) + + result = _run_single_task( + agent, + adapter, + task, + EvaluationConfig( + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + assert result.error_type == "evaluation" + assert "no evaluator response" in (result.error or "") + + def test_unavailable_evaluator_stays_unmeasured_through_public_runner( + self, monkeypatch + ): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + adapter.evaluate = Mock( + return_value=BenchmarkResult( + task_id=task.task_id, + success=False, + score=0.0, + error_type="evaluation", + reason="evaluator unavailable", + ) + ) + completed_event = Mock() + monkeypatch.setattr( + "openadapt_evals.benchmarks.runner.track_agent_run_completed", + completed_event, + ) + + results = evaluate_agent_on_benchmark( + ScriptedAgent([BenchmarkAction(type="done")]), + adapter, + config=EvaluationConfig( + verbose=False, + save_execution_traces=False, + enable_live_tracking=False, + ), + ) + + metrics = compute_metrics(results) + assert results[0].error_type == "evaluation" + assert metrics["num_tasks"] == 1 + assert metrics["num_outcome_tasks"] == 0 + assert metrics["error_count"] == 1 + assert metrics["success_count"] == 0 + completed_event.assert_called_once() + event = completed_event.call_args.kwargs + assert event["num_tasks"] == 1 + assert event["attempt_count"] == 1 + assert event["outcome_count"] == 0 + assert event["error_count"] == 1 + + def test_parallel_fallback_preserves_typed_error(self, monkeypatch): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + + def _raise(*_args, **_kwargs): + raise RolloutInfrastructureError("worker lost") + + monkeypatch.setattr( + "openadapt_evals.benchmarks.runner._run_single_task", _raise + ) + results = _evaluate_parallel( + Mock(spec=BenchmarkAgent), + adapter, + [task], + EvaluationConfig(verbose=False), + ) + + assert len(results) == 1 + assert results[0].error_type == "infrastructure" + assert results[0].reason == "worker lost" + + def test_sequential_fallback_preserves_typed_error(self, monkeypatch): + adapter = WAAMockAdapter(num_tasks=1, domains=["browser"]) + task = adapter.list_tasks()[0] + + def _raise(*_args, **_kwargs): + raise ActionParseError("worker parse failed") + + monkeypatch.setattr( + "openadapt_evals.benchmarks.runner._run_single_task", _raise + ) + results = _evaluate_sequential( + Mock(spec=BenchmarkAgent), + adapter, + [task], + EvaluationConfig(verbose=False), + ) + + assert len(results) == 1 + assert results[0].error_type == "agent" + assert results[0].reason == "worker parse failed" diff --git a/tests/test_standalone_trainer.py b/tests/test_standalone_trainer.py index 81de10b..3c0f5cf 100644 --- a/tests/test_standalone_trainer.py +++ b/tests/test_standalone_trainer.py @@ -6,12 +6,39 @@ from __future__ import annotations +import io import re +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest - +from PIL import Image + +from openadapt_evals.errors import ( + ActionParseError, + RolloutEvaluationError, + RolloutInfrastructureError, + RolloutLossError, +) +from openadapt_evals.task_config import Milestone, TaskCheck from openadapt_evals.training.standalone.config import TrainingConfig +from openadapt_evals.training.standalone.prompt import ( + SimpleAction, + format_action_as_text, + parse_vlm_output_to_action, +) +from openadapt_evals.training.standalone.reward import ( + compute_group_advantages, + evaluate_milestones_screenshot, +) from openadapt_evals.training.standalone.trainer import GRPOTrainer +from openadapt_evals.training.standalone.waa_direct import Rollout, RolloutStep + + +def _tiny_png() -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (2, 2)).save(buffer, format="PNG") + return buffer.getvalue() # --------------------------------------------------------------------------- # Action regex @@ -28,7 +55,7 @@ class TestActionRegex: "output", [ "Thought: I need to click the start menu.\nAction: CLICK(x=0.50, y=0.30)", - "Thought: Type notepad in the search box.\nAction: TYPE(text=\"notepad\")", + 'Thought: Type notepad in the search box.\nAction: TYPE(text="notepad")', "Thought: Wait for the UI to load.\nAction: WAIT()", "Thought: The task is complete.\nAction: DONE()", "Thought: Click Chrome icon.\nAction: CLICK(x=0.05, y=0.20)", @@ -71,10 +98,279 @@ def test_no_large_bounded_quantifiers(self) -> None: """Bounded quantifiers > 10 cause DFA state explosion in Outlines.""" bounds = re.findall(r"\{(\d+),(\d+)\}", self.full_regex) for lo, hi in bounds: - assert int(hi) <= 10, ( - f"Quantifier {{{lo},{hi}}} will explode Outlines DFA. Use +/*." + assert int(hi) <= 10, f"Quantifier {{{lo},{hi}}} will explode Outlines DFA. Use +/*." + + def test_unparseable_output_is_not_task_completion(self) -> None: + with pytest.raises(ActionParseError): + parse_vlm_output_to_action("I do not know what action to take") + + @pytest.mark.parametrize( + "output", + [ + "CLICK(x=-0.1, y=0.5)", + '{"action_type": "click", "coordinate": [-1, 20]}', + '{"action_type": "type"}', + ], + ) + def test_invalid_action_arguments_are_rejected(self, output: str) -> None: + with pytest.raises(ActionParseError): + parse_vlm_output_to_action(output) + + def test_normalized_edge_stays_inside_viewport(self) -> None: + action = parse_vlm_output_to_action("CLICK(x=1.0, y=1.0)", screen_size=(1920, 1080)) + assert (action.x, action.y) == (1919, 1079) + + @pytest.mark.parametrize( + "coordinate", + [ + [0.5, 200], + [100.5, 200], + [True, 200], + ], + ) + def test_json_click_rejects_mixed_or_fractional_pixel_coordinates( + self, coordinate + ) -> None: + with pytest.raises(ActionParseError): + parse_vlm_output_to_action( + '{"action_type":"click","coordinate":' + f"{coordinate!r}".replace("True", "true") + + "}", + screen_size=(1920, 1080), ) + def test_json_click_accepts_explicit_integer_pixel_coordinates(self) -> None: + action = parse_vlm_output_to_action( + '{"action_type":"click","coordinate":[100,200]}', + screen_size=(1920, 1080), + ) + assert (action.x, action.y) == (100, 200) + + @pytest.mark.parametrize( + "action", + [ + SimpleAction(type="click", x=None, y=10), + SimpleAction(type="type", text=None), + SimpleAction(type="scroll"), + ], + ) + def test_formatter_does_not_invent_missing_or_unknown_actions( + self, action: SimpleAction + ) -> None: + with pytest.raises(ActionParseError): + format_action_as_text(action) + + +class TestFailureVisibility: + @pytest.mark.parametrize( + ("field", "value"), + [ + ("num_training_steps", 0), + ("num_rollouts_per_step", 0), + ("max_steps_per_episode", 0), + ("save_every_steps", 0), + ("learning_rate", float("nan")), + ("learning_rate", 0.0), + ], + ) + def test_invalid_training_config_stops_before_checkpoint( + self, tmp_path, field, value + ) -> None: + config = TrainingConfig(output_dir=str(tmp_path / "checkpoints")) + setattr(config, field, value) + trainer = GRPOTrainer(config) + + with pytest.raises(ValueError, match=field): + trainer.train() + + assert not (tmp_path / "checkpoints").exists() + + def test_all_skipped_steps_do_not_publish_a_trained_checkpoint( + self, tmp_path + ) -> None: + config = TrainingConfig( + task_ids=["task-1"], + num_training_steps=2, + output_dir=str(tmp_path / "checkpoints"), + ) + trainer = GRPOTrainer(config) + model = MagicMock() + model.parameters.return_value = [] + environment = MagicMock() + environment.health_check.return_value = True + fake_torch = SimpleNamespace( + optim=SimpleNamespace(AdamW=MagicMock(return_value=MagicMock())) + ) + + with ( + patch.object(trainer, "_load_task_configs"), + patch.object(trainer, "_collect_group", return_value=[MagicMock()]), + patch.object( + trainer, + "_training_step", + return_value={"reward_mean": 0.5, "loss": 0.0, "skipped": True}, + ), + patch.object(trainer, "_save_checkpoint") as save_checkpoint, + patch( + "openadapt_evals.training.standalone.trainer.load_model_and_processor", + return_value=(model, MagicMock()), + ), + patch( + "openadapt_evals.training.standalone.trainer.WAADirect", + return_value=environment, + ), + patch.dict("sys.modules", {"torch": fake_torch}), + ): + with pytest.raises(RolloutLossError, match="no optimizer updates"): + trainer.train() + + save_checkpoint.assert_not_called() + + def test_probe_failure_does_not_return_an_empty_group(self) -> None: + trainer = GRPOTrainer(TrainingConfig()) + trainer._env = MagicMock() + trainer._env.probe.return_value = {"screenshot_ok": False} + + with pytest.raises(RolloutInfrastructureError): + trainer._collect_group("task-1") + + @pytest.mark.parametrize( + "steps", + [ + [], + [RolloutStep(screenshot=b"not an image", action=MagicMock())], + ], + ) + def test_missing_loss_evidence_does_not_return_zero(self, steps) -> None: + trainer = GRPOTrainer(TrainingConfig()) + trainer._model = MagicMock() + rollout = Rollout(task_id="task-1", steps=steps) + + with pytest.raises(RolloutLossError): + trainer._compute_rollout_loss(rollout, advantage=1.0, scale=1.0) + + def test_empty_training_group_does_not_return_zero_metrics(self) -> None: + trainer = GRPOTrainer(TrainingConfig()) + with pytest.raises(RolloutLossError): + trainer._training_step([]) + + @pytest.mark.parametrize( + "reward", + [-0.1, 1.1, float("nan"), float("inf"), True, "1.0"], + ) + def test_invalid_reward_cannot_become_an_advantage(self, reward) -> None: + with pytest.raises(RolloutLossError, match="Reward 0"): + compute_group_advantages([reward]) + + def test_milestone_evaluator_failure_is_not_a_zero_reward(self) -> None: + task = SimpleNamespace( + milestones=[ + Milestone( + name="Saved", + check=TaskCheck(check="screenshot", description="Saved result"), + ) + ] + ) + with patch( + "openadapt_evals.vlm_evaluator.vlm_judge", + side_effect=RuntimeError("judge unavailable"), + ): + with pytest.raises(RolloutEvaluationError): + evaluate_milestones_screenshot(task, _tiny_png()) + + def test_measured_failed_milestone_remains_zero(self) -> None: + task = SimpleNamespace( + milestones=[ + Milestone( + name="Saved", + check=TaskCheck(check="screenshot", description="Saved result"), + ) + ] + ) + with patch( + "openadapt_evals.vlm_evaluator.vlm_judge", + return_value=(False, 0.1), + ): + assert evaluate_milestones_screenshot(task, _tiny_png()) == 0.0 + + def test_unreadable_screenshot_is_not_a_measured_reward(self) -> None: + task = SimpleNamespace( + milestones=[ + Milestone( + name="Saved", + check=TaskCheck(check="screenshot", description="Saved result"), + ) + ] + ) + with patch("openadapt_evals.vlm_evaluator.vlm_judge") as judge: + with pytest.raises(RolloutEvaluationError, match="decodable"): + evaluate_milestones_screenshot(task, b"not an image") + judge.assert_not_called() + + def test_malformed_judge_result_is_not_a_measured_reward(self) -> None: + task = SimpleNamespace( + milestones=[ + Milestone( + name="Saved", + check=TaskCheck(check="screenshot", description="Saved result"), + ) + ] + ) + with patch( + "openadapt_evals.vlm_evaluator.vlm_judge", + return_value=(True, float("nan")), + ): + with pytest.raises(RolloutEvaluationError, match="confidence"): + evaluate_milestones_screenshot(task, _tiny_png()) + + def test_nonfinite_loss_stops_before_optimizer_step(self) -> None: + trainer = GRPOTrainer(TrainingConfig()) + trainer._optimizer = MagicMock() + trainer._compute_rollout_loss = MagicMock(return_value=float("nan")) + rollouts = [ + Rollout(task_id="task", reward=0.0), + Rollout(task_id="task", reward=1.0), + ] + + with pytest.raises(RolloutLossError, match="loss must be finite"): + trainer._training_step(rollouts) + trainer._optimizer.step.assert_not_called() + + def test_overflowed_loss_metrics_stop_before_optimizer_step(self) -> None: + trainer = GRPOTrainer(TrainingConfig()) + trainer._optimizer = MagicMock() + trainer._compute_rollout_loss = MagicMock(return_value=1e308) + rollouts = [ + Rollout(task_id="task", reward=0.0), + Rollout(task_id="task", reward=1.0), + ] + + with pytest.raises(RolloutLossError, match="metrics must be finite"): + trainer._training_step(rollouts) + trainer._optimizer.step.assert_not_called() + + def test_nonfinite_gradient_stops_before_optimizer_step(self) -> None: + torch = pytest.importorskip("torch") + trainer = GRPOTrainer(TrainingConfig()) + trainer._optimizer = MagicMock() + parameter = torch.nn.Parameter(torch.tensor(1.0)) + trainer._model = MagicMock() + trainer._model.parameters.return_value = [parameter] + + def write_bad_gradient(*_args, **_kwargs): + parameter.grad = torch.tensor(float("nan")) + return 1.0 + + trainer._compute_rollout_loss = MagicMock(side_effect=write_bad_gradient) + rollouts = [ + Rollout(task_id="task", reward=0.0), + Rollout(task_id="task", reward=1.0), + ] + + with pytest.raises(RolloutLossError, match="non-finite gradient"): + trainer._training_step(rollouts) + trainer._optimizer.step.assert_not_called() + # --------------------------------------------------------------------------- # Outlines integration @@ -122,6 +418,7 @@ def test_image_wrapper(self) -> None: import io from PIL import Image as PILImage + img = PILImage.new("RGB", (10, 10)) buf = io.BytesIO() img.save(buf, format="PNG") @@ -137,10 +434,7 @@ def test_generator_callable_with_kwargs(self) -> None: except ImportError: pytest.skip("outlines not installed") sig = inspect.signature(SteerableGenerator.__call__) - has_kwargs = any( - p.kind == inspect.Parameter.VAR_KEYWORD - for p in sig.parameters.values() - ) + has_kwargs = any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) assert has_kwargs, f"SteerableGenerator.__call__ needs **kwargs: {sig}" @@ -176,34 +470,54 @@ class TestTaskRotation: def test_all_tasks_loaded(self, tmp_path) -> None: import yaml + for i in range(5): - (tmp_path / f"t{i}.yaml").write_text(yaml.dump({ - "name": f"Task {i}", "id": f"task-{i}", - "setup": [], - "evaluate": [{"check": "screenshot", "description": "done"}], - })) + (tmp_path / f"t{i}.yaml").write_text( + yaml.dump( + { + "name": f"Task {i}", + "id": f"task-{i}", + "setup": [], + "evaluate": [{"check": "screenshot", "description": "done"}], + } + ) + ) config = TrainingConfig(task_dir=str(tmp_path)) GRPOTrainer(config)._load_task_configs() assert len(config.task_ids) == 5 def test_explicit_ids_preserved(self, tmp_path) -> None: import yaml + for i in range(3): - (tmp_path / f"t{i}.yaml").write_text(yaml.dump({ - "name": f"Task {i}", "id": f"task-{i}", "setup": [], "evaluate": [], - })) + (tmp_path / f"t{i}.yaml").write_text( + yaml.dump( + { + "name": f"Task {i}", + "id": f"task-{i}", + "setup": [], + "evaluate": [], + } + ) + ) config = TrainingConfig(task_dir=str(tmp_path), task_ids=["task-1"]) GRPOTrainer(config)._load_task_configs() assert config.task_ids == ["task-1"] def test_rotation_covers_all(self, tmp_path) -> None: import yaml + for i in range(3): - (tmp_path / f"t{i}.yaml").write_text(yaml.dump({ - "name": f"Task {i}", "id": f"task-{i}", - "setup": [], - "evaluate": [{"check": "screenshot", "description": "done"}], - })) + (tmp_path / f"t{i}.yaml").write_text( + yaml.dump( + { + "name": f"Task {i}", + "id": f"task-{i}", + "setup": [], + "evaluate": [{"check": "screenshot", "description": "done"}], + } + ) + ) config = TrainingConfig(task_dir=str(tmp_path)) GRPOTrainer(config)._load_task_configs() selected = {config.task_ids[s % len(config.task_ids)] for s in range(9)} diff --git a/tests/test_task_config.py b/tests/test_task_config.py index 6b7f72a..32a21e5 100644 --- a/tests/test_task_config.py +++ b/tests/test_task_config.py @@ -2,15 +2,25 @@ from __future__ import annotations +import io import json import os import textwrap from unittest.mock import patch import pytest +from PIL import Image +from openadapt_evals.adapters.base import EvaluationUnavailableError +from openadapt_evals.errors import RolloutEvaluationError from openadapt_evals.task_config import Milestone, TaskCheck, TaskConfig + +def _tiny_png() -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (2, 2)).save(buffer, format="PNG") + return buffer.getvalue() + # --------------------------------------------------------------------------- # YAML loading tests # --------------------------------------------------------------------------- @@ -316,9 +326,25 @@ def test_no_milestones(self): max_steps=15, milestones=[], ) - passed, total = task.evaluate_milestones(b"fake", "http://localhost:5001") - assert passed == 0 - assert total == 0 + with pytest.raises(EvaluationUnavailableError, match="No milestone"): + task.evaluate_milestones(_tiny_png(), "http://localhost:5001") + + def test_unmeasured_milestone_is_not_counted_as_failed(self): + task = TaskConfig( + name="Test", + id="test-001", + domain="desktop", + setup=[], + checks=[], + combine="and", + max_steps=15, + milestones=[ + Milestone(name="Unknown", check=TaskCheck(check="file")), + ], + ) + + with pytest.raises(EvaluationUnavailableError, match="could not be evaluated"): + task.evaluate_milestones(_tiny_png(), "http://localhost:5001") # --------------------------------------------------------------------------- @@ -332,7 +358,7 @@ def test_vlm_judge_yes(self, mock_vlm): mock_vlm.return_value = '{"verdict": "YES", "confidence": 0.95, "explanation": "Font is Arial"}' from openadapt_evals.vlm_evaluator import vlm_judge - success, confidence = vlm_judge(b"fake-png", "Font is Arial") + success, confidence = vlm_judge(_tiny_png(), "Font is Arial") assert success is True assert confidence == 0.95 @@ -341,26 +367,44 @@ def test_vlm_judge_no(self, mock_vlm): mock_vlm.return_value = '{"verdict": "NO", "confidence": 0.8, "explanation": "Font is Times"}' from openadapt_evals.vlm_evaluator import vlm_judge - success, confidence = vlm_judge(b"fake-png", "Font is Arial") + success, confidence = vlm_judge(_tiny_png(), "Font is Arial") assert success is False assert confidence == 0.8 @patch("openadapt_evals.vlm.vlm_call") - def test_vlm_judge_bad_json_fallback(self, mock_vlm): + def test_vlm_judge_bad_json_is_unmeasured(self, mock_vlm): mock_vlm.return_value = "YES, the font looks like Arial to me." from openadapt_evals.vlm_evaluator import vlm_judge - success, confidence = vlm_judge(b"fake-png", "Font is Arial") - assert success is True - assert confidence == 0.5 # fallback confidence + with pytest.raises(RolloutEvaluationError, match="required JSON"): + vlm_judge(_tiny_png(), "Font is Arial") @patch("openadapt_evals.vlm.vlm_call") - def test_vlm_judge_no_fallback(self, mock_vlm): - mock_vlm.return_value = "NO, I don't see that." + def test_vlm_judge_rejects_non_exact_verdict(self, mock_vlm): + mock_vlm.return_value = '{"verdict":"YES, probably","confidence":0.8}' from openadapt_evals.vlm_evaluator import vlm_judge - success, confidence = vlm_judge(b"fake-png", "Font is Arial") - assert success is False + with pytest.raises(RolloutEvaluationError, match="exactly"): + vlm_judge(_tiny_png(), "Font is Arial") + + @pytest.mark.parametrize("confidence", [-0.1, 1.1, float("nan"), "0.9", True]) + @patch("openadapt_evals.vlm.vlm_call") + def test_vlm_judge_rejects_invalid_confidence(self, mock_vlm, confidence): + mock_vlm.return_value = json.dumps( + {"verdict": "YES", "confidence": confidence} + ) + from openadapt_evals.vlm_evaluator import vlm_judge + + with pytest.raises(RolloutEvaluationError, match="confidence"): + vlm_judge(_tiny_png(), "Font is Arial") + + @patch("openadapt_evals.vlm.vlm_call") + def test_vlm_judge_rejects_unreadable_screenshot_before_model_call(self, mock_vlm): + from openadapt_evals.vlm_evaluator import vlm_judge + + with pytest.raises(RolloutEvaluationError, match="decodable"): + vlm_judge(b"not an image", "Font is Arial") + mock_vlm.assert_not_called() # --------------------------------------------------------------------------- diff --git a/tests/test_task_config_receipts.py b/tests/test_task_config_receipts.py new file mode 100644 index 0000000..e666050 --- /dev/null +++ b/tests/test_task_config_receipts.py @@ -0,0 +1,193 @@ +"""Regression tests for command-backed evaluation receipts.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from openadapt_evals.adapters import EvaluationUnavailableError +from openadapt_evals.task_config import TaskCheck, TaskConfig + + +def _response( + *, + returncode: int = 0, + stdout: str = "saved", + stderr: str = "", + **outer: object, +) -> MagicMock: + response = MagicMock(status_code=200) + response.json.return_value = { + "output": json.dumps( + { + "returncode": returncode, + "stdout": stdout, + "stderr": stderr, + } + ), + **outer, + } + return response + + +def _task(check: TaskCheck) -> TaskConfig: + return TaskConfig( + name="Receipt check", + id="receipt-check", + domain="desktop", + setup=[], + checks=[check], + combine="and", + max_steps=1, + milestones=[], + ) + + +def test_successful_empty_receipt_can_prove_exact_empty() -> None: + with patch("requests.post", return_value=_response(stdout="")): + actual = TaskConfig._run_vm_command("true", "http://vm") + + assert actual == "" + assert TaskConfig._check_match(actual, "", "exact") is True + + +@pytest.mark.parametrize( + ("response", "error_type"), + [ + (_response(returncode=1, stdout=""), "evaluation"), + (_response(stderr="warning"), "evaluation"), + (_response(success=False), "infrastructure"), + (_response(error="server failed"), "infrastructure"), + (_response(returncode=0, stdout="ok", stderr="", returncode_outer=1), None), + ], +) +def test_failed_command_or_server_receipt_is_unscored( + response: MagicMock, + error_type: str | None, +) -> None: + if error_type is None: + payload = response.json.return_value + payload["returncode"] = payload.pop("returncode_outer") + error_type = "infrastructure" + + with ( + patch("requests.post", return_value=response), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("command", "http://vm") + + assert excinfo.value.error_type == error_type + + +def test_plain_stdout_without_receipt_is_unscored() -> None: + response = MagicMock(status_code=200) + response.json.return_value = {"output": ""} + + with ( + patch("requests.post", return_value=response), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("false", "http://vm") + + assert excinfo.value.error_type == "evaluation" + + +def test_duplicate_command_receipt_fields_are_unscored() -> None: + response = MagicMock(status_code=200) + response.json.return_value = { + "output": ( + '{"returncode":1,"returncode":0,"stdout":"",' + '"stderr":"failed","stderr":""}' + ) + } + + with ( + patch("requests.post", return_value=response), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("false", "http://vm") + + assert excinfo.value.error_type == "evaluation" + + +def test_duplicate_outer_receipt_fields_are_unscored() -> None: + response = requests.Response() + response.status_code = 200 + response._content = ( + b'{"returncode":1,"returncode":0,' + b'"output":"{\\"returncode\\":0,\\"stdout\\":\\"\\",' + b'\\"stderr\\":\\"\\"}"}' + ) + + with ( + patch("requests.post", return_value=response), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("command", "http://vm") + + assert excinfo.value.error_type == "infrastructure" + + +@pytest.mark.parametrize("success", [False, 0, "false", None]) +def test_invalid_explicit_outer_success_is_unscored(success: object) -> None: + with ( + patch("requests.post", return_value=_response(success=success)), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("command", "http://vm") + + assert excinfo.value.error_type == "infrastructure" + + +def test_request_error_preserves_infrastructure_category() -> None: + with ( + patch("requests.post", side_effect=requests.ConnectionError("offline")), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + TaskConfig._run_vm_command("command", "http://vm") + + assert excinfo.value.error_type == "infrastructure" + + +def test_failed_command_plus_empty_exact_cannot_score() -> None: + task = _task(TaskCheck(check="command", run="false", expect="", match="exact")) + + with ( + patch("requests.post", return_value=_response(returncode=1, stdout="")), + pytest.raises(EvaluationUnavailableError), + ): + task.evaluate_checks_local(b"", "http://vm") + + +def test_whitespace_only_command_cannot_score_exact_empty() -> None: + task = _task(TaskCheck(check="command", run=" ", expect="", match="exact")) + + with pytest.raises(EvaluationUnavailableError) as excinfo: + task.evaluate_checks_local(b"", "http://vm") + + assert excinfo.value.error_type == "evaluation" + + +@pytest.mark.parametrize("match_type", ["contains", "regex", "fuzzy"]) +def test_match_contract_that_empty_expected_would_trivially_pass_is_refused( + match_type: str, +) -> None: + with pytest.raises(ValueError, match="non-empty"): + TaskConfig._check_match("anything", "", match_type) + + +def test_empty_contains_is_unscored_in_local_evaluation() -> None: + task = _task( + TaskCheck(check="command", run="echo saved", expect="", match="contains") + ) + + with ( + patch("requests.post", return_value=_response(stdout="saved")), + pytest.raises(EvaluationUnavailableError) as excinfo, + ): + task.evaluate_checks_local(b"", "http://vm") + + assert excinfo.value.error_type == "evaluation" diff --git a/tests/test_trace_analysis.py b/tests/test_trace_analysis.py index f6f7db7..e4145a0 100644 --- a/tests/test_trace_analysis.py +++ b/tests/test_trace_analysis.py @@ -342,9 +342,11 @@ def test_summary_full_eval(self, full_eval_jsonl: Path): analyzer = TraceAnalyzer(full_eval_jsonl) s = analyzer.summary() assert s["total_episodes"] == 5 - assert s["success_rate"] == 0.4 + assert s["outcome_episodes"] == 4 + assert s["unscored_episodes"] == 1 + assert s["success_rate"] == 0.5 assert s["total_steps"] == 33 - assert s["avg_score"] == 0.5 + assert s["avg_score"] == 0.625 assert s["model"] == "claude-sonnet-4-6" assert s["cost_estimate_usd"] > 0 diff --git a/tests/test_trl_robustness.py b/tests/test_trl_robustness.py index 4c137fb..caa3700 100644 --- a/tests/test_trl_robustness.py +++ b/tests/test_trl_robustness.py @@ -6,7 +6,6 @@ - Stuck detection (P1) - Truncation warning (P1) - DiagnosticsCallback (P2) -- _empty_rollout_result helper All tests are "light" -- they mock heavy deps (torch, transformers, PIL) and run with [dev] deps only. @@ -17,13 +16,16 @@ import logging from unittest.mock import MagicMock, patch +import pytest + from openadapt_evals.adapters.base import ( BenchmarkObservation, BenchmarkResult, BenchmarkTask, ) +from openadapt_evals.errors import RolloutInfrastructureError +from openadapt_evals.task_config import TaskConfig from openadapt_evals.training.trl_rollout import ( - _empty_rollout_result, _run_episode, make_waa_rollout_func, ) @@ -53,14 +55,20 @@ def _make_mock_adapter(screenshot=_GOOD_SCREENSHOT): adapter.observe.return_value = BenchmarkObservation( screenshot=screenshot, + viewport=(1920, 1200), raw_observation={}, ) adapter.reset.return_value = BenchmarkObservation( screenshot=screenshot, + viewport=(1920, 1200), raw_observation={}, ) adapter.step.return_value = ( - BenchmarkObservation(screenshot=screenshot, raw_observation={}), + BenchmarkObservation( + screenshot=screenshot, + viewport=(1920, 1200), + raw_observation={}, + ), False, {}, ) @@ -77,6 +85,19 @@ def _make_mock_adapter(screenshot=_GOOD_SCREENSHOT): return adapter +class _HealthAdapter: + """A non-mock adapter type used to exercise the live health gate.""" + + def __init__(self, screenshot=_GOOD_SCREENSHOT, error=None): + self._screenshot = screenshot + self._error = error + + def observe(self): + if self._error is not None: + raise self._error + return BenchmarkObservation(screenshot=self._screenshot, raw_observation={}) + + def _make_mock_trainer(num_generations=2): """Create a mock TRL GRPOTrainer.""" trainer = MagicMock() @@ -90,6 +111,19 @@ def _make_mock_trainer(num_generations=2): return trainer +def _make_task_config() -> TaskConfig: + return TaskConfig( + name="Test task", + id="test-task", + domain="desktop", + setup=[], + checks=[], + combine="and", + max_steps=5, + milestones=[], + ) + + # --------------------------------------------------------------------------- # Feature 1: Pre-rollout health check # --------------------------------------------------------------------------- @@ -101,7 +135,9 @@ class TestHealthCheck: def test_health_check_passes(self): """When adapter.observe() returns a good screenshot, rollout proceeds.""" adapter = _make_mock_adapter() - func = make_waa_rollout_func(adapter, max_steps=3) + func = make_waa_rollout_func( + adapter, task_configs=[_make_task_config()], max_steps=3 + ) trainer = _make_mock_trainer(num_generations=1) from openadapt_evals.training import trl_rollout @@ -118,40 +154,31 @@ def mock_run(env, gfn, instr, tid, ms, stuck_threshold=3): assert result["env_reward"] == [0.5] assert result["completion_ids"] == [[2, 3]] - def test_health_check_fails_returns_zeros(self): - """When adapter.observe() raises, rollout returns zero rewards.""" - adapter = _make_mock_adapter() - adapter.observe.side_effect = ConnectionError("server down") + def test_health_check_failure_is_not_zero_reward(self): + adapter = _HealthAdapter(error=ConnectionError("server down")) func = make_waa_rollout_func(adapter, max_steps=3) trainer = _make_mock_trainer(num_generations=2) - result = func(["Task A", "Task B"], trainer) - - # 2 prompts x 2 generations = 4 entries, all zero - assert len(result["env_reward"]) == 4 - assert all(r == 0.0 for r in result["env_reward"]) - assert all(ids == [] for ids in result["completion_ids"]) + with pytest.raises(RolloutInfrastructureError, match="server down"): + func(["Task A", "Task B"], trainer) def test_health_check_empty_screenshot(self): """When adapter returns empty screenshot bytes, returns zeros.""" - adapter = _make_mock_adapter(screenshot=b"") + adapter = _HealthAdapter(screenshot=b"") func = make_waa_rollout_func(adapter, max_steps=3) trainer = _make_mock_trainer(num_generations=1) - result = func(["Test task"], trainer) - - assert result["env_reward"] == [0.0] - assert result["completion_ids"] == [[]] + with pytest.raises(RolloutInfrastructureError, match="0 bytes"): + func(["Test task"], trainer) def test_health_check_small_screenshot(self): """Screenshot smaller than 100 bytes triggers health check failure.""" - adapter = _make_mock_adapter(screenshot=b"\x89PNG" + b"\x00" * 50) + adapter = _HealthAdapter(screenshot=b"\x89PNG" + b"\x00" * 50) func = make_waa_rollout_func(adapter, max_steps=3) trainer = _make_mock_trainer(num_generations=1) - result = func(["Test task"], trainer) - - assert result["env_reward"] == [0.0] + with pytest.raises(RolloutInfrastructureError, match="54 bytes"): + func(["Test task"], trainer) # --------------------------------------------------------------------------- @@ -177,6 +204,7 @@ def test_corrupt_screenshot_retry_succeeds(self, caplog): adapter = _make_mock_adapter() func = make_waa_rollout_func( adapter, + task_configs=[_make_task_config()], max_steps=3, screenshot_retries=3, screenshot_retry_delay=0, @@ -234,10 +262,11 @@ def mock_run(env, gfn, instr, tid, ms, stuck_threshold=3): ) def test_corrupt_screenshot_all_fail(self): - """All retry attempts fail -- returns DONE action ("done", [], []).""" + """Unreadable observation data must stop the rollout.""" adapter = _make_mock_adapter() func = make_waa_rollout_func( adapter, + task_configs=[_make_task_config()], max_steps=3, screenshot_retries=3, screenshot_retry_delay=0, @@ -253,17 +282,11 @@ def mock_run(env, gfn, instr, tid, ms, stuck_threshold=3): env.reset(config=ResetConfig(task_id=tid)) with patch.object(Image, "open", side_effect=OSError("always corrupt")): - text, ids, lps = gfn(b"not a png", "test instruction") - - assert text == "done" - assert ids == [] - assert lps == [] - return [1], [2], [-0.1], 0.0 + gfn(b"not a png", "test instruction") with patch.object(trl_rollout, "_run_episode", side_effect=mock_run): - result = func(["Test task"], trainer) - - assert result["env_reward"] == [0.0] + with pytest.raises(RolloutInfrastructureError, match="unreadable"): + func(["Test task"], trainer) # --------------------------------------------------------------------------- @@ -329,7 +352,9 @@ def varying_observe(*args, **kwargs): adapter.observe = varying_observe # Also vary reset return adapter.reset.return_value = BenchmarkObservation( - screenshot=screenshots[0], raw_observation={}, + screenshot=screenshots[0], + viewport=(1920, 1200), + raw_observation={}, ) env = RLEnvironment(adapter) @@ -350,7 +375,9 @@ def varying_step(action): step_idx["n"] += 1 return ( BenchmarkObservation( - screenshot=screenshots[idx], raw_observation={}, + screenshot=screenshots[idx], + viewport=(1920, 1200), + raw_observation={}, ), False, {}, @@ -411,6 +438,35 @@ def fake_generate(screenshot_bytes, instruction): # Without stuck detection, all 8 steps should run assert call_count["n"] == 8 + def test_scroll_up_uses_structured_step_dispatch(self): + from openadapt_evals.adapters.rl_env import RLEnvironment + + adapter = _make_mock_adapter() + env = RLEnvironment(adapter) + outputs = iter( + [ + ( + '{"type":"scroll","x":0.5,"y":0.5,"direction":"up"}', + [1], + [-0.1], + ), + ('{"type":"done"}', [2], [-0.1]), + ] + ) + + _run_episode( + env, + lambda screenshot, instruction: next(outputs), + "Test", + "test-task", + max_steps=2, + stuck_threshold=0, + ) + + dispatched = adapter.step.call_args_list[0].args[0] + assert dispatched.type == "scroll" + assert dispatched.scroll_direction == "up" + # --------------------------------------------------------------------------- # Feature 4: Truncation warning @@ -444,6 +500,7 @@ def test_truncation_warning_logged_hf_path(self, caplog): max_new_tokens = 10 func = make_waa_rollout_func( adapter, + task_configs=[_make_task_config()], max_steps=1, max_new_tokens=max_new_tokens, screenshot_retries=1, @@ -522,6 +579,7 @@ def test_no_truncation_warning_when_short(self, caplog): max_new_tokens = 256 func = make_waa_rollout_func( adapter, + task_configs=[_make_task_config()], max_steps=1, max_new_tokens=max_new_tokens, screenshot_retries=1, @@ -652,53 +710,3 @@ def test_diagnostics_callback_missing_metrics(self, caplog): # Should still log with zeroed metrics assert any("Step 1" in r.message for r in caplog.records) - - -# --------------------------------------------------------------------------- -# _empty_rollout_result helper -# --------------------------------------------------------------------------- - - -class TestEmptyRolloutResult: - """Tests for the _empty_rollout_result helper.""" - - def test_empty_rollout_result_shape(self): - """Verify correct dict structure for various prompt/gen combos.""" - result = _empty_rollout_result(["A", "B", "C"], num_generations=4) - - assert len(result["prompt_ids"]) == 12 # 3 * 4 - assert len(result["completion_ids"]) == 12 - assert len(result["logprobs"]) == 12 - assert len(result["env_reward"]) == 12 - - # All rewards should be zero - assert all(r == 0.0 for r in result["env_reward"]) - - # All lists should be empty - assert all(ids == [] for ids in result["prompt_ids"]) - assert all(ids == [] for ids in result["completion_ids"]) - assert all(lps == [] for lps in result["logprobs"]) - - def test_empty_rollout_result_single(self): - """Single prompt, single generation.""" - result = _empty_rollout_result(["X"], num_generations=1) - - assert len(result["env_reward"]) == 1 - assert result["env_reward"] == [0.0] - - def test_empty_rollout_result_has_all_keys(self): - """Result contains all four required keys.""" - result = _empty_rollout_result(["A"], num_generations=1) - - assert "prompt_ids" in result - assert "completion_ids" in result - assert "logprobs" in result - assert "env_reward" in result - - def test_empty_rollout_result_lists_are_independent(self): - """Each inner list is a distinct object (not shared references).""" - result = _empty_rollout_result(["A", "B"], num_generations=2) - - # Mutating one inner list should not affect others - result["prompt_ids"][0].append(999) - assert result["prompt_ids"][1] == [] diff --git a/tests/test_trl_rollout.py b/tests/test_trl_rollout.py index d5e68be..3f73095 100644 --- a/tests/test_trl_rollout.py +++ b/tests/test_trl_rollout.py @@ -4,12 +4,15 @@ from unittest.mock import MagicMock, patch +import pytest + from openadapt_evals.adapters.base import ( BenchmarkAction, BenchmarkObservation, BenchmarkResult, BenchmarkTask, ) +from openadapt_evals.errors import ActionParseError, RolloutInfrastructureError from openadapt_evals.task_config import Milestone, TaskCheck, TaskConfig from openadapt_evals.training.trl_rollout import ( make_waa_rollout_func, @@ -42,10 +45,8 @@ def test_done_action(self): action = parse_action_json('{"type": "done"}') assert action.type == "done" - def test_with_thinking_prefix(self): - text = """I need to click the button. - -```json + def test_fenced_json(self): + text = """```json {"type": "click", "x": 0.2, "y": 0.8} ```""" action = parse_action_json(text) @@ -53,25 +54,46 @@ def test_with_thinking_prefix(self): assert action.x == 0.2 def test_with_markdown_fence(self): - action = parse_action_json('```json\n{"type": "scroll", "x": 0.5, "y": 0.5}\n```') + action = parse_action_json( + '```json\n{"type":"scroll","x":0.5,"y":0.5,"direction":"up"}\n```' + ) assert action.type == "scroll" + assert action.scroll_direction == "up" - def test_no_json_returns_done(self): - action = parse_action_json("I'm not sure what to do") - assert action.type == "done" + def test_no_action_is_not_task_completion(self): + with pytest.raises(ActionParseError): + parse_action_json("I'm not sure what to do") - def test_invalid_json_returns_done(self): - action = parse_action_json("{broken json}") - assert action.type == "done" + def test_invalid_json_is_not_task_completion(self): + with pytest.raises(ActionParseError): + parse_action_json("{broken json}") - def test_unknown_type_returns_done(self): - action = parse_action_json('{"type": "fly_to_moon", "x": 0}') - assert action.type == "done" + def test_unknown_type_is_not_task_completion(self): + with pytest.raises(ActionParseError): + parse_action_json('{"type": "fly_to_moon", "x": 0}') def test_noop(self): action = parse_action_json('{"type": "noop"}') assert action.type == "noop" + @pytest.mark.parametrize( + "text", + [ + '{"type": "click", "x": 0.5}', + '{"type": "click", "x": -0.1, "y": 0.5}', + '{"type": "click", "x": 0.5, "y": 2}', + '{"type": "type"}', + '{"type": "key", "key": ""}', + '{"type": "scroll", "y": 0.5}', + '{"type": "scroll", "x": 0.5, "y": 0.5}', + '{"type": "scroll", "x": 0.5, "y": 0.5, "direction": "left"}', + "CLICK(x=-0.2, y=0.5)", + ], + ) + def test_missing_or_invalid_arguments_are_rejected(self, text): + with pytest.raises(ActionParseError): + parse_action_json(text) + # --------------------------------------------------------------------------- # make_waa_rollout_func tests @@ -195,9 +217,15 @@ def mock_run_episode(env, generate_fn, instruction, task_id, max_steps, **kwargs def test_rollout_output_shape_multiple_prompts(self): """Verify output shape with multiple prompts × generations.""" adapter = _make_mock_adapter(steps_to_done=1) - tc = _make_task_config() + tc1 = _make_task_config() + tc1.name = "Task A" + tc2 = _make_task_config() + tc2.name = "Task B" + tc2.id = "test-task-b" - func = make_waa_rollout_func(adapter, task_configs=[tc], max_steps=3) + func = make_waa_rollout_func( + adapter, task_configs=[tc1, tc2], max_steps=3 + ) trainer = _make_mock_trainer() trainer.args.num_generations = 3 @@ -221,10 +249,11 @@ def mock_run(env, gfn, instr, tid, ms): assert len(result["completion_ids"]) == 6 assert len(result["logprobs"]) == 6 - def test_rollout_handles_episode_failure(self): - """Verify graceful handling when an episode throws an exception.""" + def test_rollout_propagates_episode_failure(self): adapter = _make_mock_adapter() - func = make_waa_rollout_func(adapter, max_steps=3) + func = make_waa_rollout_func( + adapter, task_configs=[_make_task_config()], max_steps=3 + ) trainer = _make_mock_trainer() trainer.args.num_generations = 1 @@ -235,13 +264,8 @@ def failing_run(env, gfn, instr, tid, ms): raise RuntimeError("VM connection lost") with patch.object(trl_rollout, "_run_episode", side_effect=failing_run): - result = func(["Failing task"], trainer) - - # Should return zeros, not crash - assert result["env_reward"] == [0.0] - assert result["prompt_ids"] == [[]] - assert result["completion_ids"] == [[]] - assert result["logprobs"] == [[]] + with pytest.raises(RuntimeError, match="VM connection lost"): + func(["Test task"], trainer) def test_task_config_lookup_by_name(self): """Verify task_configs are indexed by name for prompt matching.""" @@ -394,23 +418,19 @@ def mock_run(env, gfn, instr, tid, ms): assert len(result["env_reward"]) == 1 assert result["env_reward"][0] == 0.5 - def test_broken_callback_does_not_crash_training(self): - """A callback that raises should be caught and logged, not crash.""" + def test_broken_setup_callback_aborts_collection(self): + """A failed pre-rollout setup must not evaluate stale task state.""" adapter = _make_mock_adapter(steps_to_done=1) tc = _make_task_config() def exploding_before(task_id, env): raise ValueError("Boom in before_collect!") - def exploding_complete(rollout, index): - raise RuntimeError("Boom in rollout_complete!") - func = make_waa_rollout_func( adapter=adapter, task_configs=[tc], max_steps=3, on_before_collect=exploding_before, - on_rollout_complete=exploding_complete, ) trainer = _make_mock_trainer() @@ -424,8 +444,45 @@ def mock_run(env, gfn, instr, tid, ms): env.step(BenchmarkAction(type="done")) return [1], [2], [-0.1], 0.5 - with patch.object(trl_rollout, "_run_episode", side_effect=mock_run): - result = func(["Test task"], trainer) + with patch.object(trl_rollout, "_run_episode", side_effect=mock_run) as run: + with pytest.raises(RolloutInfrastructureError, match="before_collect"): + func(["Test task"], trainer) + run.assert_not_called() - assert len(result["env_reward"]) == 1 - assert result["env_reward"][0] == 0.5 + def test_unmatched_prompt_refuses_before_episode(self): + adapter = _make_mock_adapter() + func = make_waa_rollout_func( + adapter, task_configs=[_make_task_config()], max_steps=3 + ) + trainer = _make_mock_trainer() + trainer.args.num_generations = 1 + + from openadapt_evals.training import trl_rollout + + with patch.object(trl_rollout, "_run_episode") as run: + with pytest.raises(RolloutInfrastructureError, match="no exact TaskConfig"): + func(["A different task"], trainer) + run.assert_not_called() + + def test_zero_generations_refuses_before_episode(self): + adapter = _make_mock_adapter() + func = make_waa_rollout_func( + adapter, task_configs=[_make_task_config()], max_steps=3 + ) + trainer = _make_mock_trainer() + trainer.args.num_generations = 0 + + from openadapt_evals.training import trl_rollout + + with patch.object(trl_rollout, "_run_episode") as run: + with pytest.raises(RolloutInfrastructureError, match="num_generations"): + func(["Test task"], trainer) + run.assert_not_called() + + def test_zero_max_steps_is_rejected_when_rollout_is_built(self): + with pytest.raises(ValueError, match="max_steps"): + make_waa_rollout_func( + _make_mock_adapter(), + task_configs=[_make_task_config()], + max_steps=0, + ) diff --git a/tests/test_verl_env.py b/tests/test_verl_env.py index b349719..23e23f6 100644 --- a/tests/test_verl_env.py +++ b/tests/test_verl_env.py @@ -7,9 +7,11 @@ from __future__ import annotations import asyncio +from unittest.mock import MagicMock import pytest +from openadapt_evals.adapters.base import BenchmarkObservation from openadapt_evals.adapters.rl_env import RLEnvironment from openadapt_evals.adapters.verl_env import ( _ACTION_PATTERN, @@ -22,6 +24,7 @@ register_in_vagen, ) from openadapt_evals.adapters.waa.mock import WAAMockAdapter +from openadapt_evals.errors import ActionParseError # --- Action parsing tests --- @@ -65,13 +68,13 @@ def test_scroll_up(self): action = _parse_action_str('SCROLL(x=0.50, y=0.50, direction="up")') assert action.scroll_direction == "up" - def test_scroll_default_direction(self): - action = _parse_action_str("SCROLL(x=0.50, y=0.50)") - assert action.scroll_direction == "down" + def test_scroll_requires_direction(self): + with pytest.raises(ActionParseError, match="direction"): + _parse_action_str("SCROLL(x=0.50, y=0.50)") - def test_invalid_returns_done(self): - action = _parse_action_str("random garbage text") - assert action.type == "done" + def test_invalid_is_not_task_completion(self): + with pytest.raises(ActionParseError): + _parse_action_str("random garbage text") def test_drag_with_end_coords(self): action = _parse_action_str("DRAG(x=0.20, y=0.30, end_x=0.80, end_y=0.70)") @@ -82,10 +85,24 @@ def test_drag_with_end_coords(self): assert action.end_y == pytest.approx(0.70) def test_drag_without_end_coords(self): - action = _parse_action_str("DRAG(x=0.20, y=0.30)") - assert action.type == "drag" - assert action.end_x is None - assert action.end_y is None + with pytest.raises(ActionParseError, match="end_x"): + _parse_action_str("DRAG(x=0.20, y=0.30)") + + @pytest.mark.parametrize( + "text", + [ + "CLICK(x=0.5)", + "CLICK(x=-0.1, y=0.5)", + "CLICK(x=nan, y=0.5)", + "SCROLL(x=0.5)", + 'SCROLL(x=0.5, y=0.5, direction="left")', + "TYPE()", + "KEY()", + ], + ) + def test_missing_or_invalid_arguments_are_rejected(self, text): + with pytest.raises(ActionParseError): + _parse_action_str(text) def test_with_thinking(self): action = _parse_action_str( @@ -203,6 +220,52 @@ def test_step_done_triggers_eval(self): assert done is True assert isinstance(reward, float) + def test_step_does_not_turn_evaluator_failure_into_zero_reward(self): + env = _make_mock_env() + asyncio.run(env.reset(seed=42)) + env._rl_env.evaluate = MagicMock(side_effect=RuntimeError("evaluator offline")) + + with pytest.raises(RuntimeError, match="evaluator offline"): + asyncio.run(env.step("DONE()")) + + def test_partial_reward_is_not_reported_as_success(self): + env = _make_mock_env() + asyncio.run(env.reset(seed=42)) + env._rl_env.evaluate = MagicMock(return_value=0.75) + + _, reward, done, info = asyncio.run(env.step("DONE()")) + + assert done is True + assert reward == 0.75 + assert info["success"] is False + + @pytest.mark.parametrize( + "action_text, expected", + [ + ("CLICK(x=1.0, y=1.0)", (1919, 1199, None, None)), + ( + "DRAG(x=1.0, y=1.0, end_x=1.0, end_y=1.0)", + (1919, 1199, 1919, 1199), + ), + ], + ) + def test_fractional_edge_stays_inside_viewport(self, action_text, expected): + env = _make_mock_env() + asyncio.run(env.reset(seed=42)) + + asyncio.run(env.step(action_text)) + + action = env._rl_env.trajectory[-1].action + assert (action.x, action.y, action.end_x, action.end_y) == expected + + def test_fractional_action_rejects_invalid_viewport_dimensions(self): + env = _make_mock_env() + asyncio.run(env.reset(seed=42)) + env._rl_env._last_obs = BenchmarkObservation(viewport=(0, 1200)) + + with pytest.raises(ValueError, match="dimension must be positive"): + asyncio.run(env.step("CLICK(x=0.5, y=0.5)")) + def test_max_steps_triggers_done(self): env = _make_mock_env() env._max_steps = 3 @@ -261,12 +324,11 @@ def test_is_action_valid_on_done(self): _, _, _, info = asyncio.run(env.step("DONE()")) assert info["is_action_valid"] is True - def test_is_action_valid_on_garbage(self): - """Unparseable actions should have is_action_valid=False.""" + def test_garbage_does_not_end_the_episode(self): env = _make_mock_env() asyncio.run(env.reset(seed=42)) - _, _, _, info = asyncio.run(env.step("random garbage text")) - assert info["is_action_valid"] is False + with pytest.raises(ActionParseError): + asyncio.run(env.step("random garbage text")) def test_obs_contains_image_placeholder(self): """Test that observations with screenshots include placeholder.""" diff --git a/tests/test_waa.py b/tests/test_waa.py index 5b7a40e..cbe8e3e 100644 --- a/tests/test_waa.py +++ b/tests/test_waa.py @@ -1,5 +1,7 @@ """Tests for WAA benchmark adapter.""" +from unittest.mock import MagicMock + import pytest from openadapt_evals import ( @@ -9,10 +11,127 @@ BenchmarkTask, RandomAgent, ScriptedAgent, + WAAAdapter, WAAMockAdapter, compute_metrics, evaluate_agent_on_benchmark, ) +from openadapt_evals.errors import RolloutEvaluationError, RolloutInfrastructureError + + +class TestWAAAdapterEvaluation: + """Tests for strict native WAA evaluator result handling.""" + + @staticmethod + def _adapter(tmp_path, response=None, error=None): + adapter = WAAAdapter(waa_repo_path=tmp_path) + adapter._desktop_env = MagicMock() + if error is not None: + adapter._desktop_env.evaluate.side_effect = error + else: + adapter._desktop_env.evaluate.return_value = response + return adapter + + @pytest.mark.parametrize( + "response", + [ + None, + [], + {}, + True, + float("nan"), + float("inf"), + -0.1, + 1.1, + {"success": 1, "score": 1.0}, + {"success": True, "score": True}, + {"success": False, "score": float("nan")}, + {"success": False, "score": float("inf")}, + {"success": False, "score": -0.1}, + {"success": True, "score": 1.1}, + {"success": True, "score": 0.5}, + {"success": False, "score": 1.0}, + ], + ) + def test_malformed_measurement_stays_unscored(self, tmp_path, response): + task = BenchmarkTask(task_id="task", instruction="test", domain="desktop") + result = self._adapter(tmp_path, response=response).evaluate(task) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == "evaluation" + assert result.reason and result.reason.startswith( + "Malformed evaluation response:" + ) + + @pytest.mark.parametrize("error_type", ["infrastructure", "evaluation"]) + def test_unavailable_response_preserves_type(self, tmp_path, error_type): + task = BenchmarkTask(task_id="task", instruction="test", domain="desktop") + response = { + "success": False, + "score": 0.0, + "scored": False, + "error_type": error_type, + "reason": "evaluator unavailable", + } + + result = self._adapter(tmp_path, response=response).evaluate(task) + + assert (result.success, result.score, result.error_type) == ( + False, + 0.0, + error_type, + ) + assert result.reason == "evaluator unavailable" + + @pytest.mark.parametrize( + ("error", "expected_type"), + [ + (RuntimeError("invalid evaluator"), "evaluation"), + (RolloutInfrastructureError("VM offline"), "infrastructure"), + (RolloutEvaluationError("oracle offline"), "evaluation"), + ], + ) + def test_evaluator_exception_preserves_stable_type( + self, tmp_path, error, expected_type + ): + task = BenchmarkTask(task_id="task", instruction="test", domain="desktop") + + result = self._adapter(tmp_path, error=error).evaluate(task) + + assert result.success is False + assert result.score == 0.0 + assert result.error_type == expected_type + assert result.error == str(error) + + @pytest.mark.parametrize( + ("response", "expected"), + [ + ({"success": False, "score": 0.25}, (False, 0.25)), + ({"success": True, "score": 1.0}, (True, 1.0)), + ], + ) + def test_valid_measurement_is_preserved(self, tmp_path, response, expected): + task = BenchmarkTask(task_id="task", instruction="test", domain="desktop") + + result = self._adapter(tmp_path, response=response).evaluate(task) + + assert (result.success, result.score) == expected + assert result.error_type is None + + @pytest.mark.parametrize( + ("response", "expected"), + [(0.0, (False, 0.0)), (0.25, (False, 0.25)), (1.0, (True, 1.0))], + ) + def test_native_scalar_measurement_is_preserved( + self, tmp_path, response, expected + ): + task = BenchmarkTask(task_id="task", instruction="test", domain="desktop") + + result = self._adapter(tmp_path, response=response).evaluate(task) + + assert (result.success, result.score) == expected + assert result.error_type is None class TestWAAMockAdapter: diff --git a/tests/test_wandb_result_surfaces.py b/tests/test_wandb_result_surfaces.py new file mode 100644 index 0000000..0d4c875 --- /dev/null +++ b/tests/test_wandb_result_surfaces.py @@ -0,0 +1,107 @@ +"""W&B surfaces must keep unavailable attempts separate from outcomes.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from openadapt_evals.adapters import BenchmarkResult +from openadapt_evals.integrations.wandb_logger import WandbLogger + + +def _logger() -> WandbLogger: + logger = object.__new__(WandbLogger) + logger._run = MagicMock() + logger._results_buffer = [] + logger._tasks_logged = 0 + return logger + + +def test_live_rate_excludes_unavailable_attempts() -> None: + logger = _logger() + logger.log_task_result(BenchmarkResult(task_id="ok", success=True, score=1.0)) + logger.log_task_result( + BenchmarkResult( + task_id="infra", + success=False, + score=0.0, + error_type="infrastructure", + ) + ) + + payload = logger._run.log.call_args_list[-1].args[0] + assert payload["task/current_success_rate"] == 1.0 + assert payload["task/outcomes_completed"] == 1 + assert payload["task/last_task_is_outcome"] is False + assert payload["task/last_task_success"] is None + + +def test_task_table_keeps_classification_and_reason() -> None: + logger = _logger() + result = BenchmarkResult( + task_id="infra", + success=False, + score=0.0, + error="offline", + reason="evaluator offline", + error_type="infrastructure", + ) + table = MagicMock() + with patch("openadapt_evals.integrations.wandb_logger.Table", return_value=table) as cls: + logger._log_task_table([result], {}) + + columns = cls.call_args.kwargs["columns"] + row = cls.call_args.kwargs["data"][0] + assert row[columns.index("is_outcome")] is False + assert row[columns.index("error_type")] == "infrastructure" + assert row[columns.index("reason")] == "evaluator offline" + + +def test_error_breakdown_groups_by_error_type_not_message() -> None: + logger = _logger() + results = [ + BenchmarkResult( + task_id="a", + success=False, + score=0.0, + error="offline A", + error_type="infrastructure", + ), + BenchmarkResult( + task_id="b", + success=False, + score=0.0, + error="offline B", + error_type="infrastructure", + ), + ] + with patch("openadapt_evals.integrations.wandb_logger.Table") as table: + logger._log_error_breakdown(results) + + assert table.call_args.kwargs["data"] == [["infrastructure", 2, 100.0]] + + +def test_aggregate_rate_publishes_its_outcome_denominator() -> None: + logger = _logger() + results = [ + BenchmarkResult(task_id="mail_1", success=True, score=1.0), + BenchmarkResult( + task_id="mail_2", + success=False, + score=0.0, + error_type="infrastructure", + ), + ] + with ( + patch.object(logger, "_log_task_table"), + patch.object(logger, "_log_error_breakdown"), + patch.object(logger, "_log_step_distribution"), + ): + logger.log_results(results) + + global_payload = logger._run.log.call_args_list[0].args[0] + domain_payload = logger._run.log.call_args_list[1].args[0] + assert global_payload["eval/success_rate"] == 1.0 + assert global_payload["eval/num_outcome_tasks"] == 1 + assert global_payload["eval/num_infrastructure_failures"] == 1 + assert domain_payload["eval/domain/mail/num_outcome_tasks"] == 1 + assert domain_payload["eval/domain/mail/num_infrastructure_failures"] == 1 From 07b0bd0cdc47a5c7d312a142cc25de7b57353722 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 28 Jul 2026 09:49:46 -0400 Subject: [PATCH 2/2] fix: require a positive evaluator receipt signal --- openadapt_evals/evaluation/receipts.py | 3 ++- tests/test_evaluator_boundaries.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/openadapt_evals/evaluation/receipts.py b/openadapt_evals/evaluation/receipts.py index e6245eb..5ead4ac 100644 --- a/openadapt_evals/evaluation/receipts.py +++ b/openadapt_evals/evaluation/receipts.py @@ -66,7 +66,8 @@ def require_successful_receipt( raise ReceiptValidationError( f"{context} receipt has failed={payload['failed']!r}" ) - positive = True + # An explicit lack of failure is not proof that the operation + # completed. Require a separate positive outcome signal. for field in ("error", "stderr"): if field not in payload: diff --git a/tests/test_evaluator_boundaries.py b/tests/test_evaluator_boundaries.py index bad398f..d02ff76 100644 --- a/tests/test_evaluator_boundaries.py +++ b/tests/test_evaluator_boundaries.py @@ -83,6 +83,7 @@ def test_postconfig_request_exception_is_unscored_infrastructure() -> None: "receipt", [ {"success": False}, + {"failed": False}, {"success": True, "stderr": "warning"}, {"delivery_state": "uncertain"}, {"delivery_state": "invalid"},