fix: preserve agent and readiness failures

openadapt_evals/agents/api_agent.py

@@ -400,13 +400,17 @@ def act(
         self._last_step_logs = logs
 
         # Parse response into BenchmarkAction
-        if actions and actions[0] in ("DONE", "FAIL", "WAIT"):
-            return BenchmarkAction(type="done", raw_action={"waa_action": actions[0]})
+        if actions and actions[0] == "DONE":
+            return BenchmarkAction(type="done", raw_action={"waa_action": "DONE"})
+        if actions and actions[0] == "FAIL":
+            return BenchmarkAction(type="error", raw_action={"waa_action": "FAIL"})
+        if actions and actions[0] == "WAIT":
+            return BenchmarkAction(type="wait", raw_action={"waa_action": "WAIT"})
 
         if actions and actions[0].startswith("computer."):
             return self._parse_computer_action(actions[0], observation)
 
-        return BenchmarkAction(type="done", raw_action={"error": "Could not parse action"})
+        return BenchmarkAction(type="error", raw_action={"error": "Could not parse action"})
 
     def predict(self, instruction: str, obs: dict) -> tuple:
         """WAA-compatible interface: Predict the next action based on observation.

openadapt_evals/agents/base.py

@@ -155,9 +155,10 @@ def parse_action_response(
                 break
 
     if not action_line:
-        # Could not parse action, return done
+        # Parsing did not produce an agent decision. Keep that distinct from
+        # the model explicitly issuing DONE().
         raw_action["parse_error"] = "No action pattern found"
-        return BenchmarkAction(type="done", raw_action=raw_action)
+        return BenchmarkAction(type="error", raw_action=raw_action)
 
     # Parse CLICK action
     click_match = re.match(
@@ -292,4 +293,4 @@ def parse_action_response(
 
     # Unknown action format
     raw_action["parse_error"] = f"Unknown action format: {action_line}"
-    return BenchmarkAction(type="done", raw_action=raw_action)
+    return BenchmarkAction(type="error", raw_action=raw_action)

openadapt_evals/agents/baseline_agent.py

@@ -143,7 +143,7 @@ def act(
         screenshot = self._observation_to_image(observation)
         if screenshot is None:
             logger.warning("No screenshot in observation")
-            return BenchmarkAction(type="done", raw_action={"error": "No screenshot"})
+            return BenchmarkAction(type="error", raw_action={"error": "No screenshot"})
 
         # Format accessibility tree
         a11y_tree = self._format_a11y_tree(observation.accessibility_tree)
@@ -161,7 +161,7 @@ def act(
             )
         except Exception as e:
             logger.error(f"Adapter predict failed: {e}")
-            return BenchmarkAction(type="done", raw_action={"error": str(e)})
+            return BenchmarkAction(type="error", raw_action={"error": str(e)})
 
         # Convert ParsedAction to BenchmarkAction
         return self._convert_action(parsed, observation)
@@ -245,7 +245,7 @@ def _convert_action(
                     raw_action=raw_action,
                 )
 
-            return BenchmarkAction(type="done", raw_action={**raw_action, "error": "Missing coords"})
+            return BenchmarkAction(type="error", raw_action={**raw_action, "error": "Missing coords"})
 
         elif action_type == "type":
             return BenchmarkAction(
@@ -273,7 +273,7 @@ def _convert_action(
 
         else:
             return BenchmarkAction(
-                type="done",
+                type="error",
                 raw_action={**raw_action, "error": f"Unknown action: {action_type}"},
             )
 

openadapt_evals/agents/claude_computer_use_agent.py

@@ -983,12 +983,15 @@ def _map_action(
                 f"'{action_type}' reached _map_action (should be handled by "
                 "act() loop). Treating as no-op."
             )
-            return BenchmarkAction(type="done", raw_action=raw)
+            return BenchmarkAction(
+                type="error",
+                raw_action={"error": f"Unexpected internal action: {action_type}", **raw},
+            )
 
         # Unknown action
         logger.warning(f"Unknown computer_use action: {action_type}")
         return BenchmarkAction(
-            type="done",
+            type="error",
             raw_action={"error": f"Unknown action: {action_type}", **raw},
         )
 

openadapt_evals/agents/planner_grounder_agent.py

@@ -375,22 +375,29 @@ def act(
                 },
             )
 
-        if decision == "FAIL":
+        if decision in ("FAIL", "ERROR"):
             self._action_history.append("FAIL()")
+            error = (
+                planner_output.get("error")
+                or reasoning
+                or "planner reported failure"
+            )
             return BenchmarkAction(
-                type="done",
+                type="error",
                 raw_action={
                     "planner_output": planner_output,
                     "source": "planner",
                     "fail_reason": reasoning,
+                    "error": error,
+                    "error_type": planner_output.get("error_type", "agent"),
                 },
             )
 
         if not instruction and not action_type:
-            logger.warning("Planner returned empty instruction, treating as DONE")
-            self._action_history.append("DONE() [empty instruction]")
+            logger.warning("Planner returned an empty instruction")
+            self._action_history.append("ERROR() [empty instruction]")
             return BenchmarkAction(
-                type="done",
+                type="error",
                 raw_action={
                     "planner_output": planner_output,
                     "parse_error": "empty_instruction",
@@ -831,7 +838,11 @@ def _call_grounder(
             if action.type != "done":
                 return action
 
-            logger.warning("Grounder retry also failed, returning done")
+            logger.warning("Grounder retry also failed")
+            return BenchmarkAction(
+                type="error",
+                raw_action={"parse_error": "grounder output was not actionable"},
+            )
 
         return action
 
@@ -892,7 +903,10 @@ def _call_grounder_http(
             raw = resp.json()["choices"][0]["message"]["content"]
         except Exception as exc:
             logger.error("HTTP grounder call failed: %s", exc)
-            return BenchmarkAction(type="done")
+            return BenchmarkAction(
+                type="error",
+                raw_action={"error": f"HTTP grounder call failed: {exc}"},
+            )
 
         logger.info("HTTP grounder raw output: %s", raw[:200])
 
@@ -932,7 +946,13 @@ def _parse_bbox_to_action(raw: str) -> BenchmarkAction:
         if not match:
             # Last resort: try parse_action_json
             from openadapt_evals.training.trl_rollout import parse_action_json
-            return parse_action_json(raw)
+            action = parse_action_json(raw)
+            if action.type == "done":
+                return BenchmarkAction(
+                    type="error",
+                    raw_action={"parse_error": "grounder output was not actionable"},
+                )
+            return action
 
         nums = [float(x) for x in match.groups() if x is not None]
 
@@ -944,7 +964,10 @@ def _parse_bbox_to_action(raw: str) -> BenchmarkAction:
             x, y = nums[0], nums[1]
         else:
             logger.warning("Unexpected number count in bbox: %s", nums)
-            return BenchmarkAction(type="done")
+            return BenchmarkAction(
+                type="error",
+                raw_action={"parse_error": f"unexpected bbox values: {nums}"},
+            )
 
         # Normalize coordinates
         if x > 1 and y > 1:
@@ -978,6 +1001,22 @@ def _action_to_planner_output(action: BenchmarkAction) -> dict[str, Any]:
             "reasoning": action.raw_action.get("reasoning", "") if action.raw_action else "",
         }
 
+    if action.type == "error":
+        raw_action = action.raw_action or {}
+        error = (
+            raw_action.get("reason")
+            or raw_action.get("error")
+            or raw_action.get("parse_error")
+            or "planner reported failure"
+        )
+        return {
+            "decision": "ERROR",
+            "instruction": "",
+            "reasoning": str(error),
+            "error": str(error),
+            "error_type": raw_action.get("error_type", "agent"),
+        }
+
     # Use the action string representation as the instruction.
     instruction = action_to_string(action)
 

openadapt_evals/agents/policy_agent.py

@@ -122,7 +122,7 @@ def act(
             return action
         except Exception as e:
             logger.error(f"Inference failed: {e}")
-            return BenchmarkAction(type="done", raw_action={"error": str(e)})
+            return BenchmarkAction(type="error", raw_action={"error": str(e)})
 
     def _build_prompt(
         self,

openadapt_evals/agents/qwen3vl_agent.py

@@ -297,7 +297,7 @@ def parse_qwen_action(
         raw["action_string"] = action_str
     else:
         raw["parse_error"] = "No action found in response"
-        return BenchmarkAction(type="done", raw_action=raw)
+        return BenchmarkAction(type="error", raw_action=raw)
 
     # --- finished() ---
     if _RE_FINISHED.search(action_str):
@@ -382,7 +382,7 @@ def parse_qwen_action(
                 type="key", key=keys[-1], modifiers=keys[:-1], raw_action=raw
             )
         raw["parse_error"] = "Empty keys list in press()"
-        return BenchmarkAction(type="done", raw_action=raw)
+        return BenchmarkAction(type="error", raw_action=raw)
 
     # --- scroll(direction="<dir>", amount=<int>) ---
     m = _RE_SCROLL.search(action_str)
@@ -416,7 +416,7 @@ def parse_qwen_action(
 
     # Unknown action format
     raw["parse_error"] = f"Unknown action format: {action_str}"
-    return BenchmarkAction(type="done", raw_action=raw)
+    return BenchmarkAction(type="error", raw_action=raw)
 
 
 # ---------------------------------------------------------------------------
@@ -631,7 +631,7 @@ def act(
         if image is None:
             logger.error("No screenshot available in observation")
             return BenchmarkAction(
-                type="done", raw_action={"error": "no_screenshot"}
+                type="error", raw_action={"error": "no_screenshot"}
             )
 
         # Build user content (aligned with convert_demos training format)
@@ -647,7 +647,7 @@ def act(
         except Exception as e:
             logger.error(f"Inference failed: {e}")
             return BenchmarkAction(
-                type="done", raw_action={"error": f"inference_failed: {e}"}
+                type="error", raw_action={"error": f"inference_failed: {e}"}
             )
 
         logger.info(f"Step {self._step_count} raw response: {response_text!r}")

openadapt_evals/agents/smol_agent.py

@@ -176,7 +176,7 @@ def parse_smol_action(
         raw["action_string"] = action_str
     else:
         raw["parse_error"] = "No action found in response"
-        return BenchmarkAction(type="done", raw_action=raw)
+        return BenchmarkAction(type="error", raw_action=raw)
 
     # --- final_answer('...') → done ---
     m = _RE_FINAL_ANSWER.search(action_str)
@@ -228,7 +228,7 @@ def parse_smol_action(
                 type="key", key=keys[-1], modifiers=keys[:-1], raw_action=raw
             )
         raw["parse_error"] = "Empty keys list in press()"
-        return BenchmarkAction(type="done", raw_action=raw)
+        return BenchmarkAction(type="error", raw_action=raw)
 
     # --- scroll(direction='...', amount=...) ---
     m = _RE_SCROLL.search(action_str)
@@ -270,7 +270,7 @@ def parse_smol_action(
 
     # Unknown action format
     raw["parse_error"] = f"Unknown action format: {action_str}"
-    return BenchmarkAction(type="done", raw_action=raw)
+    return BenchmarkAction(type="error", raw_action=raw)
 
 
 # ---------------------------------------------------------------------------
@@ -381,7 +381,7 @@ def act(
         if image is None:
             logger.error("No screenshot available in observation")
             return BenchmarkAction(
-                type="done", raw_action={"error": "no_screenshot"}
+                type="error", raw_action={"error": "no_screenshot"}
             )
 
         user_content = self._build_prompt(task.instruction)
@@ -392,7 +392,7 @@ def act(
         except Exception as e:
             logger.error(f"Inference failed: {e}")
             return BenchmarkAction(
-                type="done", raw_action={"error": f"inference_failed: {e}"}
+                type="error", raw_action={"error": f"inference_failed: {e}"}
             )
 
         logger.info(f"Step {self._step_count} raw response: {response_text!r}")

openadapt_evals/benchmarks/runner.py

@@ -509,10 +509,24 @@ def _run_single_task(
         logger.info("Evaluating task result")
         result = adapter.evaluate(task)
 
-        # Propagate error_type from agent error action
-        if action is not None and action.type == "error" and action.raw_action:
-            result.error_type = action.raw_action.get("error_type", "agent")
-            result.error = result.error or action.raw_action.get("reason")
+        # An evaluator observes target state, not whether the agent itself
+        # completed this attempt safely. A pre-existing target state can score
+        # as successful even after the agent reported a provider or parse
+        # failure, so the terminal agent error remains authoritative.
+        if action is not None and action.type == "error":
+            raw_action = action.raw_action or {}
+            error_reason = (
+                raw_action.get("reason")
+                or raw_action.get("error")
+                or raw_action.get("parse_error")
+                or raw_action.get("fail_reason")
+                or "agent reported a terminal error"
+            )
+            result.success = False
+            result.score = 0.0
+            result.error_type = raw_action.get("error_type", "agent")
+            result.error = str(error_reason)
+            result.reason = str(error_reason)
 
         # Update result with trajectory info
         result.steps = history if config.save_trajectories else []