fix(ios): double-check off-screen click refusals against a direct element read

packages/contracts/src/interaction-guarantees.ts

@@ -147,9 +147,18 @@ const RUNTIME_TREE_SHARED_GUARANTEES = {
     kind: 'runtime',
     via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
   },
+  // #1542: the base decision is isNodeVisibleOnScreen (bulk accessibility
+  // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end
+  // enforcement point — on iOS (local, non-provider sessions only) a would-be
+  // refusal is re-checked against a live, tree-independent read via the
+  // optional AgentDeviceBackend.confirmOffscreenTargetVisible hook before
+  // erroring, and a confirmed rescue re-targets the action at the LIVE rect,
+  // not the bulk one. Every other platform, and any backend that omits the
+  // hook, refuses on isNodeVisibleOnScreen's verdict unchanged — this is a
+  // rescue-only override, never a way to relax a genuine refusal.
   offscreen: {
     kind: 'runtime',
-    via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen',
+    via: 'src/commands/interaction/runtime/resolution.ts#throwIfOffscreenInteractionTarget',
   },
   nonHittable: {
     kind: 'runtime',
@@ -291,9 +300,16 @@ export const INTERACTION_DISPATCH_PATHS: Record<InteractionPathId, InteractionPa
         kind: 'runtime',
         via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked',
       },
+      // Same enforcement point as the runtime-tree paths (#1542): the
+      // preflight guard IS throwIfOffscreenInteractionTarget, which can
+      // rescue via the optional iOS confirmOffscreenTargetVisible hook — see
+      // the comment on RUNTIME_TREE_SHARED_GUARANTEES.offscreen above. This
+      // path is web-only in production (no mobile backend implements
+      // tapTarget/fillTarget), so the rescue hook never fires here in
+      // practice, but the code path is identical.
       offscreen: {
         kind: 'runtime',
-        via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen',
+        via: 'src/commands/interaction/runtime/resolution.ts#throwIfOffscreenInteractionTarget',
       },
       // Annotation only (targetHittable/hint on the result): promotion to a
       // hittable ancestor stays a runtime-path behavior — the preflight never

src/backend.ts

@@ -431,6 +431,37 @@ export type AgentDeviceBackend = {
   ): Promise<BackendScreenshotResult | void>;
   readText?(context: BackendCommandContext, node: SnapshotNode): Promise<BackendReadTextResult>;
   findText?(context: BackendCommandContext, text: string): Promise<BackendFindTextResult>;
+  /**
+   * #1542 off-screen refusal double-check: called ONLY at the moment the
+   * shared off-screen interaction guard is about to REFUSE a click/tap/
+   * gesture-target resolution, to re-confirm the target directly — bypassing
+   * whatever bulk accessibility tree the guard's verdict came from (observed
+   * on iOS: a keyboard-dismiss content-offset correction can leave a
+   * ScrollView's bulk AX frame squeezed to a stale value, or the whole bulk
+   * tree pinned at pre-gesture values, while the target is genuinely fine).
+   *
+   * Conceptually a boolean ("is this actually visible?"), but returns the
+   * confirmed LIVE rect rather than a bare `true`/`false`: a rescue must tap
+   * at the live coordinate, never the stale bulk-tree one the guard was
+   * about to refuse — a caller that used the original rect after a rescue
+   * would silently tap the wrong place when the bulk tree is stale, not just
+   * stale-looking. `rootViewport` is the guard's own already-resolved root
+   * viewport (Application/Window frame), passed in so an implementation can
+   * validate the live rect's tap point against it without recomputing it.
+   *
+   * Returns `null` when the target cannot be positively confirmed on-screen
+   * (no stable id/label, not found, ambiguous match, not hittable, outside
+   * `rootViewport`, or any transport failure) — the guard MUST fail closed
+   * (refuse) on `null`. This is a rescue path only, never a way to relax a
+   * genuine refusal. Backends that do not support a direct, tree-independent
+   * read simply omit this method, which leaves today's refuse-on-off-screen
+   * behavior byte-for-byte unchanged.
+   */
+  confirmOffscreenTargetVisible?(
+    context: BackendCommandContext,
+    node: Pick<SnapshotNode, 'identifier' | 'label'>,
+    rootViewport: Rect | null,
+  ): Promise<Rect | null>;
   tap?(
     context: BackendCommandContext,
     point: Point,

src/commands/interaction/runtime/__tests__/test-utils/index.ts

@@ -298,6 +298,7 @@ export function createInteractionDevice(
       | 'longPress'
       | 'scroll'
       | 'performGesture'
+      | 'confirmOffscreenTargetVisible'
     >
   > & {
     platform?: AgentDeviceBackend['platform'];
@@ -327,6 +328,12 @@ export function createInteractionDevice(
         : undefined,
       scroll: overrides.scroll ? async (...args) => await overrides.scroll?.(...args) : undefined,
       performGesture: overrides.performGesture,
+      // #1542: undefined by default, so every existing test keeps proving the
+      // fail-closed refusal unchanged; only tests that opt in exercise the
+      // rescue/agreement paths.
+      confirmOffscreenTargetVisible: overrides.confirmOffscreenTargetVisible
+        ? async (...args) => (await overrides.confirmOffscreenTargetVisible?.(...args)) ?? null
+        : undefined,
     } satisfies AgentDeviceBackend,
     artifacts: createLocalArtifactAdapter(),
     sessions: createMemorySessionStore([

src/commands/interaction/runtime/offscreen-double-check.test.ts

@@ -0,0 +1,149 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import type { Rect } from '@agent-device/kernel/snapshot';
+import { ref, selector } from './selector-read-utils.ts';
+import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts';
+import { createInteractionDevice } from './__tests__/test-utils/index.ts';
+
+// #1542: end-to-end coverage for the off-screen refusal double-check, next to
+// resolution.ts (AGENTS.md forbids adding to daemon/handlers/__tests__/
+// interaction.test.ts). Uses the same low-level runtime harness
+// resolution.test.ts uses (createInteractionDevice), mocking
+// `confirmOffscreenTargetVisible` directly rather than the local XCTest
+// runner — the runner-level probe I/O is covered separately in
+// src/daemon/__tests__/offscreen-target-probe.test.ts.
+
+// The bulk tree's ScrollView ancestor is corrupted (squeezed to a sliver),
+// so the target's rect doesn't overlap it — the guard rejects even though
+// the target's OWN bulk rect ({x:126,y:136}) happens to already be correct.
+function keyboardSqueezedAncestorSnapshot() {
+  return makeSnapshotState([
+    { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'ScrollView',
+      label: 'Checkout form',
+      rect: { x: 18, y: 381, width: 366, height: 109 },
+    },
+    {
+      index: 2,
+      depth: 2,
+      parentIndex: 1,
+      type: 'Button',
+      label: 'Pickup',
+      identifier: 'shipping-pickup',
+      rect: { x: 126, y: 136, width: 75, height: 38 },
+      hittable: true,
+    },
+  ]);
+}
+
+// The FROZEN-TREE manifestation: the whole bulk tree is pinned at
+// pre-gesture values, so the target's OWN bulk rect is stale — a DIFFERENT
+// place than where it actually is now. This is the shape #1542's live
+// review flagged: a naive rescue that only checks "is it confirmed
+// on-screen?" without also swapping in the live rect would tap the STALE
+// bulk coordinate.
+function frozenTreeStaleTargetSnapshot() {
+  return makeSnapshotState([
+    { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      label: 'Pickup',
+      identifier: 'shipping-pickup',
+      // Stale pre-gesture position, off the bottom of the (frozen) viewport.
+      rect: { x: 20, y: 2000, width: 100, height: 40 },
+      hittable: true,
+    },
+  ]);
+}
+
+const STALE_TARGET_REF = '@e2';
+// Where the runner's live probe says the element ACTUALLY is right now.
+const LIVE_RECT: Rect = { x: 150, y: 300, width: 100, height: 40 };
+const LIVE_CENTER = { x: LIVE_RECT.x + LIVE_RECT.width / 2, y: LIVE_RECT.y + LIVE_RECT.height / 2 };
+const STALE_CENTER = { x: 70, y: 2020 }; // center of the stale bulk rect above
+
+test('rescue: click succeeds when confirmOffscreenTargetVisible confirms the target on-screen', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), {
+    tap: async (_context, point) => {
+      calls.push(point);
+    },
+    confirmOffscreenTargetVisible: async (_context, node) =>
+      node.identifier === 'shipping-pickup' ? { x: 126, y: 136, width: 75, height: 38 } : null,
+  });
+
+  const result = await device.interactions.click(selector('id="shipping-pickup"'), {
+    session: 'default',
+  });
+
+  assert.equal(result.kind, 'selector');
+  // centerOfRect rounds: (126 + 75/2, 136 + 38/2) = (163.5, 155) -> (164, 155).
+  assert.deepEqual(calls, [{ x: 164, y: 155 }]);
+});
+
+test('FROZEN-TREE regression (#1542): a rescued tap lands at the LIVE rect, never the stale bulk one', async () => {
+  // Counterfactual: revert resolution.ts's resolveRefInteractionTarget /
+  // resolveSelectorInteractionTarget to compute the point from the
+  // pre-guard `node` instead of the guard's returned (possibly
+  // rescue-patched) node. This test goes red — `calls` records the STALE
+  // center ({x:70,y:2020}) instead of the LIVE one asserted below.
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(frozenTreeStaleTargetSnapshot(), {
+    tap: async (_context, point) => {
+      calls.push(point);
+    },
+    confirmOffscreenTargetVisible: async (_context, node) =>
+      node.identifier === 'shipping-pickup' ? LIVE_RECT : null,
+  });
+
+  const result = await device.interactions.click(ref(STALE_TARGET_REF), { session: 'default' });
+
+  assert.equal(result.kind, 'ref');
+  assert.deepEqual(calls, [LIVE_CENTER]);
+  assert.notDeepEqual(calls, [STALE_CENTER]);
+  // The response's own point must agree with what was actually dispatched.
+  assert.deepEqual(result.point, LIVE_CENTER);
+});
+
+test('genuine refusal: confirmOffscreenTargetVisible returns null -> still refuses, no tap', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), {
+    tap: async (_context, point) => {
+      calls.push(point);
+    },
+    confirmOffscreenTargetVisible: async () => null,
+  });
+
+  await assert.rejects(
+    () => device.interactions.click(selector('id="shipping-pickup"'), { session: 'default' }),
+    (error: unknown) => {
+      assert.ok(error instanceof Error);
+      assert.match(error.message, /off-screen element/);
+      return true;
+    },
+  );
+  assert.equal(calls.length, 0);
+});
+
+test('no rescue hook wired (e.g. non-iOS backend) -> refuses exactly as before', async () => {
+  const calls: unknown[] = [];
+  const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), {
+    tap: async (_context, point) => {
+      calls.push(point);
+    },
+    // confirmOffscreenTargetVisible intentionally omitted.
+  });
+
+  await assert.rejects(
+    () => device.interactions.click(selector('id="shipping-pickup"'), { session: 'default' }),
+    /off-screen element/,
+  );
+  assert.equal(calls.length, 0);
+});

src/commands/interaction/runtime/resolution.test.ts

@@ -3,7 +3,7 @@ import { test } from 'vitest';
 import type { BackendSnapshotOptions } from '../../../backend.ts';
 import { ref, selector } from './selector-read-utils.ts';
 import { resolveActionableTouchResolution } from '../../../core/interaction-targeting.ts';
-import { tryResolveRefNode } from './resolution.ts';
+import { throwIfOffscreenInteractionTarget, tryResolveRefNode } from './resolution.ts';
 import { parseSelectorChain, resolveSelectorChain } from '../../../selectors/index.ts';
 import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts';
 import type { Point } from '@agent-device/kernel/snapshot';
@@ -614,3 +614,93 @@ test('tryResolveRefNode discloses exact for a resolved ref and label-fallback fo
 
   assert.equal(tryResolveRefNode(nodes, '@e9', { fallbackLabel: '' }), null);
 });
+
+// #1542: throwIfOffscreenInteractionTarget is exported for ADR 0011 registry
+// honesty (interaction-guarantees.ts's `offscreen` cells point their `via`
+// here); this direct-import test is its real consumer, mirroring
+// tryResolveRefNode above. End-to-end rescue/refuse coverage through the
+// public click/press surface lives in offscreen-double-check.test.ts.
+function fakeOffscreenFailure() {
+  return {
+    message: 'off-screen',
+    details: { reason: 'test' },
+    hint: () => 'scroll toward it',
+  };
+}
+
+test('throwIfOffscreenInteractionTarget: an on-screen node passes through unchanged', async () => {
+  const device = createInteractionDevice(makeSnapshotState([]));
+  const nodes = makeSnapshotState([
+    { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      rect: { x: 20, y: 20, width: 40, height: 40 },
+    },
+  ]).nodes;
+
+  const result = await throwIfOffscreenInteractionTarget(
+    device,
+    { session: 'default' },
+    nodes[1]!,
+    nodes,
+    fakeOffscreenFailure(),
+  );
+
+  assert.equal(result, nodes[1]);
+});
+
+test('throwIfOffscreenInteractionTarget: off-screen + backend confirms -> returns the node patched with the LIVE rect', async () => {
+  const device = createInteractionDevice(makeSnapshotState([]), {
+    confirmOffscreenTargetVisible: async () => ({ x: 30, y: 30, width: 40, height: 40 }),
+  });
+  const nodes = makeSnapshotState([
+    { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      rect: { x: 20, y: 2000, width: 40, height: 40 },
+    },
+  ]).nodes;
+
+  const result = await throwIfOffscreenInteractionTarget(
+    device,
+    { session: 'default' },
+    nodes[1]!,
+    nodes,
+    fakeOffscreenFailure(),
+  );
+
+  assert.deepEqual(result.rect, { x: 30, y: 30, width: 40, height: 40 });
+  assert.equal(result.index, nodes[1]!.index);
+});
+
+test('throwIfOffscreenInteractionTarget: off-screen + no rescue -> throws with the supplied failure shape', async () => {
+  const device = createInteractionDevice(makeSnapshotState([]));
+  const nodes = makeSnapshotState([
+    { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } },
+    {
+      index: 1,
+      depth: 1,
+      parentIndex: 0,
+      type: 'Button',
+      rect: { x: 20, y: 2000, width: 40, height: 40 },
+    },
+  ]).nodes;
+
+  await assert.rejects(
+    () =>
+      throwIfOffscreenInteractionTarget(
+        device,
+        { session: 'default' },
+        nodes[1]!,
+        nodes,
+        fakeOffscreenFailure(),
+      ),
+    /off-screen/,
+  );
+});