diff --git a/packages/contracts/src/interaction-guarantees.ts b/packages/contracts/src/interaction-guarantees.ts index 8b82578f9c..c111b693eb 100644 --- a/packages/contracts/src/interaction-guarantees.ts +++ b/packages/contracts/src/interaction-guarantees.ts @@ -147,9 +147,18 @@ const RUNTIME_TREE_SHARED_GUARANTEES = { kind: 'runtime', via: 'src/snapshot/snapshot-occlusion.ts#isSnapshotNodeInteractionBlocked', }, + // #1542: the base decision is isNodeVisibleOnScreen (bulk accessibility + // tree), but throwIfOffscreenInteractionTarget is the actual end-to-end + // enforcement point — on iOS (local, non-provider sessions only) a would-be + // refusal is re-checked against a live, tree-independent read via the + // optional AgentDeviceBackend.confirmOffscreenTargetVisible hook before + // erroring, and a confirmed rescue re-targets the action at the LIVE rect, + // not the bulk one. Every other platform, and any backend that omits the + // hook, refuses on isNodeVisibleOnScreen's verdict unchanged — this is a + // rescue-only override, never a way to relax a genuine refusal. offscreen: { kind: 'runtime', - via: 'src/snapshot/mobile-snapshot-semantics.ts#isNodeVisibleOnScreen', + via: 'src/commands/interaction/runtime/resolution.ts#throwIfOffscreenInteractionTarget', }, nonHittable: { kind: 'runtime', @@ -291,9 +300,16 @@ export const INTERACTION_DISPATCH_PATHS: Record; readText?(context: BackendCommandContext, node: SnapshotNode): Promise; findText?(context: BackendCommandContext, text: string): Promise; + /** + * #1542 off-screen refusal double-check: called ONLY at the moment the + * shared off-screen interaction guard is about to REFUSE a click/tap/ + * gesture-target resolution, to re-confirm the target directly — bypassing + * whatever bulk accessibility tree the guard's verdict came from (observed + * on iOS: a keyboard-dismiss content-offset correction can leave a + * ScrollView's bulk AX frame squeezed to a stale value, or the whole bulk + * tree pinned at pre-gesture values, while the target is genuinely fine). + * + * Conceptually a boolean ("is this actually visible?"), but returns the + * confirmed LIVE rect rather than a bare `true`/`false`: a rescue must tap + * at the live coordinate, never the stale bulk-tree one the guard was + * about to refuse — a caller that used the original rect after a rescue + * would silently tap the wrong place when the bulk tree is stale, not just + * stale-looking. `rootViewport` is the guard's own already-resolved root + * viewport (Application/Window frame), passed in so an implementation can + * validate the live rect's tap point against it without recomputing it. + * + * Returns `null` when the target cannot be positively confirmed on-screen + * (no stable id/label, not found, ambiguous match, not hittable, outside + * `rootViewport`, or any transport failure) — the guard MUST fail closed + * (refuse) on `null`. This is a rescue path only, never a way to relax a + * genuine refusal. Backends that do not support a direct, tree-independent + * read simply omit this method, which leaves today's refuse-on-off-screen + * behavior byte-for-byte unchanged. + */ + confirmOffscreenTargetVisible?( + context: BackendCommandContext, + node: Pick, + rootViewport: Rect | null, + ): Promise; tap?( context: BackendCommandContext, point: Point, diff --git a/src/commands/interaction/runtime/__tests__/test-utils/index.ts b/src/commands/interaction/runtime/__tests__/test-utils/index.ts index fb7b77ac92..04d8cca644 100644 --- a/src/commands/interaction/runtime/__tests__/test-utils/index.ts +++ b/src/commands/interaction/runtime/__tests__/test-utils/index.ts @@ -298,6 +298,7 @@ export function createInteractionDevice( | 'longPress' | 'scroll' | 'performGesture' + | 'confirmOffscreenTargetVisible' > > & { platform?: AgentDeviceBackend['platform']; @@ -327,6 +328,12 @@ export function createInteractionDevice( : undefined, scroll: overrides.scroll ? async (...args) => await overrides.scroll?.(...args) : undefined, performGesture: overrides.performGesture, + // #1542: undefined by default, so every existing test keeps proving the + // fail-closed refusal unchanged; only tests that opt in exercise the + // rescue/agreement paths. + confirmOffscreenTargetVisible: overrides.confirmOffscreenTargetVisible + ? async (...args) => (await overrides.confirmOffscreenTargetVisible?.(...args)) ?? null + : undefined, } satisfies AgentDeviceBackend, artifacts: createLocalArtifactAdapter(), sessions: createMemorySessionStore([ diff --git a/src/commands/interaction/runtime/offscreen-double-check.test.ts b/src/commands/interaction/runtime/offscreen-double-check.test.ts new file mode 100644 index 0000000000..c58d046bfe --- /dev/null +++ b/src/commands/interaction/runtime/offscreen-double-check.test.ts @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { Rect } from '@agent-device/kernel/snapshot'; +import { ref, selector } from './selector-read-utils.ts'; +import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; +import { createInteractionDevice } from './__tests__/test-utils/index.ts'; + +// #1542: end-to-end coverage for the off-screen refusal double-check, next to +// resolution.ts (AGENTS.md forbids adding to daemon/handlers/__tests__/ +// interaction.test.ts). Uses the same low-level runtime harness +// resolution.test.ts uses (createInteractionDevice), mocking +// `confirmOffscreenTargetVisible` directly rather than the local XCTest +// runner — the runner-level probe I/O is covered separately in +// src/daemon/__tests__/offscreen-target-probe.test.ts. + +// The bulk tree's ScrollView ancestor is corrupted (squeezed to a sliver), +// so the target's rect doesn't overlap it — the guard rejects even though +// the target's OWN bulk rect ({x:126,y:136}) happens to already be correct. +function keyboardSqueezedAncestorSnapshot() { + return makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'ScrollView', + label: 'Checkout form', + rect: { x: 18, y: 381, width: 366, height: 109 }, + }, + { + index: 2, + depth: 2, + parentIndex: 1, + type: 'Button', + label: 'Pickup', + identifier: 'shipping-pickup', + rect: { x: 126, y: 136, width: 75, height: 38 }, + hittable: true, + }, + ]); +} + +// The FROZEN-TREE manifestation: the whole bulk tree is pinned at +// pre-gesture values, so the target's OWN bulk rect is stale — a DIFFERENT +// place than where it actually is now. This is the shape #1542's live +// review flagged: a naive rescue that only checks "is it confirmed +// on-screen?" without also swapping in the live rect would tap the STALE +// bulk coordinate. +function frozenTreeStaleTargetSnapshot() { + return makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 402, height: 874 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + label: 'Pickup', + identifier: 'shipping-pickup', + // Stale pre-gesture position, off the bottom of the (frozen) viewport. + rect: { x: 20, y: 2000, width: 100, height: 40 }, + hittable: true, + }, + ]); +} + +const STALE_TARGET_REF = '@e2'; +// Where the runner's live probe says the element ACTUALLY is right now. +const LIVE_RECT: Rect = { x: 150, y: 300, width: 100, height: 40 }; +const LIVE_CENTER = { x: LIVE_RECT.x + LIVE_RECT.width / 2, y: LIVE_RECT.y + LIVE_RECT.height / 2 }; +const STALE_CENTER = { x: 70, y: 2020 }; // center of the stale bulk rect above + +test('rescue: click succeeds when confirmOffscreenTargetVisible confirms the target on-screen', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + confirmOffscreenTargetVisible: async (_context, node) => + node.identifier === 'shipping-pickup' ? { x: 126, y: 136, width: 75, height: 38 } : null, + }); + + const result = await device.interactions.click(selector('id="shipping-pickup"'), { + session: 'default', + }); + + assert.equal(result.kind, 'selector'); + // centerOfRect rounds: (126 + 75/2, 136 + 38/2) = (163.5, 155) -> (164, 155). + assert.deepEqual(calls, [{ x: 164, y: 155 }]); +}); + +test('FROZEN-TREE regression (#1542): a rescued tap lands at the LIVE rect, never the stale bulk one', async () => { + // Counterfactual: revert resolution.ts's resolveRefInteractionTarget / + // resolveSelectorInteractionTarget to compute the point from the + // pre-guard `node` instead of the guard's returned (possibly + // rescue-patched) node. This test goes red — `calls` records the STALE + // center ({x:70,y:2020}) instead of the LIVE one asserted below. + const calls: unknown[] = []; + const device = createInteractionDevice(frozenTreeStaleTargetSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + confirmOffscreenTargetVisible: async (_context, node) => + node.identifier === 'shipping-pickup' ? LIVE_RECT : null, + }); + + const result = await device.interactions.click(ref(STALE_TARGET_REF), { session: 'default' }); + + assert.equal(result.kind, 'ref'); + assert.deepEqual(calls, [LIVE_CENTER]); + assert.notDeepEqual(calls, [STALE_CENTER]); + // The response's own point must agree with what was actually dispatched. + assert.deepEqual(result.point, LIVE_CENTER); +}); + +test('genuine refusal: confirmOffscreenTargetVisible returns null -> still refuses, no tap', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + confirmOffscreenTargetVisible: async () => null, + }); + + await assert.rejects( + () => device.interactions.click(selector('id="shipping-pickup"'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /off-screen element/); + return true; + }, + ); + assert.equal(calls.length, 0); +}); + +test('no rescue hook wired (e.g. non-iOS backend) -> refuses exactly as before', async () => { + const calls: unknown[] = []; + const device = createInteractionDevice(keyboardSqueezedAncestorSnapshot(), { + tap: async (_context, point) => { + calls.push(point); + }, + // confirmOffscreenTargetVisible intentionally omitted. + }); + + await assert.rejects( + () => device.interactions.click(selector('id="shipping-pickup"'), { session: 'default' }), + /off-screen element/, + ); + assert.equal(calls.length, 0); +}); diff --git a/src/commands/interaction/runtime/resolution.test.ts b/src/commands/interaction/runtime/resolution.test.ts index dd1cc64288..9aa1ac96d3 100644 --- a/src/commands/interaction/runtime/resolution.test.ts +++ b/src/commands/interaction/runtime/resolution.test.ts @@ -3,7 +3,7 @@ import { test } from 'vitest'; import type { BackendSnapshotOptions } from '../../../backend.ts'; import { ref, selector } from './selector-read-utils.ts'; import { resolveActionableTouchResolution } from '../../../core/interaction-targeting.ts'; -import { tryResolveRefNode } from './resolution.ts'; +import { throwIfOffscreenInteractionTarget, tryResolveRefNode } from './resolution.ts'; import { parseSelectorChain, resolveSelectorChain } from '../../../selectors/index.ts'; import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; import type { Point } from '@agent-device/kernel/snapshot'; @@ -614,3 +614,93 @@ test('tryResolveRefNode discloses exact for a resolved ref and label-fallback fo assert.equal(tryResolveRefNode(nodes, '@e9', { fallbackLabel: '' }), null); }); + +// #1542: throwIfOffscreenInteractionTarget is exported for ADR 0011 registry +// honesty (interaction-guarantees.ts's `offscreen` cells point their `via` +// here); this direct-import test is its real consumer, mirroring +// tryResolveRefNode above. End-to-end rescue/refuse coverage through the +// public click/press surface lives in offscreen-double-check.test.ts. +function fakeOffscreenFailure() { + return { + message: 'off-screen', + details: { reason: 'test' }, + hint: () => 'scroll toward it', + }; +} + +test('throwIfOffscreenInteractionTarget: an on-screen node passes through unchanged', async () => { + const device = createInteractionDevice(makeSnapshotState([])); + const nodes = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + rect: { x: 20, y: 20, width: 40, height: 40 }, + }, + ]).nodes; + + const result = await throwIfOffscreenInteractionTarget( + device, + { session: 'default' }, + nodes[1]!, + nodes, + fakeOffscreenFailure(), + ); + + assert.equal(result, nodes[1]); +}); + +test('throwIfOffscreenInteractionTarget: off-screen + backend confirms -> returns the node patched with the LIVE rect', async () => { + const device = createInteractionDevice(makeSnapshotState([]), { + confirmOffscreenTargetVisible: async () => ({ x: 30, y: 30, width: 40, height: 40 }), + }); + const nodes = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + rect: { x: 20, y: 2000, width: 40, height: 40 }, + }, + ]).nodes; + + const result = await throwIfOffscreenInteractionTarget( + device, + { session: 'default' }, + nodes[1]!, + nodes, + fakeOffscreenFailure(), + ); + + assert.deepEqual(result.rect, { x: 30, y: 30, width: 40, height: 40 }); + assert.equal(result.index, nodes[1]!.index); +}); + +test('throwIfOffscreenInteractionTarget: off-screen + no rescue -> throws with the supplied failure shape', async () => { + const device = createInteractionDevice(makeSnapshotState([])); + const nodes = makeSnapshotState([ + { index: 0, depth: 0, type: 'Application', rect: { x: 0, y: 0, width: 400, height: 800 } }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'Button', + rect: { x: 20, y: 2000, width: 40, height: 40 }, + }, + ]).nodes; + + await assert.rejects( + () => + throwIfOffscreenInteractionTarget( + device, + { session: 'default' }, + nodes[1]!, + nodes, + fakeOffscreenFailure(), + ), + /off-screen/, + ); +}); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 3d91517dea..c473866fba 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -238,15 +238,23 @@ async function resolveRefInteractionTarget( }) : resolved.node; assertInteractionNotBlocked(node, `Ref ${target.ref}`, params.action); - assertVisibleRefTarget(node, capture.snapshot.nodes, target.ref, params.action); - const point = resolveNodeCenter(node, `Ref ${target.ref} not found or has invalid bounds`); + // #1542: point/response read from the returned (possibly rescue-patched) node. + const visibleNode = await assertVisibleRefTarget( + runtime, + options, + node, + capture.snapshot.nodes, + target.ref, + params.action, + ); + const point = resolveNodeCenter(visibleNode, `Ref ${target.ref} not found or has invalid bounds`); return { kind: 'ref', point, target: { kind: 'ref', ref: `@${resolved.ref}` }, ...describeResolvedInteractionNode( runtime, - node, + visibleNode, capture.snapshot.nodes, params.action, resolved.resolution, @@ -311,9 +319,17 @@ async function resolveSelectorInteractionTarget( }) : resolved.node; assertInteractionNotBlocked(node, `Selector ${resolved.selector.raw}`, params.action); - assertVisibleSelectorTarget(node, capture.snapshot.nodes, resolved.selector.raw, params.action); - const point = resolveNodeCenter( + // #1542: see the ref-target twin above. + const visibleNode = await assertVisibleSelectorTarget( + runtime, + options, node, + capture.snapshot.nodes, + resolved.selector.raw, + params.action, + ); + const point = resolveNodeCenter( + visibleNode, `Selector ${resolved.selector.raw} resolved to invalid bounds`, ); return { @@ -322,7 +338,7 @@ async function resolveSelectorInteractionTarget( target: { kind: 'selector', selector: resolved.selector.raw }, ...describeResolvedInteractionNode( runtime, - node, + visibleNode, capture.snapshot.nodes, params.action, buildSelectorResolutionDisclosure(resolved, capture.snapshot.nodes), @@ -690,13 +706,15 @@ function isUsableResolvedNode(node: SnapshotNode | null | undefined): node is Sn // resolving to a closed drawer/carousel item "succeeds" by tapping coordinates // outside the viewport (observed as `Tapped (-161, 265)` against Bluesky's // closed drawer) while the same node via @ref is refused. -function assertVisibleSelectorTarget( +async function assertVisibleSelectorTarget( + runtime: AgentDeviceRuntime, + options: CommandContext, node: SnapshotNode, nodes: SnapshotState['nodes'], selector: string, action: InteractionAction, -): void { - throwIfOffscreenInteractionTarget(node, nodes, { +): Promise { + return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, { message: `Selector ${selector} resolved to an off-screen element and is not safe to ${action}`, details: { reason: 'offscreen_selector', selector }, // A selector re-resolves against a fresh snapshot on every attempt, so the @@ -710,13 +728,15 @@ function assertVisibleSelectorTarget( }); } -function assertVisibleRefTarget( +async function assertVisibleRefTarget( + runtime: AgentDeviceRuntime, + options: CommandContext, node: SnapshotNode, nodes: SnapshotState['nodes'], refInput: string, action: InteractionAction, -): void { - throwIfOffscreenInteractionTarget(node, nodes, { +): Promise { + return await throwIfOffscreenInteractionTarget(runtime, options, node, nodes, { message: `Ref ${refInput} is off-screen and not safe to ${action}`, details: { reason: 'offscreen_ref', ref: normalizeRef(refInput) }, // The scroll that reveals the target expires the ref frame (#1366, ADR @@ -747,11 +767,14 @@ function scrollRevealClause(direction: OffscreenScrollDirection | null): string * `assertVisibleRefTarget`) ERROR with the runtime path's exact shapes, and * the non-hittable annotation is returned for the fast-path result. * - * Zero extra round trips by construction: no session, no stored snapshot, an - * unresolvable/invalid ref, or a node without a usable rect all make the - * preflight a no-op and the fast path proceeds exactly as before. Promotion - * to a hittable ancestor stays a runtime-path behavior — the preflight never - * changes which element the backend acts on. + * Zero extra round trips by construction on the accept path: no session, no + * stored snapshot, an unresolvable/invalid ref, or a node without a usable + * rect all make the preflight a no-op and the fast path proceeds exactly as + * before. Promotion to a hittable ancestor stays a runtime-path behavior — + * the preflight never changes which element the backend acts on. Exception: + * a would-be off-screen refusal may spend one extra iOS runner round trip + * (#1542's double-check) before erroring — cost only on the path that was + * about to fail anyway. */ export async function preflightNativeRefInteraction( runtime: AgentDeviceRuntime, @@ -772,12 +795,21 @@ export async function preflightNativeRefInteraction( }); if (!resolved) return {}; assertInteractionNotBlocked(resolved.node, `Ref ${target.ref}`, action); - assertVisibleRefTarget(resolved.node, nodes, target.ref, action); + // #1542: dispatches by REF, not coordinate, so no point to re-derive — but + // evidence/annotation below still describes the returned (visible) node. + const visibleNode = await assertVisibleRefTarget( + runtime, + options, + resolved.node, + nodes, + target.ref, + action, + ); return { - ...describeNonHittableTarget(resolved.node, action), + ...describeNonHittableTarget(visibleNode, action), // ADR 0012 decision 3: the guard lookup above doubles as the record-time // evidence source for the fast path, at zero extra capture cost. - node: resolved.node, + node: visibleNode, preActionNodes: nodes, }; } @@ -785,7 +817,25 @@ export async function preflightNativeRefInteraction( // isNodeVisibleOnScreen (not the effective-viewport form): items inside an // off-screen scrollable container (closed drawer) must also count as // off-screen, not just items scrolled out of an on-screen container. -function throwIfOffscreenInteractionTarget( +// +// #1542: once the bulk tree says off-screen, the guard gives iOS one chance +// to rescue a FALSE refusal via the optional backend.confirmOffscreenTargetVisible +// hook — a stale/corrupted bulk tree can say off-screen while the app is +// visually fine (zero cost on the accept path; runs only here). A confirmed +// rescue returns the node PATCHED WITH THE LIVE RECT: the caller must act on +// that returned node, never the original, because in the frozen-bulk-tree +// manifestation the original rect can be stale even when the rescue verdict +// is correct — tapping it would silently land at the wrong coordinate. The +// hook fails closed (null) on anything short of a positive confirmation, so +// a genuine refusal, or any backend without the hook, is unchanged. +// +// Exported (not just for callers here) for ADR 0011 registry honesty: +// interaction-guarantees.ts's `offscreen` cells point their `via` at this +// function, not at isNodeVisibleOnScreen alone, since this is the actual +// end-to-end enforcement point. +export async function throwIfOffscreenInteractionTarget( + runtime: AgentDeviceRuntime, + options: CommandContext, node: SnapshotNode, nodes: SnapshotState['nodes'], failure: { @@ -793,9 +843,16 @@ function throwIfOffscreenInteractionTarget( details: Record; hint: (direction: OffscreenScrollDirection | null) => string; }, -): void { +): Promise { const viewport = node.rect ? resolveEffectiveViewportRect(node, nodes) : null; - if (!node.rect || !viewport || isNodeVisibleOnScreen(node, nodes)) return; + if (!node.rect || !viewport || isNodeVisibleOnScreen(node, nodes)) return node; + const rootViewport = resolveViewportRect(nodes, node.rect); + const liveRect = await runtime.backend.confirmOffscreenTargetVisible?.( + toBackendContext(runtime, options), + node, + rootViewport, + ); + if (liveRect) return { ...node, rect: liveRect }; // The direction that scrolls this off-screen target into view. Named in the // hint (and surfaced as a machine-readable detail) so the recovery is a single // deterministic move instead of a guess (#1366). Derived from the same diff --git a/src/daemon/__tests__/direct-ios-selector.test.ts b/src/daemon/__tests__/direct-ios-selector.test.ts index a944933ec2..2c7d503ff8 100644 --- a/src/daemon/__tests__/direct-ios-selector.test.ts +++ b/src/daemon/__tests__/direct-ios-selector.test.ts @@ -1,7 +1,26 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { AppError } from '@agent-device/kernel/errors'; -import { isDirectIosSelectorFallbackError } from '../direct-ios-selector.ts'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import type { SessionState } from '../types.ts'; +import { + deriveDirectIosNodeSelector, + isDirectIosSelectorFallbackError, + isLocalIosRunnerSession, +} from '../direct-ios-selector.ts'; + +function makeSession( + platform: 'ios' | 'android' = 'ios', + overrides: Partial = {}, +): SessionState { + return { + name: platform, + device: platform === 'android' ? ANDROID_EMULATOR : IOS_SIMULATOR, + createdAt: Date.now(), + actions: [], + ...overrides, + }; +} test('runner ELEMENT_OFFSCREEN delegates normally but stays typed for Maestro replay', () => { const error = new AppError('ELEMENT_OFFSCREEN', 'element resolved off-screen at (-161, 265)'); @@ -65,3 +84,60 @@ test('transport-level COMMAND_FAILED errors fall back, semantic ones do not', () false, ); }); + +// #1542: isLocalIosRunnerSession is the ONE shared eligibility predicate for +// both the direct-selector tap fast path and the offscreen refusal +// double-check probe. Its two callers differ in exactly one parameter. + +test('isLocalIosRunnerSession: iOS local sessions are eligible, Android and undefined are not', () => { + assert.equal( + isLocalIosRunnerSession(makeSession('ios'), { skipPendingPostGestureStabilization: true }), + true, + ); + assert.equal( + isLocalIosRunnerSession(makeSession('android'), { + skipPendingPostGestureStabilization: true, + }), + false, + ); + assert.equal( + isLocalIosRunnerSession(undefined, { skipPendingPostGestureStabilization: true }), + false, + ); +}); + +test('isLocalIosRunnerSession: skipPendingPostGestureStabilization:true excludes a pending session (the tap fast path)', () => { + const pending = makeSession('ios', { + postGestureStabilization: { action: 'scroll', markedAt: Date.now() }, + }); + assert.equal( + isLocalIosRunnerSession(pending, { skipPendingPostGestureStabilization: true }), + false, + ); +}); + +test('isLocalIosRunnerSession: skipPendingPostGestureStabilization:false keeps a pending session eligible (the offscreen double-check)', () => { + const pending = makeSession('ios', { + postGestureStabilization: { action: 'scroll', markedAt: Date.now() }, + }); + assert.equal( + isLocalIosRunnerSession(pending, { skipPendingPostGestureStabilization: false }), + true, + ); +}); + +test('deriveDirectIosNodeSelector: prefers id, falls back to label, null when neither is usable', () => { + assert.deepEqual( + deriveDirectIosNodeSelector({ identifier: 'shipping-pickup', label: 'Pickup' }), + { + key: 'id', + value: 'shipping-pickup', + }, + ); + assert.deepEqual(deriveDirectIosNodeSelector({ label: 'Checkout form' }), { + key: 'label', + value: 'Checkout form', + }); + assert.equal(deriveDirectIosNodeSelector({ identifier: ' ', label: ' ' }), null); + assert.equal(deriveDirectIosNodeSelector({}), null); +}); diff --git a/src/daemon/__tests__/offscreen-target-probe.test.ts b/src/daemon/__tests__/offscreen-target-probe.test.ts new file mode 100644 index 0000000000..9c4981f439 --- /dev/null +++ b/src/daemon/__tests__/offscreen-target-probe.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import type { SessionState } from '../types.ts'; + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ + mockRunAppleRunnerCommand: vi.fn(), +})); + +vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; +}); + +import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts'; + +beforeEach(() => { + mockRunAppleRunnerCommand.mockReset(); +}); + +function makeSession(): SessionState { + return { name: 'default', device: IOS_SIMULATOR, createdAt: Date.now(), actions: [] }; +} + +const ROOT_VIEWPORT = { x: 0, y: 0, width: 402, height: 874 }; + +test('confirmIosOffscreenTargetVisible: returns the live rect when the runner confirms hittable + inside the viewport', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [{ index: 0, rect: { x: 126, y: 136, width: 75, height: 38 }, hittable: true }], + }); + + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { identifier: 'shipping-pickup' }, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.deepEqual(rect, { x: 126, y: 136, width: 75, height: 38 }); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1].selectorKey, 'id'); + assert.equal(mockRunAppleRunnerCommand.mock.calls[0]?.[1].selectorValue, 'shipping-pickup'); +}); + +test('confirmIosOffscreenTargetVisible: null when the node has no usable id/label (never calls the runner)', async () => { + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: {}, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.equal(rect, null); + assert.equal(mockRunAppleRunnerCommand.mock.calls.length, 0); +}); + +test('confirmIosOffscreenTargetVisible: null when the runner reports not found', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ found: false, nodes: [] }); + + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { label: 'Pickup' }, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.equal(rect, null); +}); + +test('confirmIosOffscreenTargetVisible: null on an ambiguous match / any runner error (fail closed)', async () => { + mockRunAppleRunnerCommand.mockRejectedValue( + new AppError('AMBIGUOUS_MATCH', 'selector matched multiple elements'), + ); + + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { label: 'Checkout form' }, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.equal(rect, null); +}); + +test('confirmIosOffscreenTargetVisible: null when the live read is not hittable, even with a plausible rect', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [{ index: 0, rect: { x: 126, y: 136, width: 75, height: 38 }, hittable: false }], + }); + + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { identifier: 'shipping-pickup' }, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.equal(rect, null); +}); + +test('confirmIosOffscreenTargetVisible: null when the live rect is hittable but outside the root viewport', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [{ index: 0, rect: { x: 5000, y: 5000, width: 75, height: 38 }, hittable: true }], + }); + + const rect = await confirmIosOffscreenTargetVisible({ + session: makeSession(), + node: { identifier: 'shipping-pickup' }, + rootViewport: ROOT_VIEWPORT, + requestOptions: {}, + }); + + assert.equal(rect, null); +}); diff --git a/src/daemon/__tests__/selector-runtime.test.ts b/src/daemon/__tests__/selector-runtime.test.ts new file mode 100644 index 0000000000..6bfac17973 --- /dev/null +++ b/src/daemon/__tests__/selector-runtime.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import type { SessionState } from '../types.ts'; + +const { mockRunAppleRunnerCommand } = vi.hoisted(() => ({ + mockRunAppleRunnerCommand: vi.fn(), +})); + +vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => { + const actual = + await importOriginal(); + return { ...actual, runAppleRunnerCommand: mockRunAppleRunnerCommand }; +}); + +import { queryDirectIosSelector } from '../selector-runtime.ts'; + +beforeEach(() => { + mockRunAppleRunnerCommand.mockReset(); +}); + +function makeSession(): SessionState { + return { name: 'default', device: IOS_SIMULATOR, createdAt: Date.now(), actions: [] }; +} + +// #1542: queryDirectIosSelector is the ONE querySelector client for the local +// XCTest runner — the offscreen refusal double-check probe +// (src/daemon/offscreen-target-probe.ts) reuses this exact function rather +// than opening a second client, so its request-shape and node-extraction +// contract is covered here, independent of any selector-runtime request. + +test('queryDirectIosSelector: builds the querySelector runner command from key/value/appBundleId', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ found: false, nodes: [] }); + const session = { ...makeSession(), appBundleId: 'com.example.demo' }; + + await queryDirectIosSelector(session, { key: 'id', value: 'submit-order' }, {}); + + const [device, command] = mockRunAppleRunnerCommand.mock.calls[0] ?? []; + assert.equal(device, session.device); + assert.deepEqual(command, { + command: 'querySelector', + selectorKey: 'id', + selectorValue: 'submit-order', + appBundleId: 'com.example.demo', + }); +}); + +test('queryDirectIosSelector: accepts a bare {key, value} selector — no `raw` field required', async () => { + // The offscreen double-check derives a selector from a node's own + // identifier/label (deriveDirectIosNodeSelector), which has no `raw` + // string — this must type/run without one. + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + nodes: [{ index: 0, rect: { x: 1, y: 2, width: 3, height: 4 }, hittable: true }], + }); + + const result = await queryDirectIosSelector(makeSession(), { key: 'label', value: 'Pickup' }, {}); + + assert.equal(result.found, true); + assert.deepEqual(result.node, { + index: 0, + rect: { x: 1, y: 2, width: 3, height: 4 }, + hittable: true, + }); +}); + +test('queryDirectIosSelector: found:false with no nodes reports not-found without a node', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ found: false, nodes: [] }); + + const result = await queryDirectIosSelector(makeSession(), { key: 'id', value: 'missing' }, {}); + + assert.equal(result.found, false); + assert.equal(result.node, undefined); +}); + +test('queryDirectIosSelector: surfaces text when the runner includes it', async () => { + mockRunAppleRunnerCommand.mockResolvedValue({ + found: true, + text: 'Ada Lovelace', + nodes: [{ index: 0 }], + }); + + const result = await queryDirectIosSelector( + makeSession(), + { key: 'id', value: 'field-name' }, + {}, + ); + + assert.equal(result.text, 'Ada Lovelace'); +}); + +test('queryDirectIosSelector: propagates a runner AMBIGUOUS_MATCH rather than swallowing it', async () => { + mockRunAppleRunnerCommand.mockRejectedValue( + new AppError('AMBIGUOUS_MATCH', 'selector matched multiple elements'), + ); + + await assert.rejects( + () => queryDirectIosSelector(makeSession(), { key: 'label', value: 'Checkout form' }, {}), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'AMBIGUOUS_MATCH'); + return true; + }, + ); +}); diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 0df9cd39d2..cd11c57822 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -1,4 +1,5 @@ import { isIosFamily } from '@agent-device/kernel/device'; +import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { isActiveProviderDevice } from '../provider-device-runtime.ts'; import type { SessionState } from './types.ts'; import { tryParseSelectorChain } from '../selectors/index.ts'; @@ -7,18 +8,44 @@ import type { ElementSelectorTapOptions } from '@agent-device/contracts/interact export type DirectIosSelectorTarget = ElementSelectorTapOptions & { raw: string }; +/** + * Is this session eligible for a direct, tree-independent local XCTest + * runner read/tap? Both the direct-selector tap fast path and the offscreen + * refusal double-check probe (`src/daemon/offscreen-target-probe.ts`) share + * this "local runner, not a provider-owned device" boundary — provider-owned + * iOS devices resolve through their own interactor-backed runtime instead. + * + * The one difference between the two callers is explicit, not baked in: the + * tap fast path skips itself while `session.postGestureStabilization` is + * pending (it hands off to the tree-based runtime path instead), but the + * double-check must NOT inherit that skip — it exists specifically to cover + * the window where the bulk AX tree is stale (pending or just-cleared + * stabilization), so excluding that window would defeat its purpose. + */ +export function isLocalIosRunnerSession( + session: SessionState | undefined, + options: { skipPendingPostGestureStabilization: boolean }, +): session is SessionState { + if (!session) return false; + if (!isIosFamily(session.device)) return false; + // This fast path talks directly to the local XCTest runner. Provider-owned + // iOS devices must resolve through their interactor-backed snapshot runtime + // instead, which keeps selectors and interaction guarantees on one backend. + if (isActiveProviderDevice(session.device)) return false; + if (options.skipPendingPostGestureStabilization && session.postGestureStabilization) { + return false; + } + return true; +} + export function readSimpleIosSelectorTarget(params: { session: SessionState | undefined; selectorExpression: string; }): DirectIosSelectorTarget | null { const { session, selectorExpression } = params; - if (!session) return null; - if (!isIosFamily(session.device)) return null; - // This fast path talks directly to the local XCTest runner. Provider-owned - // iOS devices must resolve through their interactor-backed snapshot runtime - // instead, which keeps selectors and interaction guarantees on one backend. - if (isActiveProviderDevice(session.device)) return null; - if (session.postGestureStabilization) return null; + if (!isLocalIosRunnerSession(session, { skipPendingPostGestureStabilization: true })) { + return null; + } const chain = tryParseSelectorChain(selectorExpression); if (!chain) return null; if (chain.selectors.length !== 1) return null; @@ -34,6 +61,22 @@ function isRunnerNativeSelectorKey(key: string): key is DirectIosSelectorTarget[ return key === 'id' || key === 'label' || key === 'text' || key === 'value'; } +/** + * The selector a bulk-tree node's OWN attributes give us for a direct runner + * re-read: prefer the stable `id` (accessibility identifier), fall back to + * `label`. Neither is guaranteed unique on the runner side — an ambiguous + * match is the caller's problem to fail closed on, not this parser's. + */ +export function deriveDirectIosNodeSelector( + node: Pick, +): { key: 'id' | 'label'; value: string } | null { + const identifier = node.identifier?.trim(); + if (identifier) return { key: 'id', value: identifier }; + const label = node.label?.trim(); + if (label) return { key: 'label', value: label }; + return null; +} + export function isDirectIosSelectorFallbackError( error: unknown, options: { diff --git a/src/daemon/handlers/interaction-runtime.ts b/src/daemon/handlers/interaction-runtime.ts index 6f4bc5d9e7..918a31db72 100644 --- a/src/daemon/handlers/interaction-runtime.ts +++ b/src/daemon/handlers/interaction-runtime.ts @@ -21,8 +21,11 @@ import { createDaemonRuntimeSessionStore } from '../runtime-session.ts'; import { resolveWebProvider, type WebProvider } from '../../platforms/web/provider.ts'; import { stripAtPrefix } from './interaction-touch-targets.ts'; import { NO_ACTIVE_SESSION_MESSAGE } from './response.ts'; -import type { Rect } from '@agent-device/kernel/snapshot'; +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; import { getRequestSignal } from '../../request/cancel.ts'; +import { buildAppleRunnerRequestOptions } from '../apple-runner-options.ts'; +import { isLocalIosRunnerSession } from '../direct-ios-selector.ts'; +import { confirmIosOffscreenTargetVisible } from '../offscreen-target-probe.ts'; type InteractionRuntimeParams = InteractionHandlerParams & { captureSnapshotForSession: CaptureSnapshotForSession; @@ -81,6 +84,27 @@ function createInteractionBackend( session.device, params.contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), )), + // #1542: iOS-only escape hatch for the off-screen refusal double-check. + // Local (non-provider) iOS sessions get a direct, AX-tree-independent + // probe (deliberately NOT skipped while postGestureStabilization is + // pending — see isLocalIosRunnerSession); every other platform/session + // omits this field, so the guard's decision stays exactly what it is + // today (fail closed). + confirmOffscreenTargetVisible: isLocalIosRunnerSession(session, { + skipPendingPostGestureStabilization: false, + }) + ? async (_context, node: Pick, rootViewport) => + await confirmIosOffscreenTargetVisible({ + session, + node, + rootViewport, + requestOptions: buildAppleRunnerRequestOptions({ + req, + logPath: params.logPath, + traceLogPath: session.trace?.outPath, + }), + }) + : undefined, tap: async (_context, point): Promise => { // ADR 0014 side-effect seam: the point is resolved; expire the ref frame // synchronously before dispatching so a later step cannot reuse it. diff --git a/src/daemon/offscreen-target-probe.ts b/src/daemon/offscreen-target-probe.ts new file mode 100644 index 0000000000..fe2132b573 --- /dev/null +++ b/src/daemon/offscreen-target-probe.ts @@ -0,0 +1,59 @@ +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; +import { isConfirmedOnScreenProbe } from '../snapshot/mobile-snapshot-semantics.ts'; +import { deriveDirectIosNodeSelector } from './direct-ios-selector.ts'; +import { queryDirectIosSelector } from './selector-runtime.ts'; +import type { AppleRunnerRequestOptions } from './apple-runner-options.ts'; +import type { SessionState } from './types.ts'; + +/** + * #1542 off-screen refusal double-check: the I/O side of + * `AgentDeviceBackend.confirmOffscreenTargetVisible`. Called ONLY at the + * moment the off-screen interaction guard (`src/commands/interaction/runtime/ + * resolution.ts`) is about to REFUSE a click/tap/gesture-target resolution on + * iOS, to re-confirm the target directly — bypassing whatever bulk + * accessibility tree the guard's verdict came from. + * + * Root cause this rescues: a keyboard-dismiss content-offset correction can + * leave a ScrollView's bulk AX frame squeezed to a stale value (or, in the + * frozen-tree manifestation, the WHOLE bulk tree pinned at pre-gesture + * values) while the actual on-screen state is fine. `queryDirectIosSelector` + * (reused from `selector-runtime.ts` — no second querySelector client) reads + * the target fresh, straight from XCTest, so it is not subject to either + * failure mode. + * + * Returns the element's LIVE rect (which the caller must use for the + * eventual tap point — the bulk-tree rect can be stale even when the + * decision to proceed is correct) only when the read positively confirms + * on-screen: unambiguously resolved, `hittable`, and its tap point falls + * inside `rootViewport`. Any other outcome — no usable id/label, not found, + * ambiguous match, transport error, not hittable, or outside the viewport — + * returns `null`, and the guard MUST fail closed (refuse) on `null`. This is + * a rescue path only, never a way to relax a genuine refusal. + * + * Session eligibility (local, non-provider iOS; postGestureStabilization NOT + * skipped — see `isLocalIosRunnerSession`) is the caller's job: this + * function assumes it is only ever wired up for an eligible session, same as + * every other optional `AgentDeviceBackend` method. + */ +export async function confirmIosOffscreenTargetVisible(params: { + session: SessionState; + node: Pick; + rootViewport: Rect | null; + requestOptions: AppleRunnerRequestOptions; +}): Promise { + const { session, node, rootViewport, requestOptions } = params; + const selector = deriveDirectIosNodeSelector(node); + if (!selector) return null; + let result: Awaited>; + try { + result = await queryDirectIosSelector(session, selector, requestOptions); + } catch { + // Ambiguous matches, ELEMENT_NOT_FOUND, and transport failures all land + // here — every one of them means "could not confirm on-screen," which + // must fail the double-check closed, not be classified further. + return null; + } + if (!result.found || !result.node?.rect) return null; + const probe = { rect: result.node.rect, hittable: result.node.hittable === true }; + return isConfirmedOnScreenProbe(probe, rootViewport) ? probe.rect : null; +} diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 65fe287b07..d30f3cfb72 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -3,7 +3,10 @@ import type { WaitParsed } from '../core/wait-positionals.ts'; import { AppError, asAppError, normalizeError } from '@agent-device/kernel/errors'; import type { SnapshotNode } from '@agent-device/kernel/snapshot'; import { runAppleRunnerCommand } from '../platforms/apple/core/runner/runner-client.ts'; -import { buildAppleRunnerRequestOptions } from './apple-runner-options.ts'; +import { + buildAppleRunnerRequestOptions, + type AppleRunnerRequestOptions, +} from './apple-runner-options.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from './types.ts'; import { errorResponse, requireCommandSupported } from './handlers/response.ts'; import { markSessionPartialRefsIssued, resolveRefStalenessWarning } from './session-snapshot.ts'; @@ -47,7 +50,7 @@ import { type SelectorRuntimeParams, } from './selector-runtime-backend.ts'; -type DirectIosSelectorQueryResult = { +export type DirectIosSelectorQueryResult = { found: boolean; text?: string; node?: SnapshotNode; @@ -466,10 +469,18 @@ async function resolveDirectIosSelectorQuery( return { session, selector, result }; } -async function queryDirectIosSelector( - params: SelectorRuntimeParams, +/** + * The single querySelector client for the local XCTest runner: a live, + * tree-independent read (and its found/text/node shape) for exactly one + * selector. Decoupled from `SelectorRuntimeParams` on purpose — the offscreen + * refusal double-check (`src/daemon/offscreen-target-probe.ts`) reuses this + * SAME function from a plain daemon session, not a selector-runtime request, + * so it must not depend on that request bag. + */ +export async function queryDirectIosSelector( session: SessionState, - selector: DirectIosSelectorTarget, + selector: Pick, + requestOptions: AppleRunnerRequestOptions, ): Promise { const data = await runAppleRunnerCommand( session.device, @@ -479,11 +490,7 @@ async function queryDirectIosSelector( selectorValue: selector.value, appBundleId: session.appBundleId, }, - buildAppleRunnerRequestOptions({ - req: params.req, - logPath: params.logPath, - traceLogPath: session.trace?.outPath, - }), + requestOptions, ); const found = data.found === true; const node = readDirectIosSelectorNode(data); @@ -500,7 +507,15 @@ async function queryDirectIosSelectorOrFallback( selector: DirectIosSelectorTarget, ): Promise { try { - return await queryDirectIosSelector(params, session, selector); + return await queryDirectIosSelector( + session, + selector, + buildAppleRunnerRequestOptions({ + req: params.req, + logPath: params.logPath, + traceLogPath: session.trace?.outPath, + }), + ); } catch (error) { if (isDirectIosSelectorFallbackError(error, { allowElementNotFound: true })) return null; return { kind: 'error', response: { ok: false, error: normalizeError(error) } }; diff --git a/src/snapshot/mobile-snapshot-semantics.ts b/src/snapshot/mobile-snapshot-semantics.ts index efa94323f9..e59859f40f 100644 --- a/src/snapshot/mobile-snapshot-semantics.ts +++ b/src/snapshot/mobile-snapshot-semantics.ts @@ -138,6 +138,27 @@ export function isTapPointInsideViewport(rect: Rect, viewport: Rect | null): boo return containsPoint(viewport, rect.x + rect.width / 2, rect.y + rect.height / 2); } +/** + * #1542: the pure geometry boundary the off-screen refusal double-check's + * direct probe (`src/daemon/offscreen-target-probe.ts`) reduces its decision + * to, once it has a fresh, tree-independent read of one element. A probe + * confirms the element genuinely on-screen only when BOTH hold: XCTest's own + * live hit-test says `hittable`, AND the tap point sits inside the root + * viewport (`isTapPointInsideViewport`, above). Either signal alone is + * insufficient — `hittable` with no viewport check could confirm an element + * that is technically tappable but whose reported rect drifted outside the + * app window; a viewport check with no `hittable` check could confirm an + * element occluded or clipped in a way geometry alone can't see. Kept pure + * (and separate from the network read) so it is unit-testable without a + * runner mock. + */ +export function isConfirmedOnScreenProbe( + probe: { rect: Rect; hittable: boolean }, + rootViewport: Rect | null, +): boolean { + return probe.hittable && isTapPointInsideViewport(probe.rect, rootViewport); +} + export function resolveEffectiveViewportRect( node: Pick, nodes: SnapshotNode[], diff --git a/src/utils/__tests__/mobile-snapshot-semantics.test.ts b/src/utils/__tests__/mobile-snapshot-semantics.test.ts index 38b9803b05..40ca84f6cd 100644 --- a/src/utils/__tests__/mobile-snapshot-semantics.test.ts +++ b/src/utils/__tests__/mobile-snapshot-semantics.test.ts @@ -3,9 +3,10 @@ import assert from 'node:assert/strict'; import { buildMobileSnapshotPresentation, classifyOffscreenScrollDirection, + isConfirmedOnScreenProbe, isNodeVisibleInEffectiveViewport, } from '../../snapshot/mobile-snapshot-semantics.ts'; -import type { SnapshotNode } from '@agent-device/kernel/snapshot'; +import type { Rect, SnapshotNode } from '@agent-device/kernel/snapshot'; test('mobile presentation keeps only visible nodes and adds off-screen summary fallback', () => { const nodes: SnapshotNode[] = [ @@ -562,3 +563,47 @@ test('mobile presentation does not let contradictory scroll indicator add hidden assert.equal(container?.hiddenContentAbove, true); assert.equal(container?.hiddenContentBelow, undefined); }); + +// #1542: isConfirmedOnScreenProbe is the pure geometry boundary the off-screen +// refusal double-check's direct probe (src/daemon/offscreen-target-probe.ts) +// reduces its confirm/don't-confirm decision to, once it has a fresh, +// tree-independent read of the target. Each branch below is proved with a +// counterfactual per docs/agents/testing.md — see the comment on each test +// for the one-line mutation that turns it red. + +const CONFIRM_PROBE_ROOT_VIEWPORT: Rect = { x: 0, y: 0, width: 400, height: 800 }; + +test('isConfirmedOnScreenProbe: hittable + inside the root viewport -> confirmed', () => { + const probe = { rect: { x: 100, y: 100, width: 50, height: 50 }, hittable: true }; + assert.equal(isConfirmedOnScreenProbe(probe, CONFIRM_PROBE_ROOT_VIEWPORT), true); +}); + +test('isConfirmedOnScreenProbe: inside the viewport but NOT hittable -> not confirmed', () => { + // Counterfactual ("ignore hittable"): change the function to + // `return isTapPointInsideViewport(probe.rect, rootViewport);`, dropping + // the hittable check entirely. This test goes red — it pins that a probe + // reporting a plausible rect but a live "not hittable" state (occluded, + // disabled, or otherwise not actually tappable) must not rescue. + const probe = { rect: { x: 100, y: 100, width: 50, height: 50 }, hittable: false }; + assert.equal(isConfirmedOnScreenProbe(probe, CONFIRM_PROBE_ROOT_VIEWPORT), false); +}); + +test('isConfirmedOnScreenProbe: hittable but OUTSIDE the root viewport -> not confirmed', () => { + // Counterfactual ("ignore viewport"): change the function to + // `return probe.hittable;`, dropping the viewport containment check + // entirely. This test goes red — it pins that a probe's live rect is still + // checked against the root viewport: hittable alone is not sufficient + // (the reported rect could still be nonsensical or off-window). + const probe = { rect: { x: 5000, y: 5000, width: 50, height: 50 }, hittable: true }; + assert.equal(isConfirmedOnScreenProbe(probe, CONFIRM_PROBE_ROOT_VIEWPORT), false); +}); + +test('isConfirmedOnScreenProbe: neither hittable nor inside the viewport -> not confirmed', () => { + const probe = { rect: { x: 5000, y: 5000, width: 50, height: 50 }, hittable: false }; + assert.equal(isConfirmedOnScreenProbe(probe, CONFIRM_PROBE_ROOT_VIEWPORT), false); +}); + +test('isConfirmedOnScreenProbe: a missing root viewport fails open on the geometry half (matches isTapPointInsideViewport)', () => { + const probe = { rect: { x: 5000, y: 5000, width: 50, height: 50 }, hittable: true }; + assert.equal(isConfirmedOnScreenProbe(probe, null), true); +});