From e76d9dc32aa12b55bff2d2104aa067d13a53302f Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 30 Jul 2026 19:22:17 -0400 Subject: [PATCH 1/3] feat: bind remote volatility comparison contract --- docs/REMOTE_FRAME_CONTRACT.md | 10 ++++ openadapt_flow/backends/factory.py | 3 ++ openadapt_flow/backends/rdp_backend.py | 5 ++ openadapt_flow/backends/remote_display.py | 13 ++++- openadapt_flow/deployment.py | 5 ++ openadapt_flow/remote_frame_contract.py | 66 +++++++++++++++++++++++ tests/test_remote_frame_contract.py | 57 ++++++++++++++++++++ 7 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 docs/REMOTE_FRAME_CONTRACT.md create mode 100644 openadapt_flow/remote_frame_contract.py create mode 100644 tests/test_remote_frame_contract.py diff --git a/docs/REMOTE_FRAME_CONTRACT.md b/docs/REMOTE_FRAME_CONTRACT.md new file mode 100644 index 00000000..5a05608c --- /dev/null +++ b/docs/REMOTE_FRAME_CONTRACT.md @@ -0,0 +1,10 @@ +# Remote frame contract + +`remote_frame_contract` is a versioned deployment field for remote RDP and +remote-display settling. It binds exact frame dimensions and reviewed volatile +rectangles. The runtime rejects a geometry change or an overlap with a declared +protected region. It retains raw frame bytes and raw lease hashes. It masks only +a transient derived input used for pointer-settle comparison. + +The current deployment schema can declare protected regions. Desktop editing +of these reviewed regions is a separate follow-up; no runtime learning occurs. diff --git a/openadapt_flow/backends/factory.py b/openadapt_flow/backends/factory.py index 991b38f9..4fe0a322 100644 --- a/openadapt_flow/backends/factory.py +++ b/openadapt_flow/backends/factory.py @@ -196,6 +196,7 @@ def _build_rdp_backend( application_version_marker=cfg.rdp_application_version_marker, environment_marker=cfg.rdp_environment_marker, session_marker=cfg.rdp_session_marker, + remote_frame_contract=cfg.remote_frame_contract, ) if rdp_transport is not None or has_host: @@ -220,6 +221,7 @@ def _build_rdp_backend( application_version_marker=cfg.rdp_application_version_marker, environment_marker=cfg.rdp_environment_marker, session_marker=cfg.rdp_session_marker, + remote_frame_contract=cfg.remote_frame_contract, ) if window_client is not None or has_window: @@ -236,6 +238,7 @@ def _build_rdp_backend( kwargs["application_version_marker"] = cfg.rdp_application_version_marker kwargs["environment_marker"] = cfg.rdp_environment_marker kwargs["session_marker"] = cfg.rdp_session_marker + kwargs["remote_frame_contract"] = cfg.remote_frame_contract return RemoteDisplayBackend(window_client, **kwargs) raise ValueError( diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index 05b6e7fb..5e7daf04 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -69,6 +69,7 @@ StructuralResolutionRefused, ) from openadapt_flow.ir import ActionDeliveryReceipt, Point +from openadapt_flow.remote_frame_contract import RemoteFrameContract from openadapt_flow.runtime.resolver import visual_resolution_point_fingerprint # What a transport may hand back as the current frame: a PIL image, or raw @@ -307,8 +308,10 @@ def __init__( session_marker: Optional[str] = None, session_marker_probe: Optional[Callable[[bytes], bool]] = None, session_identity_observer: Optional[Callable[[], Optional[str]]] = None, + remote_frame_contract: Optional["RemoteFrameContract"] = None, ) -> None: self._transport = transport + self._remote_frame_contract = remote_frame_contract self._viewport = viewport self._max_frame_age_s = float(max_frame_age_s) if self._max_frame_age_s <= 0: @@ -398,6 +401,8 @@ def screenshot(self) -> bytes: # and screenshot can never disagree. self._viewport = img.size png = self._png_bytes(img) + if self._remote_frame_contract is not None: + self._remote_frame_contract.require_geometry(img.size) self._last_frame_monotonic = time.monotonic() self._last_frame_digest = self._canonical_frame_digest(img) self._last_session_identity = self._session_identity_from_frame(png) diff --git a/openadapt_flow/backends/remote_display.py b/openadapt_flow/backends/remote_display.py index aeb79d2d..765054fd 100644 --- a/openadapt_flow/backends/remote_display.py +++ b/openadapt_flow/backends/remote_display.py @@ -73,6 +73,7 @@ StructuralResolutionRefused, ) from openadapt_flow.ir import ActionDeliveryReceipt +from openadapt_flow.remote_frame_contract import RemoteFrameContract from openadapt_flow.runtime.resolver import visual_resolution_point_fingerprint _PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" @@ -526,8 +527,10 @@ def __init__( session_marker: Optional[str] = None, session_marker_probe: Optional[Callable[[bytes], bool]] = None, session_identity_observer: Optional[Callable[[], Optional[str]]] = None, + remote_frame_contract: Optional["RemoteFrameContract"] = None, ) -> None: self._client = client if client is not None else _default_window_client() + self._remote_frame_contract = remote_frame_contract self._owner_substr = owner_substr self._title_substr = title_substr self._require_input_trust = require_input_trust @@ -589,6 +592,7 @@ def __init__( self._frame_window: Optional[WindowInfo] = None self._last_frame_monotonic: Optional[float] = None self._last_frame_digest: Optional[bytes] = None + self._last_comparison_digest: Optional[bytes] = None self._actuation_frame_png: Optional[bytes] = None self._last_session_identity: Optional[str] = None self._qualification_environment: Optional[tuple[str, str, str, str]] = None @@ -714,11 +718,18 @@ def screenshot(self) -> bytes: f"({scale_x:.4f}x vs {scale_y:.4f}y); refusing uncalibrated input" ) self._viewport = (w, h) + if self._remote_frame_contract is not None: + self._remote_frame_contract.require_geometry(self._viewport) self._scale_x, self._scale_y = scale_x, scale_y self._scale = scale_x # compatibility for existing diagnostics self._frame_window = win self._last_frame_monotonic = time.monotonic() self._last_frame_digest = _canonical_rgb_digest(png) + self._last_comparison_digest = ( + self._remote_frame_contract.comparison_digest(png) + if self._remote_frame_contract is not None + else self._last_frame_digest + ) self._last_session_identity = self._session_identity_from_frame(png) # An ordinary observation is not permission to perform a # consequential remote action. Only acquire_actuation_frame arms @@ -1291,7 +1302,7 @@ def _wait_for_pointer_settle(self) -> None: poll_s = max(0.01, self._settle_s) while time.monotonic() < deadline: self.screenshot() - digest = self._last_frame_digest + digest = self._last_comparison_digest if digest is not None and digest == previous_digest: stable_frames += 1 else: diff --git a/openadapt_flow/deployment.py b/openadapt_flow/deployment.py index 91c4ee2e..34dffa80 100644 --- a/openadapt_flow/deployment.py +++ b/openadapt_flow/deployment.py @@ -26,6 +26,8 @@ from pydantic import BaseModel, Field, field_validator, model_validator +from openadapt_flow.remote_frame_contract import RemoteFrameContract + # Import-light (pydantic only): the effect CONTRACT types double as the # declarative config vocabulary, so a deployment YAML binds run parameters # with the exact ``{param: ...}`` / ``{literal: ...}`` form bundles use. @@ -141,6 +143,9 @@ class BackendConfig(BaseModel): #: Optional exact case-insensitive title. Zero or multiple exact matches are #: refused; the backend never selects a largest partial match. rdp_window_title: Optional[str] = None + # Exact reviewed geometry for derived remote settle comparisons. The + # enclosing DeploymentConfig serialization binds this value at admission. + remote_frame_contract: Optional[RemoteFrameContract] = None class EffectsConfig(BaseModel): diff --git a/openadapt_flow/remote_frame_contract.py b/openadapt_flow/remote_frame_contract.py new file mode 100644 index 00000000..3272371d --- /dev/null +++ b/openadapt_flow/remote_frame_contract.py @@ -0,0 +1,66 @@ +"""Immutable, deployment-bound comparison masks for remote frame settling.""" + +from __future__ import annotations + +import hashlib +import io + +from PIL import Image, ImageDraw +from pydantic import BaseModel, ConfigDict, Field, model_validator + +Region = tuple[int, int, int, int] + + +class RemoteFrameContract(BaseModel): + """Reviewed exact-geometry exclusions for derived settle inputs only.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + schema_version: str = "openadapt.remote-frame-contract/v1" + frame_width: int = Field(gt=0, le=32768) + frame_height: int = Field(gt=0, le=32768) + volatile_regions: tuple[Region, ...] = Field(min_length=1, max_length=32) + protected_regions: tuple[Region, ...] = Field(default_factory=tuple, max_length=128) + + @model_validator(mode="after") + def validate_regions(self) -> "RemoteFrameContract": + for volatile in self.volatile_regions: + self._validate(volatile) + for protected in self.protected_regions: + self._validate(protected) + if _overlap(volatile, protected): + raise ValueError("volatile region overlaps a protected region") + return self + + def _validate(self, region: Region) -> None: + x, y, width, height = region + if ( + width <= 0 + or height <= 0 + or x < 0 + or y < 0 + or x + width > self.frame_width + or y + height > self.frame_height + ): + raise ValueError("remote frame region is outside the exact qualified frame") + + def require_geometry(self, size: tuple[int, int]) -> None: + if size != (self.frame_width, self.frame_height): + raise ValueError("remote frame contract geometry does not match live frame") + + def comparison_digest(self, png: bytes) -> bytes: + """Hash a derived masked copy. Raw evidence and leases stay unmasked.""" + image = Image.open(io.BytesIO(png)).convert("RGB") + self.require_geometry(image.size) + derived = image.copy() + draw = ImageDraw.Draw(derived) + for x, y, width, height in self.volatile_regions: + draw.rectangle((x, y, x + width - 1, y + height - 1), fill=(0, 0, 0)) + out = io.BytesIO() + derived.save(out, format="PNG") + return hashlib.sha256(out.getvalue()).digest() + + +def _overlap(left: Region, right: Region) -> bool: + x, y, width, height = left + a, b, c, d = right + return x < a + c and a < x + width and y < b + d and b < y + height diff --git a/tests/test_remote_frame_contract.py b/tests/test_remote_frame_contract.py new file mode 100644 index 00000000..bbcb80bf --- /dev/null +++ b/tests/test_remote_frame_contract.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import hashlib +import io + +import pytest +from PIL import Image, ImageDraw + +from openadapt_flow.remote_frame_contract import RemoteFrameContract + + +def _png(clock: int, *, target: int = 0, identity: int = 0) -> bytes: + image = Image.new("RGB", (100, 80), "white") + draw = ImageDraw.Draw(image) + draw.rectangle((80, 0, 99, 15), fill=(clock, 0, 0)) + draw.text((80, 0), str(clock), fill="white") + draw.rectangle((0, 20, 40, 60), fill=(target, 0, 0)) + draw.rectangle((45, 20, 75, 60), fill=(0, identity, 0)) + output = io.BytesIO() + image.save(output, format="PNG") + return output.getvalue() + + +def _contract() -> RemoteFrameContract: + return RemoteFrameContract( + frame_width=100, + frame_height=80, + volatile_regions=((80, 0, 20, 16),), + protected_regions=((0, 20, 40, 40), (45, 20, 30, 40)), + ) + + +def test_clock_only_change_matches_only_in_derived_contract_input() -> None: + contract = _contract() + first, second = _png(1), _png(2) + assert hashlib.sha256(first).digest() != hashlib.sha256(second).digest() + assert contract.comparison_digest(first) == contract.comparison_digest(second) + + +@pytest.mark.parametrize("field", ["target", "identity"]) +def test_protected_changes_do_not_match(field: str) -> None: + kwargs = {field: 200} + assert _contract().comparison_digest(_png(1)) != _contract().comparison_digest( + _png(2, **kwargs) + ) + + +def test_overlap_and_geometry_mismatch_fail_closed() -> None: + with pytest.raises(ValueError, match="overlaps"): + RemoteFrameContract( + frame_width=100, + frame_height=80, + volatile_regions=((0, 0, 10, 10),), + protected_regions=((0, 0, 10, 10),), + ) + with pytest.raises(ValueError, match="geometry"): + _contract().require_geometry((99, 80)) From 799674032562205133370eb0da2c0a02c88eebc1 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 30 Jul 2026 19:25:01 -0400 Subject: [PATCH 2/3] fix: bind remote mask to fresh protected regions --- openadapt_flow/backend.py | 9 +++++++ openadapt_flow/backends/rdp_backend.py | 6 +++++ openadapt_flow/backends/remote_display.py | 6 +++++ openadapt_flow/remote_frame_contract.py | 7 +++++ openadapt_flow/runtime/replayer.py | 31 +++++++++++++++++++++++ tests/test_remote_frame_contract.py | 6 +++++ 6 files changed, 65 insertions(+) diff --git a/openadapt_flow/backend.py b/openadapt_flow/backend.py index e38fff05..b17eadfb 100644 --- a/openadapt_flow/backend.py +++ b/openadapt_flow/backend.py @@ -537,6 +537,15 @@ def acquire_actuation_frame(self) -> bytes: ... +@runtime_checkable +class RemoteFrameContractBackend(Protocol): + """Optional pre-input protected-region binding for remote comparison masks.""" + + def arm_remote_frame_contract( + self, *, protected_regions: tuple[tuple[int, int, int, int], ...] + ) -> None: ... + + @runtime_checkable class FreshActuationReacquisitionBackend(Protocol): """Reset one proved zero-input invalidation for bounded reacquisition. diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index 5e7daf04..eb1ac9dc 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -449,6 +449,12 @@ def acquire_actuation_frame(self) -> bytes: self._actuation_lease_state = _LEASE_ARMED return png + def arm_remote_frame_contract( + self, *, protected_regions: tuple[tuple[int, int, int, int], ...] + ) -> None: + if self._remote_frame_contract is not None: + self._remote_frame_contract.arm(protected_regions) + def reset_fresh_actuation_state(self) -> None: """Reset only a typed zero-input content invalidation. diff --git a/openadapt_flow/backends/remote_display.py b/openadapt_flow/backends/remote_display.py index 765054fd..bb5530ff 100644 --- a/openadapt_flow/backends/remote_display.py +++ b/openadapt_flow/backends/remote_display.py @@ -809,6 +809,12 @@ def acquire_actuation_frame(self) -> bytes: self._actuation_frame_png = png return png + def arm_remote_frame_contract( + self, *, protected_regions: tuple[tuple[int, int, int, int], ...] + ) -> None: + if self._remote_frame_contract is not None: + self._remote_frame_contract.arm(protected_regions) + def reset_fresh_actuation_state(self) -> None: """Reset only a typed zero-input content invalidation. diff --git a/openadapt_flow/remote_frame_contract.py b/openadapt_flow/remote_frame_contract.py index 3272371d..31567434 100644 --- a/openadapt_flow/remote_frame_contract.py +++ b/openadapt_flow/remote_frame_contract.py @@ -47,6 +47,13 @@ def require_geometry(self, size: tuple[int, int]) -> None: if size != (self.frame_width, self.frame_height): raise ValueError("remote frame contract geometry does not match live frame") + def arm(self, protected_regions: tuple[Region, ...]) -> None: + """Refuse a newly observed target/identity/effect overlap before input.""" + for region in protected_regions: + self._validate(region) + if any(_overlap(region, volatile) for volatile in self.volatile_regions): + raise ValueError("volatile region overlaps a runtime protected region") + def comparison_digest(self, png: bytes) -> bytes: """Hash a derived masked copy. Raw evidence and leases stay unmasked.""" image = Image.open(io.BytesIO(png)).convert("RGB") diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 513e64dd..24e318f8 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -61,6 +61,7 @@ GuardedSelectOptionBackend, PreparedPointerActuationBackend, RemoteActuationBackend, + RemoteFrameContractBackend, RichPointerActionBackend, SelectOptionBackend, StructuralResolutionRefused, @@ -7921,6 +7922,36 @@ def _revalidate_consequential_actuation( self._cancel_guarded_keyboard() if error is not None and focused_element_backend: self._cancel_guarded_keyboard() + if error is None and isinstance(self.backend, RemoteFrameContractBackend): + protected: list[Region] = [] + if fresh_region is not None: + protected.append(fresh_region) + elif fresh_resolution is not None: + x, y = fresh_resolution.point + protected.append((x - 1, y - 1, 3, 3)) + protected.extend( + pc.region for pc in step.expect if pc.region is not None + ) + if workflow.qualification is not None: + policy = workflow.qualification.identity_policies.get(step.id) + if policy is not None: + protected.extend( + s.region for s in policy.signals if s.region is not None + ) + try: + self.backend.arm_remote_frame_contract( + protected_regions=tuple(protected) + ) + except Exception as exc: + return ( + fresh_resolution, + fresh_region, + fresh_png, + ( + "Actuation preflight HALTED because the remote frame mask " + f"overlaps protected evidence: {type(exc).__name__}" + ), + ) # Retain the exact observation that authorizes the next input edge. # Composite TYPE/SELECT_OPTION and retry paths can re-resolve inside # ``_act`` after the outer scope captured its initial geometry. A typed diff --git a/tests/test_remote_frame_contract.py b/tests/test_remote_frame_contract.py index bbcb80bf..87992401 100644 --- a/tests/test_remote_frame_contract.py +++ b/tests/test_remote_frame_contract.py @@ -55,3 +55,9 @@ def test_overlap_and_geometry_mismatch_fail_closed() -> None: ) with pytest.raises(ValueError, match="geometry"): _contract().require_geometry((99, 80)) + + +def test_runtime_target_or_identity_overlap_refuses_after_static_review() -> None: + contract = _contract() + with pytest.raises(ValueError, match="runtime protected"): + contract.arm(((80, 0, 10, 10),)) From e4182745f25a56da6ceda2763f67ff740e750b8a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Thu, 30 Jul 2026 21:24:42 -0400 Subject: [PATCH 3/3] fix: enforce qualified remote frame input gates --- docs/REMOTE_FRAME_CONTRACT.md | 3 +- openadapt_flow/backend.py | 4 +- openadapt_flow/backends/rdp_backend.py | 18 +++++- openadapt_flow/backends/remote_display.py | 17 ++++-- openadapt_flow/remote_frame_contract.py | 5 +- tests/test_rdp_backend.py | 62 +++++++++++++++++++++ tests/test_remote_display_backend.py | 68 +++++++++++++++++++++++ tests/test_remote_frame_contract.py | 4 ++ 8 files changed, 172 insertions(+), 9 deletions(-) diff --git a/docs/REMOTE_FRAME_CONTRACT.md b/docs/REMOTE_FRAME_CONTRACT.md index 5a05608c..1f1c55e1 100644 --- a/docs/REMOTE_FRAME_CONTRACT.md +++ b/docs/REMOTE_FRAME_CONTRACT.md @@ -4,7 +4,8 @@ remote-display settling. It binds exact frame dimensions and reviewed volatile rectangles. The runtime rejects a geometry change or an overlap with a declared protected region. It retains raw frame bytes and raw lease hashes. It masks only -a transient derived input used for pointer-settle comparison. +transient derived inputs used for pointer-settle and final pre-input content +comparisons. The current deployment schema can declare protected regions. Desktop editing of these reviewed regions is a separate follow-up; no runtime learning occurs. diff --git a/openadapt_flow/backend.py b/openadapt_flow/backend.py index b17eadfb..aa22e74c 100644 --- a/openadapt_flow/backend.py +++ b/openadapt_flow/backend.py @@ -528,7 +528,9 @@ class RemoteActuationBackend(Protocol): The backend owns a one-shot content lease for the returned frame. Its next input method captures once more under the backend input lock and refuses before the first input edge if the window/session, dimensions, readiness, - or exact frame content changed. The lease is consumed once so a + or exact frame content changed. A sealed remote frame contract can exclude + reviewed volatile regions from a derived comparison. Raw frame evidence + and the exact lease stay unmodified. The lease is consumed once so a multi-character type or double-click gesture cannot invalidate itself. """ diff --git a/openadapt_flow/backends/rdp_backend.py b/openadapt_flow/backends/rdp_backend.py index eb1ac9dc..6b725c39 100644 --- a/openadapt_flow/backends/rdp_backend.py +++ b/openadapt_flow/backends/rdp_backend.py @@ -360,6 +360,7 @@ def __init__( self._session_identity_observer = session_identity_observer self._last_frame_monotonic: Optional[float] = None self._last_frame_digest: Optional[bytes] = None + self._last_comparison_digest: Optional[bytes] = None self._last_session_identity: Optional[str] = None self._qualification_environment: Optional[tuple[str, str, str, str]] = None self._qualification_input_guard: Optional[Callable[[], None]] = None @@ -405,6 +406,11 @@ def screenshot(self) -> bytes: self._remote_frame_contract.require_geometry(img.size) self._last_frame_monotonic = time.monotonic() self._last_frame_digest = self._canonical_frame_digest(img) + self._last_comparison_digest = ( + self._remote_frame_contract.comparison_digest(png) + if self._remote_frame_contract is not None + else self._last_frame_digest + ) self._last_session_identity = self._session_identity_from_frame(png) if self._actuation_lease_state == _LEASE_ARMED: self._invalidate_actuation_lease() @@ -1113,8 +1119,16 @@ def _ensure_input_ready( "target resolution; refusing input" ) if self._actuation_lease_state == _LEASE_ARMED: - digest = self._canonical_frame_digest(current_img) - if self._last_frame_digest is None or digest != self._last_frame_digest: + raw_digest = self._canonical_frame_digest(current_img) + digest = ( + self._remote_frame_contract.comparison_digest(current_png) + if self._remote_frame_contract is not None + else raw_digest + ) + if ( + self._last_comparison_digest is None + or digest != self._last_comparison_digest + ): changed_pixel_count, changed_bbox = self._frame_difference( self._actuation_frame_png, current_img, diff --git a/openadapt_flow/backends/remote_display.py b/openadapt_flow/backends/remote_display.py index bb5530ff..68f16023 100644 --- a/openadapt_flow/backends/remote_display.py +++ b/openadapt_flow/backends/remote_display.py @@ -1298,8 +1298,9 @@ def _wait_for_pointer_settle(self) -> None: hover/cursor update reaches the remote framebuffer. Sampling one frame after a fixed sleep can therefore arm a lease on the old pixels and invalidate it moments later. This gate waits for consecutive, - byte-decoded RGB-identical frames; it never masks cursor regions or - weakens the exact post-resolution digest used at the delivery edge. + byte-decoded RGB-identical frames outside any reviewed volatile regions. + The contract applies only to derived comparisons. It never changes the + exact raw frame evidence or learns regions from ordinary runs. """ deadline = time.monotonic() + self._pointer_settle_timeout_s @@ -1439,8 +1440,16 @@ def _ensure_input_ready( # Consume once before the first input edge. A double click or # multi-character type is one gesture and must not invalidate # itself after its first state-changing edge. - digest = _canonical_rgb_digest(png) - if self._last_frame_digest is None or digest != self._last_frame_digest: + raw_digest = _canonical_rgb_digest(png) + digest = ( + self._remote_frame_contract.comparison_digest(png) + if self._remote_frame_contract is not None + else raw_digest + ) + if ( + self._last_comparison_digest is None + or digest != self._last_comparison_digest + ): changed_pixel_count, changed_bbox = self._frame_difference( self._actuation_frame_png, png, diff --git a/openadapt_flow/remote_frame_contract.py b/openadapt_flow/remote_frame_contract.py index 31567434..7cc21040 100644 --- a/openadapt_flow/remote_frame_contract.py +++ b/openadapt_flow/remote_frame_contract.py @@ -4,6 +4,7 @@ import hashlib import io +from typing import Literal from PIL import Image, ImageDraw from pydantic import BaseModel, ConfigDict, Field, model_validator @@ -15,7 +16,9 @@ class RemoteFrameContract(BaseModel): """Reviewed exact-geometry exclusions for derived settle inputs only.""" model_config = ConfigDict(extra="forbid", frozen=True) - schema_version: str = "openadapt.remote-frame-contract/v1" + schema_version: Literal["openadapt.remote-frame-contract/v1"] = ( + "openadapt.remote-frame-contract/v1" + ) frame_width: int = Field(gt=0, le=32768) frame_height: int = Field(gt=0, le=32768) volatile_regions: tuple[Region, ...] = Field(min_length=1, max_length=32) diff --git a/tests/test_rdp_backend.py b/tests/test_rdp_backend.py index 21c7234a..36aa1f3d 100644 --- a/tests/test_rdp_backend.py +++ b/tests/test_rdp_backend.py @@ -37,6 +37,7 @@ RDPTransport, normalize_chord, ) +from openadapt_flow.remote_frame_contract import RemoteFrameContract from openadapt_flow.runtime.resolver import visual_resolution_point_fingerprint VIEWPORT = (1280, 800) @@ -625,6 +626,67 @@ def test_bound_actuation_refuses_same_session_content_change_before_input() -> N assert raised.value.frame_size == VIEWPORT +def test_bound_actuation_allows_only_qualified_rdp_volatile_change() -> None: + transport = FakeRDPTransport(app_screens()) + contract = RemoteFrameContract( + frame_width=VIEWPORT[0], + frame_height=VIEWPORT[1], + volatile_regions=((0, 0, 32, 32),), + protected_regions=(BUTTON,), + ) + backend = FreeRDPBackend( + transport, + readiness_probe=lambda _png: True, + remote_frame_contract=contract, + ) + backend.acquire_actuation_frame() + backend.arm_remote_frame_contract(protected_regions=(BUTTON,)) + leased_raw_digest = backend._last_frame_digest + changed = transport.screens[0].copy() + changed.putpixel((1, 1), (0, 0, 0)) + transport.screens[0] = changed + + assert leased_raw_digest is not None + assert backend._canonical_frame_digest(changed) != leased_raw_digest + + backend.click(*BUTTON_CENTER) + + assert transport.pointer_events == [ + (*BUTTON_CENTER, "left", True), + (*BUTTON_CENTER, "left", False), + ] + assert backend._last_frame_digest == leased_raw_digest + + +def test_bound_actuation_refuses_rdp_change_outside_qualified_volatile_region() -> None: + transport = FakeRDPTransport(app_screens()) + contract = RemoteFrameContract( + frame_width=VIEWPORT[0], + frame_height=VIEWPORT[1], + volatile_regions=((0, 0, 32, 32),), + protected_regions=(BUTTON,), + ) + backend = FreeRDPBackend( + transport, + readiness_probe=lambda _png: True, + remote_frame_contract=contract, + ) + backend.acquire_actuation_frame() + backend.arm_remote_frame_contract(protected_regions=(BUTTON,)) + changed = transport.screens[0].copy() + changed.putpixel((50, 50), (0, 0, 0)) + transport.screens[0] = changed + + with pytest.raises(FreshActuationRequired) as raised: + backend.click(*BUTTON_CENTER) + + assert transport.pointer_events == [] + assert raised.value.operation == "rdp_click" + assert raised.value.changed_pixel_count == 1 + assert raised.value.changed_bbox == (50, 50, 1, 1) + assert raised.value.frame_size == VIEWPORT + + def test_bound_actuation_can_reset_only_a_typed_zero_edge_invalidation() -> None: transport = FakeRDPTransport(app_screens()) backend = FreeRDPBackend(transport, readiness_probe=lambda _png: True) diff --git a/tests/test_remote_display_backend.py b/tests/test_remote_display_backend.py index 88f15352..ea9b8c17 100644 --- a/tests/test_remote_display_backend.py +++ b/tests/test_remote_display_backend.py @@ -34,9 +34,11 @@ RemoteDisplayBackend, RemoteDisplayError, WindowInfo, + _canonical_rgb_digest, _split_chord, resolve_mac_key, ) +from openadapt_flow.remote_frame_contract import RemoteFrameContract from openadapt_flow.runtime.resolver import visual_resolution_point_fingerprint @@ -72,6 +74,7 @@ def __init__( self._key_window_id = key_window_id self._hit_window_id = hit_window_id self.frame_color = (11, 22, 33) + self.frame_overrides: dict[tuple[int, int], tuple[int, int, int]] = {} self.png_kwargs = {} self.calls: list[tuple] = [] @@ -99,6 +102,8 @@ def window_at_point(self, x, y): def capture(self, window_id): img = Image.new("RGB", self.px, self.frame_color) + for point, color in self.frame_overrides.items(): + img.putpixel(point, color) buf = io.BytesIO() img.save(buf, format="PNG", **self.png_kwargs) return buf.getvalue(), self.px[0], self.px[1] @@ -674,6 +679,69 @@ def test_bound_actuation_refuses_same_window_content_change_before_input() -> No assert raised.value.frame_size == client.px +def _remote_frame_contract_backend() -> tuple[RemoteDisplayBackend, FakeClient]: + size = (300, 200) + client = FakeClient( + window=WindowInfo( + window_id=1, + owner="Parallels Desktop", + title="Windows 11", + pid=99, + bounds=(0.0, 0.0, float(size[0]), float(size[1])), + on_screen=True, + ), + px=size, + ) + contract = RemoteFrameContract( + frame_width=size[0], + frame_height=size[1], + volatile_regions=((0, 0, 32, 32),), + protected_regions=((90, 90, 20, 20),), + ) + backend = RemoteDisplayBackend( + client=client, + settle_s=0.0, + readiness_probe=lambda _png: True, + remote_frame_contract=contract, + ) + backend.screenshot() + backend.prepare_pointer_actuation(100, 100) + backend.acquire_actuation_frame() + backend.arm_remote_frame_contract(protected_regions=((90, 90, 20, 20),)) + return backend, client + + +def test_bound_actuation_allows_only_qualified_remote_display_volatile_change() -> None: + backend, client = _remote_frame_contract_backend() + leased_raw_digest = backend._last_frame_digest + client.frame_overrides[(1, 1)] = (0, 0, 0) + current_png, _, _ = client.capture(client.window.window_id) + + assert leased_raw_digest is not None + assert _canonical_rgb_digest(current_png) != leased_raw_digest + + backend.click(100, 100) + + assert len([call for call in client.calls if call[0] == "mouse"]) == 2 + assert backend._last_frame_digest == leased_raw_digest + + +def test_bound_actuation_refuses_remote_display_change_outside_volatile_region() -> ( + None +): + backend, client = _remote_frame_contract_backend() + client.frame_overrides[(50, 50)] = (0, 0, 0) + + with pytest.raises(FreshActuationRequired) as raised: + backend.click(100, 100) + + assert not any(call[0] == "mouse" for call in client.calls) + assert raised.value.operation == "remote_click" + assert raised.value.changed_pixel_count == 1 + assert raised.value.changed_bbox == (50, 50, 1, 1) + assert raised.value.frame_size == client.px + + def test_replayer_reacquires_real_remote_display_lease_after_zero_edge_change( tmp_path, ) -> None: diff --git a/tests/test_remote_frame_contract.py b/tests/test_remote_frame_contract.py index 87992401..3564d6c6 100644 --- a/tests/test_remote_frame_contract.py +++ b/tests/test_remote_frame_contract.py @@ -55,6 +55,10 @@ def test_overlap_and_geometry_mismatch_fail_closed() -> None: ) with pytest.raises(ValueError, match="geometry"): _contract().require_geometry((99, 80)) + with pytest.raises(ValueError, match="schema_version"): + RemoteFrameContract.model_validate( + {**_contract().model_dump(), "schema_version": "unsupported/v1"} + ) def test_runtime_target_or_identity_overlap_refuses_after_static_review() -> None: