diff --git a/example/screen_to_browser.py b/example/screen_to_browser.py index 6ec8e5f..ff27afa 100644 --- a/example/screen_to_browser.py +++ b/example/screen_to_browser.py @@ -256,9 +256,14 @@ def on_stripe(frame): # Copy fallback: the frame already carries its native wire header. item_to_queue = {'data': bytes(frame), 'owner': None} - asyncio.run_coroutine_threadsafe( - client_queue.put(item_to_queue), g_loop - ) + try: + asyncio.run_coroutine_threadsafe( + client_queue.put(item_to_queue), g_loop + ) + except RuntimeError: + # Loop closed between the check and the call (teardown race); + # dropping the item's references recycles the frame buffer. + pass # --- 4. Register and Start Resources for this Client --- send_task = asyncio.create_task(send_stripes_task(websocket, client_queue)) diff --git a/pixelflux/Cargo.toml b/pixelflux/Cargo.toml index e52d899..76fac70 100644 --- a/pixelflux/Cargo.toml +++ b/pixelflux/Cargo.toml @@ -10,7 +10,11 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.29.0", features = ["extension-module"] } wayland-server = { version = "0.31.10", features = ["dlopen"] } -wayland-protocols = { version = "0.32", features = ["server"] } +wayland-protocols = { version = "0.32", features = ["server", "client", "staging"] } +wayland-client = "0.31" +wayland-protocols-misc = { version = "0.3", features = ["client"] } +wayland-protocols-wlr = { version = "0.3", features = ["client"] } +memmap2 = "0.9" crossbeam-channel = "0.5" calloop = "0.14" turbojpeg = "1.3" @@ -31,7 +35,7 @@ libloading = "0.9" xcursor = "0.3.1" # X11 host capture (replaces the C++ XShm/XFixes path): shm = XShm zero-copy grab, # xfixes = hardware cursor image. Pure-Rust XCB; links the system libxcb. -x11rb = { version = "0.13", features = ["shm", "xfixes"] } +x11rb = { version = "0.13", features = ["shm", "xfixes", "xtest", "randr"] } image = "=0.25.9" # The crate's build.rs probes the FFmpeg found via pkg-config and emits # per-version cfgs; one crate version supports FFmpeg 3.0 UP TO the release it diff --git a/pixelflux/src/computer_use.rs b/pixelflux/src/computer_use.rs index 98439f2..6755eca 100644 --- a/pixelflux/src/computer_use.rs +++ b/pixelflux/src/computer_use.rs @@ -8,19 +8,29 @@ //! //! Enabled by setting the `PIXELFLUX_CU` environment variable to the listen port. The server //! handles `POST /computer-use` requests for screenshots, mouse/keyboard injection, scrolling, -//! and cursor position queries — all targeting the Wayland compositor owned by the same process. +//! and cursor position queries. Actions run against a [`CuBackend`], resolved per request: the +//! Wayland compositor owned by this process when one is registered, otherwise the X server named +//! by `DISPLAY` (XTEST injection on a private connection, no active capture required). +//! +//! The same server also exposes the built-in MP4 recorder at `/record_start`, `/record_stop` +//! and `/record_status`, so a headless script can drive a session and record it over plain HTTP. +use std::collections::HashMap; use std::sync::mpsc; +use std::sync::{Mutex, OnceLock}; use std::thread; use std::time::Duration; use std::io::Cursor; +use smithay::input::keyboard::xkb; + use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; use image::{ImageBuffer, Rgba, ImageFormat}; use serde::Deserialize; use tiny_http; +use crate::wayland::keymap::keysym_for_char; use crate::ThreadCommand; fn clamp(v: T, lo: T, hi: T) -> T { @@ -41,60 +51,6 @@ pub fn encode_png_rgba(data: &[u8], width: u32, height: u32) -> Result, Ok(png) } -fn scancode_for_char(c: char) -> Option<(u32, bool)> { - Some(match c { - 'a' | 'A' => (38, c.is_uppercase()), - 'b' | 'B' => (56, c.is_uppercase()), - 'c' | 'C' => (54, c.is_uppercase()), - 'd' | 'D' => (40, c.is_uppercase()), - 'e' | 'E' => (26, c.is_uppercase()), - 'f' | 'F' => (41, c.is_uppercase()), - 'g' | 'G' => (42, c.is_uppercase()), - 'h' | 'H' => (43, c.is_uppercase()), - 'i' | 'I' => (31, c.is_uppercase()), - 'j' | 'J' => (44, c.is_uppercase()), - 'k' | 'K' => (45, c.is_uppercase()), - 'l' | 'L' => (46, c.is_uppercase()), - 'm' | 'M' => (58, c.is_uppercase()), - 'n' | 'N' => (57, c.is_uppercase()), - 'o' | 'O' => (32, c.is_uppercase()), - 'p' | 'P' => (33, c.is_uppercase()), - 'q' | 'Q' => (24, c.is_uppercase()), - 'r' | 'R' => (27, c.is_uppercase()), - 's' | 'S' => (39, c.is_uppercase()), - 't' | 'T' => (28, c.is_uppercase()), - 'u' | 'U' => (30, c.is_uppercase()), - 'v' | 'V' => (55, c.is_uppercase()), - 'w' | 'W' => (25, c.is_uppercase()), - 'x' | 'X' => (53, c.is_uppercase()), - 'y' | 'Y' => (29, c.is_uppercase()), - 'z' | 'Z' => (52, c.is_uppercase()), - '0' => (19, false), '1' => (10, false), '2' => (11, false), - '3' => (12, false), '4' => (13, false), '5' => (14, false), - '6' => (15, false), '7' => (16, false), '8' => (17, false), - '9' => (18, false), - '!' => (10, true), '@' => (11, true), '#' => (12, true), - '$' => (13, true), '%' => (14, true), '^' => (15, true), - '&' => (16, true), '*' => (17, true), '(' => (18, true), - ')' => (19, true), - '-' => (20, false), '_' => (20, true), - '=' => (21, false), '+' => (21, true), - '[' => (34, false), '{' => (34, true), - ']' => (35, false), '}' => (35, true), - ';' => (47, false), ':' => (47, true), - '\'' => (48, false), '"' => (48, true), - '`' => (49, false), '~' => (49, true), - '\\' => (51, false), '|' => (51, true), - ',' => (59, false), '<' => (59, true), - '.' => (60, false), '>' => (60, true), - '/' => (61, false), '?' => (61, true), - ' ' => (65, false), - '\t' => (23, false), - '\n' => (36, false), - _ => return None, - }) -} - fn scancode_for_keyname(name: &str) -> Option { Some(match name.to_lowercase().as_str() { "return" | "enter" => 36, @@ -147,62 +103,239 @@ fn scancode_for_keyname(name: &str) -> Option { }) } +/// Keysym for one component of an xdotool-style key spec that is not a named key: a single +/// literal character (`a`, `7`, `-`, `#`) via the Unicode mapping, or a keysym NAME +/// (`minus`, `bracketleft`, `udiaeresis`, …) via xkb's name lookup (case-tolerant fallback +/// so `Minus` still resolves). Returns 0 when the spec names no keysym. +fn keysym_for_key_spec(name: &str) -> u32 { + let mut chars = name.chars(); + if let (Some(c), None) = (chars.next(), chars.next()) { + return keysym_for_char(c); + } + let sym = xkb::keysym_from_name(name, xkb::KEYSYM_NO_FLAGS).raw(); + if sym != 0 { + return sym; + } + xkb::keysym_from_name(name, xkb::KEYSYM_CASE_INSENSITIVE).raw() +} + fn is_modifier(scancode: u32) -> bool { matches!(scancode, 37 | 105 | 50 | 62 | 64 | 108 | 133 | 134) } -fn send_key(tx: &smithay::reexports::calloop::channel::Sender, scancode: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::KeyboardKey { - scancode, - state: if pressed { 1 } else { 0 }, - }); +/// Semantic mouse-button vocabulary of the action layer; each backend maps it to its native +/// codes (evdev `BTN_` on Wayland, X core buttons 1/2/3 on X11). +#[derive(Clone, Copy)] +pub enum CuButton { + Left, + Right, + Middle, } -fn send_mouse_move(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerMotion { x, y }); +/// The desktop-side primitives one Computer Use action decomposes into. Keycodes are X/xkb +/// keycodes (evdev + 8) — the numbering both the smithay seat and XTEST consume — and +/// coordinates are framebuffer/root pixels, already clamped by the action layer. +pub trait CuBackend { + fn name(&self) -> &'static str; + fn fb_size(&self) -> Result<(i32, i32), String>; + /// One display's framebuffer size; 0 = the primary. Unknown ids are an error. The + /// X11 backend serves only the root (display 0); Wayland resolves any live output id. + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + if display == 0 { + self.fb_size() + } else { + Err(format!("Unknown display: {display}")) + } + } + fn key(&self, scancode: u32, pressed: bool); + fn mouse_move(&self, x: f64, y: f64); + fn button(&self, btn: CuButton, pressed: bool); + fn scroll(&self, dx: f64, dy: f64); + /// PNG of one display's framebuffer; 0 = the primary. Unknown ids are an error. + fn screenshot_png(&self, display: u32) -> Result, String>; + fn cursor_pos(&self) -> Result<(f64, f64), String>; + /// Run `seq` with every keysym in `keysyms` (which `resolve_keysyms` could not place) + /// made temporarily typeable, when the backend can arrange that; `seq` receives + /// keysym -> keycode for the transient bindings (empty when none could be arranged, + /// in which case those keysyms simply stay untypeable). Wayland's `resolve_keysyms` + /// already overlay-binds anything the base keymap lacks, so the default runs `seq` + /// with no bindings; the X11 backend overrides this with a grab-atomic spare-keycode + /// remap. + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + let _ = keysyms; + seq(&HashMap::new()); + } + /// Resolve keysyms against the backend's ACTIVE keymap: one `(keycode, shift level)` + /// per input keysym, `(0, 0)` where the keymap cannot produce it. Level bit 0 = Shift, + /// bit 1 = AltGr — the order xkb two/four-level key types use. Wayland resolves through + /// the compositor's keymap policy (overlay-binding what the base lacks); X11 through + /// the server's own `GetKeyboardMapping`. + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)>; + /// Keycode synthesized around AltGr-level hits (level bit 1). The default is the + /// pc105 right-Alt position the compositor's seat keymap binds to `ISO_Level3_Shift`; + /// the X11 backend overrides it with the level-3 modifier key found in the server's + /// keymap (0 = none, in which case `resolve_keysyms` reports no AltGr levels). + fn altgr_keycode(&self) -> u32 { + KC_ALTGR + } } const BTN_LEFT: u32 = 0x110; const BTN_RIGHT: u32 = 0x111; const BTN_MIDDLE: u32 = 0x112; -fn send_mouse_button(tx: &smithay::reexports::calloop::channel::Sender, btn: u32, pressed: bool) { - let _ = tx.send(ThreadCommand::PointerButton { - btn, - state: if pressed { 1 } else { 0 }, - }); +/// Reply deadline for compositor round-trips (screenshot readback, cursor position, fb size). +/// Bounded so a wedged render path turns into a JSON error instead of hanging the sequential +/// HTTP loop — and every request behind it — forever. +const REPLY_TIMEOUT: Duration = Duration::from_secs(5); + +/// Wayland implementation: every primitive is a `ThreadCommand` on the compositor's calloop +/// channel, so injection and readback serialize naturally with rendering and encoding. +pub struct CuWaylandBackend { + tx: smithay::reexports::calloop::channel::Sender, } -fn send_scroll(tx: &smithay::reexports::calloop::channel::Sender, x: f64, y: f64) { - let _ = tx.send(ThreadCommand::PointerAxis { x, y }); +impl CuBackend for CuWaylandBackend { + fn name(&self) -> &'static str { + "wayland" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + self.display_fb_size(0) + } + + fn display_fb_size(&self, display: u32) -> Result<(i32, i32), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuGetInfo { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request compositor info".to_string())?; + let (w, h, _) = resp_rx.recv_timeout(REPLY_TIMEOUT) + .map_err(|_| "Compositor info request failed".to_string())?; + // CuGetInfo reports zeros for output ids that don't exist (and for an output + // with no mode, which live outputs always have). + if w <= 0 || h <= 0 { + return Err(format!("Unknown display: {display}")); + } + Ok((w, h)) + } + + fn key(&self, scancode: u32, pressed: bool) { + let _ = self.tx.send(ThreadCommand::KeyboardKey { + scancode, + state: if pressed { 1 } else { 0 }, + }); + } + + fn mouse_move(&self, x: f64, y: f64) { + let _ = self.tx.send(ThreadCommand::PointerMotion { x, y }); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let btn = match btn { + CuButton::Left => BTN_LEFT, + CuButton::Right => BTN_RIGHT, + CuButton::Middle => BTN_MIDDLE, + }; + let _ = self.tx.send(ThreadCommand::PointerButton { + btn, + state: if pressed { 1 } else { 0 }, + }); + } + + fn scroll(&self, dx: f64, dy: f64) { + let _ = self.tx.send(ThreadCommand::PointerAxis { x: dx, y: dy }); + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuScreenshot { display_id: display, resp: resp_tx }) + .map_err(|_| "Failed to request screenshot".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Screenshot failed".to_string())? + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let (resp_tx, resp_rx) = mpsc::channel(); + self.tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) + .map_err(|_| "Failed to request cursor position".to_string())?; + resp_rx.recv_timeout(REPLY_TIMEOUT).map_err(|_| "Cursor position failed".to_string()) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let (resp_tx, resp_rx) = mpsc::channel(); + if self + .tx + .send(ThreadCommand::BindKeysyms { keysyms: keysyms.to_vec(), reply: resp_tx }) + .is_err() + { + return vec![(0, 0); keysyms.len()]; + } + resp_rx + .recv_timeout(REPLY_TIMEOUT) + .unwrap_or_else(|_| vec![(0, 0); keysyms.len()]) + } } -fn screenshot( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result, String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuScreenshot { resp: resp_tx }) - .map_err(|_| "Failed to request screenshot".to_string())?; - resp_rx.recv().map_err(|_| "Screenshot failed".to_string()) +/// Per-request literal-key resolver: batches keysym lookups against the backend's active +/// keymap and caches them for the request's burst of key events (a fresh backend — and thus +/// a fresh cache — is resolved per HTTP request, so a runtime layout switch is picked up by +/// the next request). +struct KeyResolver<'a> { + backend: &'a dyn CuBackend, + cache: HashMap, } -fn cursor_position( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(f64, f64), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuCursorPosition { resp: resp_tx }) - .map_err(|_| "Failed to request cursor position".to_string())?; - resp_rx.recv().map_err(|_| "Cursor position failed".to_string()) +impl<'a> KeyResolver<'a> { + fn new(backend: &'a dyn CuBackend) -> Self { + Self { backend, cache: HashMap::new() } + } + + /// Resolve a batch up front so a `type` action costs one backend round trip. + fn prefetch(&mut self, keysyms: &[u32]) { + let missing: Vec = { + let mut seen = std::collections::HashSet::new(); + keysyms + .iter() + .copied() + .filter(|&s| s != 0 && !self.cache.contains_key(&s) && seen.insert(s)) + .collect() + }; + if missing.is_empty() { + return; + } + let out = self.backend.resolve_keysyms(&missing); + for (sym, hit) in missing.into_iter().zip(out) { + self.cache.insert(sym, hit); + } + } + + /// `(keycode, shift level)` for `keysym`, or `None` when the active keymap cannot + /// produce it. + fn resolve(&mut self, keysym: u32) -> Option<(u32, u32)> { + if keysym == 0 { + return None; + } + self.prefetch(&[keysym]); + let hit = self.cache.get(&keysym).copied().unwrap_or((0, 0)); + (hit.0 != 0).then_some(hit) + } } -fn get_fb_size( - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result<(i32, i32), String> { - let (resp_tx, resp_rx) = mpsc::channel(); - tx.send(ThreadCommand::CuGetInfo { resp: resp_tx }) - .map_err(|_| "Failed to request compositor info".to_string())?; - let (w, h, _) = resp_rx.recv().map_err(|_| "Compositor info request failed".to_string())?; - Ok((w, h)) +/// Shift and default AltGr (pc105 right Alt / ISO_Level3_Shift) keycodes, held to reach a +/// resolved key's shift level: bit 0 = Shift, bit 1 = AltGr. +const KC_SHIFT: u32 = 50; +const KC_ALTGR: u32 = 108; + +/// The level modifiers `level` requires beyond whatever is already held (`held_mask` uses +/// the same bit layout), in press order. `altgr` is the backend's AltGr keycode +/// ([`CuBackend::altgr_keycode`]). +fn level_modifiers(level: u32, held_mask: u32, altgr: u32) -> Vec { + let mut out = Vec::new(); + if level & 1 != 0 && held_mask & 1 == 0 { + out.push(KC_SHIFT); + } + if level & 2 != 0 && held_mask & 2 == 0 && altgr != 0 { + out.push(altgr); + } + out } #[derive(Deserialize)] @@ -216,17 +349,17 @@ struct CuActionRequest { scroll_amount: Option, duration: Option, region: Option<[f64; 4]>, + /// Display id for `screenshot` / `zoom` (default 0, the primary; `record_start` uses + /// its own `display` field with the same meaning). Unknown ids get an error reply. + display: Option, } fn ok_json() -> String { "{\"result\":\"ok\"}".to_string() } -fn handle_action( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> String { - let result = handle_action_inner(req, tx); +fn handle_action(req: CuActionRequest, backend: &dyn CuBackend) -> String { + let result = handle_action_inner(req, backend); match result { Ok(response) => response, Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), @@ -236,20 +369,17 @@ fn handle_action( /// Turn one parsed Computer Use request into a real action on the captured desktop — the /// bridge that lets an external AI agent see and drive the session. /// -/// Every action ultimately becomes a compositor thread command or a framebuffer read, so this is -/// where the API's vocabulary meets the running compositor. Coordinates are clamped to the current +/// Every action ultimately becomes a backend input primitive or a framebuffer read, so this is +/// where the API's vocabulary meets the running desktop. Coordinates are clamped to the current /// framebuffer so a mistaken agent click cannot address off-screen pixels; keyboard and pointer /// actions translate into input events; and actions that must look at the screen (`screenshot`, /// `zoom`) request a fresh capture first, so the agent reasons about the current frame rather than /// a stale one. Returns the JSON response, or an error string explaining why the action could not /// run. -fn handle_action_inner( - req: CuActionRequest, - tx: &smithay::reexports::calloop::channel::Sender, -) -> Result { +fn handle_action_inner(req: CuActionRequest, b: &dyn CuBackend) -> Result { let sleep_ms = |ms: u64| thread::sleep(Duration::from_millis(ms)); - let (fb_w, fb_h) = get_fb_size(tx)?; + let (fb_w, fb_h) = b.fb_size()?; let handle_coord = |coord: [f64; 2]| -> (f64, f64) { ( @@ -265,7 +395,7 @@ fn handle_action_inner( match req.action.as_str() { "screenshot" => { - let png = screenshot(tx)?; + let png = b.screenshot_png(req.display.unwrap_or(0))?; let b64 = BASE64.encode(&png); Ok(format!("{{\"data\":\"{}\"}}", b64)) } @@ -273,34 +403,34 @@ fn handle_action_inner( "mouse_move" => { let coord = req.coordinate.ok_or("Missing coordinate")?; let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); Ok(ok_json()) } "left_click" | "right_click" | "middle_click" => { - let btn: u32 = match req.action.as_str() { - "left_click" => BTN_LEFT, - "right_click" => BTN_RIGHT, - _ => BTN_MIDDLE, + let btn = match req.action.as_str() { + "left_click" => CuButton::Left, + "right_click" => CuButton::Right, + _ => CuButton::Middle, }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } - send_mouse_button(tx, btn, true); + b.button(btn, true); sleep_ms(20); - send_mouse_button(tx, btn, false); + b.button(btn, false); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -310,25 +440,25 @@ fn handle_action_inner( let n = if req.action == "double_click" { 2 } else { 3 }; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } for _ in 0..n { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(10); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); sleep_ms(10); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) @@ -339,91 +469,166 @@ fn handle_action_inner( let end = req.coordinate.ok_or("Missing coordinate")?; let (sx, sy) = handle_coord(start); let (ex, ey) = handle_coord(end); - send_mouse_move(tx, sx, sy); + b.mouse_move(sx, sy); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); sleep_ms(30); - send_mouse_move(tx, ex, ey); + b.mouse_move(ex, ey); sleep_ms(30); - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "left_mouse_down" => { - send_mouse_button(tx, BTN_LEFT, true); + b.button(CuButton::Left, true); Ok(ok_json()) } "left_mouse_up" => { - send_mouse_button(tx, BTN_LEFT, false); + b.button(CuButton::Left, false); Ok(ok_json()) } "type" => { let text = req.text.as_deref().ok_or("Missing text")?; - for chunk in text.as_bytes().chunks(50) { - let chunk_str = std::str::from_utf8(chunk).map_err(|_| "Invalid UTF-8")?; - for ch in chunk_str.chars() { - if ch == '\n' { - send_key(tx, 36, true); - sleep_ms(10); - send_key(tx, 36, false); - sleep_ms(10); - continue; + // A nested app compositor owns the apps on its own socket; type there as + // a virtual-keyboard client, since keys on pixelflux's own seat carry an + // overlay keymap the inner compositor never sees. Falls through to the + // local seat if the app socket is unreachable. + if b.name() == "wayland" { + if let Some(sock) = app_wayland_socket_path() { + // Failures log once per socket value; every request still + // retries, so a compositor that comes back is used again + // immediately (and re-arms the logging). + static FAILED_SOCK: Mutex> = Mutex::new(None); + match crate::wayland::vkclient::type_text_to(&sock, text) { + Ok(()) => { + *FAILED_SOCK.lock().unwrap() = None; + return Ok(ok_json()); + } + Err(e) => { + let mut last = FAILED_SOCK.lock().unwrap(); + if last.as_deref() != Some(sock.as_str()) { + eprintln!( + "[ComputerUse] app-compositor type via {sock} failed ({e}); using local seat until it is reachable" + ); + *last = Some(sock); + } + } + } + } + } + let mut resolver = KeyResolver::new(b); + let syms: Vec = text.chars().map(keysym_for_char).collect(); + resolver.prefetch(&syms); + // Base+AltGr resolution stays the preferred path; only what the active keymap + // cannot reach at all goes through the backend's transient-bind fallback. + let mut unresolved: Vec = Vec::new(); + for &sym in &syms { + if sym != 0 && resolver.resolve(sym).is_none() && !unresolved.contains(&sym) { + unresolved.push(sym); + } + } + b.with_transient_keysyms(&unresolved, &mut |bound| { + for (i, &sym) in syms.iter().enumerate() { + if i > 0 && i % 50 == 0 { + sleep_ms(20); } - if let Some((sc, need_shift)) = scancode_for_char(ch) { - if need_shift { - send_key(tx, 50, true); + if let Some((sc, level)) = resolver.resolve(sym) { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); sleep_ms(5); - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); - send_key(tx, 50, false); - } else { - send_key(tx, sc, true); - sleep_ms(10); - send_key(tx, sc, false); } + b.key(sc, true); + sleep_ms(10); + b.key(sc, false); + for &m in level_mods.iter().rev() { + b.key(m, false); + } + sleep_ms(8); + } else if let Some(&kc) = bound.get(&sym) { + // Transient binds sit at the plain level: no modifiers needed. + b.key(kc, true); + sleep_ms(10); + b.key(kc, false); sleep_ms(8); } } - sleep_ms(20); - } + }); Ok(ok_json()) } "key" => { let text = req.text.as_deref().ok_or("Missing text")?; - let parts: Vec<&str> = text.split('+').collect(); + let mut resolver = KeyResolver::new(b); let mut mods: Vec = Vec::new(); - let mut main_key: Option = None; - for part in &parts { + let mut main_key: Option<(u32, u32)> = None; + // Keysym of the last main-key spec the active keymap could not place (cleared + // when a later part resolves): typed via the transient-bind fallback. + let mut unresolved_sym: Option = None; + for part in text.split('+') { let trimmed = part.trim(); if let Some(sc) = scancode_for_keyname(trimmed) { if is_modifier(sc) { mods.push(sc); } else { - main_key = Some(sc); + main_key = Some((sc, 0)); + unresolved_sym = None; + } + } else { + let sym = keysym_for_key_spec(trimmed); + if let Some(hit) = resolver.resolve(sym) { + main_key = Some(hit); + unresolved_sym = None; + } else if sym != 0 { + main_key = None; + unresolved_sym = Some(sym); } } } - for &sc in &mods { - send_key(tx, sc, true); - sleep_ms(10); - } - if let Some(sc) = main_key { - send_key(tx, sc, true); - sleep_ms(30); - send_key(tx, sc, false); - } else if let Some(&last_mod) = mods.last() { - send_key(tx, last_mod, true); - sleep_ms(30); - send_key(tx, last_mod, false); - } - for &sc in mods.iter().rev() { - sleep_ms(10); - send_key(tx, sc, false); - } + let unresolved: Vec = unresolved_sym.into_iter().collect(); + b.with_transient_keysyms(&unresolved, &mut |bound| { + let main_key = main_key.or_else(|| { + // Transient binds sit at the plain level: no modifiers needed. + unresolved_sym.and_then(|s| bound.get(&s)).map(|&kc| (kc, 0)) + }); + for &sc in &mods { + b.key(sc, true); + sleep_ms(10); + } + if let Some((sc, level)) = main_key { + // Modifiers reaching the key's shift level are added unless the spec + // already asked for them (Shift as 50/62, AltGr as 108 or the backend's + // resolved level-3 keycode). + let altgr = b.altgr_keycode(); + let held = mods.iter().fold(0u32, |m, &sc| match sc { + 50 | 62 => m | 1, + sc if sc == KC_ALTGR || sc == altgr => m | 2, + _ => m, + }); + let level_mods = level_modifiers(level, held, altgr); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); + sleep_ms(30); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } + } else if let Some(&last_mod) = mods.last() { + b.key(last_mod, true); + sleep_ms(30); + b.key(last_mod, false); + } + for &sc in mods.iter().rev() { + sleep_ms(10); + b.key(sc, false); + } + }); Ok(ok_json()) } @@ -431,10 +636,25 @@ fn handle_action_inner( let text = req.text.as_deref().ok_or("Missing text")?; let duration = req.duration.ok_or("Missing duration")?; let duration = duration.min(100.0); - if let Some(sc) = scancode_for_keyname(text) { - send_key(tx, sc, true); + let trimmed = text.trim(); + let mut resolver = KeyResolver::new(b); + let hit = match scancode_for_keyname(trimmed) { + Some(sc) => Some((sc, 0)), + None => resolver.resolve(keysym_for_key_spec(trimmed)), + }; + if let Some((sc, level)) = hit { + let level_mods = level_modifiers(level, 0, b.altgr_keycode()); + for &m in &level_mods { + b.key(m, true); + sleep_ms(10); + } + b.key(sc, true); sleep_ms((duration * 1000.0) as u64); - send_key(tx, sc, false); + b.key(sc, false); + for &m in level_mods.iter().rev() { + sleep_ms(10); + b.key(m, false); + } } Ok(ok_json()) } @@ -444,12 +664,12 @@ fn handle_action_inner( let amount = req.scroll_amount.unwrap_or(1).max(0) as f64; if let Some(coord) = req.coordinate { let (fx, fy) = handle_coord(coord); - send_mouse_move(tx, fx, fy); + b.mouse_move(fx, fy); sleep_ms(30); } if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { - send_key(tx, sc, true); + b.key(sc, true); sleep_ms(20); } } @@ -460,19 +680,19 @@ fn handle_action_inner( "right" => (amount, 0.0), _ => return Err(format!("Invalid scroll_direction: {}", dir)), }; - send_scroll(tx, dx, dy); + b.scroll(dx, dy); sleep_ms(30); if let Some(ref mod_name) = req.text { if let Some(sc) = handle_modifier(mod_name) { sleep_ms(10); - send_key(tx, sc, false); + b.key(sc, false); } } Ok(ok_json()) } "cursor_position" => { - let (x, y) = cursor_position(tx)?; + let (x, y) = b.cursor_pos()?; Ok(format!("{{\"text\":\"X={},Y={}\"}}", x.round() as i32, y.round() as i32)) } @@ -484,14 +704,18 @@ fn handle_action_inner( "zoom" => { let region = req.region.ok_or("Missing region")?; + let display = req.display.unwrap_or(0); + // The region clamps against the TARGET display's own framebuffer, which need + // not match the primary's. + let (dw, dh) = b.display_fb_size(display)?; let (x0, y0, x1, y1) = (region[0], region[1], region[2], region[3]); - let left = clamp(x0.round() as u32, 0, fb_w as u32 - 1); - let top = clamp(y0.round() as u32, 0, fb_h as u32 - 1); - let right = clamp(x1.round() as u32, left + 1, fb_w as u32); - let bottom = clamp(y1.round() as u32, top + 1, fb_h as u32); + let left = clamp(x0.round() as u32, 0, dw as u32 - 1); + let top = clamp(y0.round() as u32, 0, dh as u32 - 1); + let right = clamp(x1.round() as u32, left + 1, dw as u32); + let bottom = clamp(y1.round() as u32, top + 1, dh as u32); let crop_w = right - left; let crop_h = bottom - top; - let png = screenshot(tx)?; + let png = b.screenshot_png(display)?; if crop_w > 0 && crop_h > 0 { if let Ok(img) = image::load_from_memory(&png) { let cropped = img.crop_imm(left, top, crop_w, crop_h); @@ -510,25 +734,180 @@ fn handle_action_inner( } } +static WAYLAND_TX: Mutex>> = + Mutex::new(None); + +/// Body of `POST /record_start`. All fields are optional; unset ones fall back to the +/// `PIXELFLUX_RECORD_*` environment variables and built-in defaults. +#[derive(Deserialize, Default)] +struct RecordStartRequest { + /// Output MP4 path (default: `$PIXELFLUX_RECORD`, else `/tmp/pixelflux-record-.mp4`). + path: Option, + /// Wayland output id to record (default 0; ignored on X11). + display: Option, + /// Capture fps cap for a recorder-owned capture. + fps: Option, + /// Bitrate override in kbps for a recorder-owned capture. + bitrate_kbps: Option, +} + +/// Handle the recorder REST endpoints sharing the CU server: `record_start`, +/// `record_stop` and `record_status`, all one JSON round-trip into the same recorder +/// implementation the Python API and env vars use. Returns `None` for any other URL. +fn handle_record_endpoint(url: &str, body: &str) -> Option { + let reply = match url { + "/record_start" => { + let req: RecordStartRequest = if body.trim().is_empty() { + RecordStartRequest::default() + } else { + match serde_json::from_str(body) { + Ok(r) => r, + Err(e) => return Some(format!("{{\"error\":\"Invalid JSON: {}\"}}", e)), + } + }; + let path = req + .path + .or_else(|| std::env::var("PIXELFLUX_RECORD").ok().filter(|p| !p.is_empty())) + .unwrap_or_else(|| { + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("/tmp/pixelflux-record-{ts}.mp4") + }); + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(d) = req.display { + opts.display_id = d; + } + if let Some(f) = req.fps { + opts.fps = f; + } + if let Some(b) = req.bitrate_kbps { + opts.bitrate_kbps = b; + } + match crate::recorder::start(opts) { + Ok(s) => crate::recorder::status_to_json(&s).to_string(), + Err(e) => serde_json::json!({ "error": e }).to_string(), + } + } + "/record_stop" => match crate::recorder::stop() { + Ok(s) => { + let mut v = crate::recorder::status_to_json(&s); + v["stopped"] = serde_json::Value::Bool(true); + v.to_string() + } + Err(e) => serde_json::json!({ "error": e }).to_string(), + }, + "/record_status" => match crate::recorder::status() { + Some(s) => crate::recorder::status_to_json(&s).to_string(), + None => serde_json::json!({ "active": false }).to_string(), + }, + _ => return None, + }; + Some(reply) +} + +/// Socket of the compositor apps run under when it is nested under pixelflux +/// (labwc/kwin): keys injected into pixelflux's own seat carry an overlay keymap +/// the inner compositor never sees, so CU text is typed as a client of this +/// socket instead. Set by selkies over the ScreenCapture ABI. +static CU_APP_WAYLAND_DISPLAY: Mutex> = Mutex::new(None); + +/// Set (or clear, with None/empty) the app compositor socket for CU text injection. +pub fn set_app_wayland_display(display: Option) { + *CU_APP_WAYLAND_DISPLAY.lock().unwrap() = display.filter(|s| !s.is_empty()); +} + +/// Resolve the app compositor socket PATH for CU typing, or None to type on the +/// local seat. The ABI value selkies set wins; a standalone CU (no selkies) falls +/// back to PIXELFLUX_APP_WAYLAND_DISPLAY. A value naming pixelflux's own +/// compositor means nothing is nested. +fn app_wayland_socket_path() -> Option { + let name = CU_APP_WAYLAND_DISPLAY + .lock() + .unwrap() + .clone() + .or_else(|| std::env::var("PIXELFLUX_APP_WAYLAND_DISPLAY").ok()) + .filter(|s| !s.is_empty())?; + if crate::wait_socket_name(Duration::from_millis(0)).as_deref() == Some(name.as_str()) { + return None; + } + crate::wayland::wlclient::socket_path(&name) +} + +/// Make the Wayland compositor the preferred CU backend: once a live calloop sender is +/// registered, every subsequent request routes to it instead of an X11 connection. +pub fn register_wayland_backend(tx: smithay::reexports::calloop::channel::Sender) { + *WAYLAND_TX.lock().unwrap() = Some(tx); +} + +/// The registered compositor's command channel, if a Wayland compositor is running in this +/// process. The recorder uses it to attach to (or start) a capture without any Python client. +pub(crate) fn wayland_command_sender( +) -> Option> { + WAYLAND_TX.lock().unwrap().clone() +} + +/// Start the CU server if `PIXELFLUX_CU` names a bind (the standalone fallback; +/// a selkies-managed session passes the setting through [`start_cu_server`]). +pub fn spawn_cu_from_env() { + if let Ok(bind) = std::env::var("PIXELFLUX_CU") { + start_cu_server(&bind); + } +} + +/// Start the CU server on `bind`: a bare port listens on all interfaces (the +/// container-deployment default), `host:port` scopes it. Guarded so that exactly +/// one server binds per process no matter how many call sites (module import, +/// Wayland compositor init, the selkies setting) race to spawn it; backend +/// selection stays per-request, so a server bound at import serves a compositor +/// that only starts later. +pub fn start_cu_server(bind: &str) { + let addr = if bind.contains(':') { + bind.to_string() + } else { + match bind.parse::() { + Ok(port) => format!("0.0.0.0:{port}"), + Err(_) => { + println!("[ComputerUse] Invalid bind '{bind}' - expected a port or host:port"); + return; + } + } + }; + static SPAWNED: OnceLock<()> = OnceLock::new(); + let mut first = false; + SPAWNED.get_or_init(|| { + first = true; + }); + if first { + thread::spawn(move || run_cu_server(addr)); + } +} + +/// Pick the backend for one request: the registered Wayland compositor when present, +/// otherwise a fresh private connection to `DISPLAY`. The X11 connection is per-request so a +/// restarted X server never leaves the CU thread holding a dead connection. +fn resolve_backend() -> Result, String> { + if let Some(tx) = WAYLAND_TX.lock().unwrap().clone() { + return Ok(Box::new(CuWaylandBackend { tx })); + } + crate::x11::computer_use::CuX11Backend::connect() + .map(|be| Box::new(be) as Box) +} + /// Expose the captured desktop to an AI agent over HTTP, so a Computer Use client can drive /// the session much as a human viewer would. /// -/// It runs on its own thread listening on `0.0.0.0:` for POST `/computer-use` JSON actions -/// (screenshot, mouse_move, click, key, scroll, …). Framebuffer dimensions are re-queried from the -/// compositor on every request rather than cached, because the stream can start, stop, or resize -/// underneath the agent and a stale size would misplace every coordinate. A screenshot forces a -/// one-frame GPU readback when the pipeline is in zero-copy mode, since that path never otherwise -/// brings host pixels back to the CPU and the agent needs an image to look at. -pub fn run_cu_server( - tx: smithay::reexports::calloop::channel::Sender, - port: u16, -) { - println!( - "[ComputerUse] Server listening on port {}", - port, - ); - - let server = match tiny_http::Server::http(format!("0.0.0.0:{}", port)) { +/// It runs on its own thread listening on `addr` for POST `/computer-use` JSON actions +/// (screenshot, mouse_move, click, key, scroll, …). The backend and the framebuffer dimensions +/// are re-resolved on every request rather than cached, because the compositor can start, the +/// stream can resize, or the X server can restart underneath the agent — a stale size would +/// misplace every coordinate. On Wayland a screenshot forces a one-frame GPU readback when the +/// pipeline is in zero-copy mode; on X11 it is a one-shot `GetImage` of the root window. +pub fn run_cu_server(addr: String) { + println!("[ComputerUse] Server listening on {}", addr); + + let server = match tiny_http::Server::http(addr.as_str()) { Ok(s) => s, Err(e) => { eprintln!("[ComputerUse] Failed to start server: {}", e); @@ -536,6 +915,7 @@ pub fn run_cu_server( } }; + let mut last_backend = ""; for mut request in server.incoming_requests() { let mut body = String::new(); if let Err(e) = request.as_reader().read_to_string(&mut body) { @@ -549,6 +929,17 @@ pub fn run_cu_server( continue; } + if let Some(json) = handle_record_endpoint(request.url(), &body) { + let _ = request.respond( + tiny_http::Response::from_string(json) + .with_status_code(200) + .with_header( + "Content-Type: application/json".parse::().unwrap() + ) + ); + continue; + } + let parsed: CuActionRequest = match serde_json::from_str(&body) { Ok(r) => r, Err(e) => { @@ -563,7 +954,16 @@ pub fn run_cu_server( } }; - let json_response = handle_action(parsed, &tx); + let json_response = match resolve_backend() { + Ok(backend) => { + if backend.name() != last_backend { + last_backend = backend.name(); + println!("[ComputerUse] Using {} backend", last_backend); + } + handle_action(parsed, backend.as_ref()) + } + Err(e) => format!("{{\"error\":\"{}\"}}", e.replace('"', "\\\"")), + }; let _ = request.respond( tiny_http::Response::from_string(json_response) .with_status_code(200) diff --git a/pixelflux/src/encoders/nvenc.rs b/pixelflux/src/encoders/nvenc.rs index bf6a32c..a74c52a 100644 --- a/pixelflux/src/encoders/nvenc.rs +++ b/pixelflux/src/encoders/nvenc.rs @@ -1417,8 +1417,7 @@ impl NvencEncoder { /// — a `0x04` tag, a picture-type byte derived from the *actual* encoded `pictureType` /// (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than the `force_idr` request, the low 16 bits /// of the frame number, a zero field, and the width and height (all big-endian). - /// 4. **Emit and fan out**: append the encoded bytes, mirror them to the recording sink when one - /// is attached, unlock the bitstream, and return the framed buffer. + /// 4. **Emit**: append the encoded bytes, unlock the bitstream, and return the framed buffer. unsafe fn submit_frame( &mut self, mapped_buffer: NV_ENC_INPUT_PTR, @@ -1968,7 +1967,7 @@ mod gpu_tests { fn gpu_resolution_reconfigure_roundtrip() { let mut s = settings(1280, 720, 60.0); let t0 = std::time::Instant::now(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let init_ms = t0.elapsed().as_secs_f64() * 1000.0; let mut stream: Vec = Vec::new(); @@ -2049,7 +2048,7 @@ mod gpu_tests { let mut s = settings(1280, 720, 60.0); s.video_cbr_mode = true; s.video_bitrate_kbps = 4000; - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init"); let f720 = frame(1280, 720, 10); for i in 0..3u64 { enc.encode_cpu_argb(&f720, 1280 * 4, i, 25, i == 0) @@ -2082,7 +2081,7 @@ mod gpu_tests { } let s = settings(1920, 1080, 60.0); let before = used_mb(); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("init"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("init"); let f = frame(1920, 1080, 5); for i in 0..3u64 { enc.encode_cpu_argb(&f, 1920 * 4, i, 25, i == 0).expect("encode"); @@ -2097,7 +2096,7 @@ mod gpu_tests { #[ignore] fn gpu_init_above_default_headroom() { let s = settings(2160, 4096, 30.0); - let mut enc = NvencEncoder::new(&s, ptr::null(), None).expect("NVENC init portrait 4K"); + let mut enc = NvencEncoder::new(&s, ptr::null()).expect("NVENC init portrait 4K"); assert_eq!(enc.init_params.maxEncodeWidth, 4096); assert_eq!(enc.init_params.maxEncodeHeight, 4096); let f = frame(2160, 4096, 20); diff --git a/pixelflux/src/encoders/oh264.rs b/pixelflux/src/encoders/oh264.rs index 1d3cff6..4a59df2 100644 --- a/pixelflux/src/encoders/oh264.rs +++ b/pixelflux/src/encoders/oh264.rs @@ -134,6 +134,10 @@ impl Openh264Encoder { .skip_frames(true) .adaptive_quantization(false) .background_detection(false) + // Scene-change detection would inject IDRs the pipeline never asked for, + // breaking the infinite-GOP / on-demand-keyframe contract (x264 parity: + // `i_scenecut_threshold = 0`). + .scene_change_detect(false) .debug(settings.debug_logging) .intra_frame_period(IntraFramePeriod::from_num_frames(INFINITE_INTRA_PERIOD)) .num_threads(threads); @@ -338,16 +342,12 @@ impl Openh264Encoder { /// byte-for-byte the layout the NVENC/VAAPI/x264 full-frame paths emit, which is what lets a /// single client demuxer serve every encoder. Its type byte is read from the *actually /// encoded* picture type (IDR = `0x01`, I = `0x02`, P = `0x00`) rather than from `force_idr`, - /// because OpenH264's scene-change detection can emit an IDR the caller never asked for, and - /// the header must label a real decode entry point, not the request. It also carries the - /// frame number, a zero y-start, and the width/height (all big-endian). With - /// `omit_stripe_headers` the output is bare Annex-B. + /// because the encoder may re-type a frame, and the header must label a real decode entry + /// point, not the request. It also carries the frame number, a zero y-start, and the + /// width/height (all big-endian). With `omit_stripe_headers` the output is bare Annex-B. /// 5. **Empty payload**: if the encoder produced no bitstream (e.g. a skipped frame), an empty /// vec is returned rather than a lone header — a header with no Annex-B behind it would be a /// malformed frame to the client, and the pipeline reads empty as "nothing to send". - /// 6. **Recording sink**: the raw Annex-B payload (past the header) is also written to the - /// recording sink when one is attached, since a recorder wants the bare elementary stream, - /// not the wire framing. pub fn encode_host_argb( &mut self, argb: &[u8], @@ -459,7 +459,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let stride = 128 * 4; let idr = enc.encode_host_argb(&busy_frame(128, 96, 0), stride, 0, true, false).expect("encode idr"); assert!(idr.len() > 10, "IDR frame should produce output"); @@ -489,7 +489,7 @@ mod tests { omit_stripe_headers: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("openh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("openh264 init"); let out = enc.encode_host_argb(&busy_frame(128, 96, 0), 128 * 4, 0, true, false).expect("encode"); assert!( out.starts_with(&[0, 0, 0, 1]) || out.starts_with(&[0, 0, 1]), @@ -511,7 +511,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum() }; @@ -533,7 +533,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let high: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -557,7 +557,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let _ = e.encode_host_argb(&busy_frame(w, h, 0), stride, 0, true, false).unwrap(); let low: usize = (1..24).map(|t| e.encode_host_argb(&busy_frame(w, h, t), stride, t as u64, false, false).unwrap().len()).sum(); @@ -584,7 +584,7 @@ mod tests { target_fps: 30.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = e.encode_host_argb(&gradient_frame(w, h, 0), stride, 0, true, false).unwrap().len(); total += (1..24) .map(|t| e.encode_host_argb(&gradient_frame(w, h, t), stride, t as u64, false, false).unwrap().len()) @@ -614,7 +614,7 @@ mod tests { }; let frame = busy_frame(w, h, 0); for rgba in [false, true] { - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let out = e.encode_host_argb(&frame, stride, 0, true, rgba).expect("encode"); assert!(out.len() > 10, "rgba_input={rgba} must produce output"); assert_eq!(out[0], 0x04, "rgba_input={rgba} output must carry the wire header"); @@ -642,7 +642,7 @@ mod slice_tests { video_cbr_mode: true, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("encoder"); + let mut enc = Openh264Encoder::new(&s).expect("encoder"); let frame = vec![0x80u8; 640 * 480 * 4]; let out = enc.encode_host_argb(&frame, 640 * 4, 0, true, false).expect("encode"); let mut nals = 0; @@ -705,7 +705,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); let mut total = 0usize; let mut idrs = 0usize; for t in 0..60u64 { @@ -752,7 +752,7 @@ mod slice_tests { target_fps: 60.0, ..Default::default() }; - let mut e = Openh264Encoder::new(&s, None).unwrap(); + let mut e = Openh264Encoder::new(&s).unwrap(); unsafe { let mut p: openh264_sys2::SEncParamExt = std::mem::zeroed(); let ret = e.encoder.raw_api().get_option( @@ -798,7 +798,7 @@ mod rebuild_cost { ..Default::default() }; let t = std::time::Instant::now(); - let mut e = Openh264Encoder::new(&s, None).expect("init"); + let mut e = Openh264Encoder::new(&s).expect("init"); let init_ms = t.elapsed().as_secs_f64() * 1000.0; let frame = vec![128u8; 1920 * 1080 * 4]; let t = std::time::Instant::now(); diff --git a/pixelflux/src/encoders/overlay.rs b/pixelflux/src/encoders/overlay.rs index e4978f8..0fd7f1b 100644 --- a/pixelflux/src/encoders/overlay.rs +++ b/pixelflux/src/encoders/overlay.rs @@ -21,7 +21,7 @@ use smithay::{ ImportMem, Renderer, Texture, }, }, - utils::{Point, Transform, Physical}, + utils::{Point, Rectangle, Transform, Physical}, }; use std::path::Path; @@ -59,12 +59,14 @@ pub struct OverlayState { wm_height: u32, wm_pos_x: i32, wm_pos_y: i32, + wm_prev_pos: Option<(i32, i32)>, wm_velocity_x: f64, wm_velocity_y: f64, wm_subpixel_x: f64, wm_subpixel_y: f64, wm_loaded: bool, is_animated: bool, + wm_pixels: Vec, render_buffer: Option, } @@ -75,17 +77,40 @@ impl Default for OverlayState { wm_height: 0, wm_pos_x: 0, wm_pos_y: 0, + wm_prev_pos: None, wm_velocity_x: 2.0, wm_velocity_y: 2.0, wm_subpixel_x: 0.0, wm_subpixel_y: 0.0, wm_loaded: false, is_animated: false, + wm_pixels: Vec::new(), render_buffer: None, } } } +/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. +/// +/// Overlay pixels are overwhelmingly either fully opaque or fully transparent, and this runs per +/// pixel per frame on the CPU, so the two extremes are special-cased to skip the blend arithmetic +/// entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source (`a == 0`) +/// is left as-is, and only genuine edge pixels pay for the integer source-over. In every case only +/// the B / G / R bytes are written; the destination's alpha byte is left as the capture delivered it. +#[inline] +pub(crate) fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { + if a == 255 { + dst[0] = b; + dst[1] = g; + dst[2] = r; + } else if a > 0 { + let ia = 255 - a as u32; + dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; + dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; + dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; + } +} + impl OverlayState { /// Load the watermark image from disk; `output_scale` is the output's fractional /// scale, ceiled to the integer buffer scale of the upload. A failed load clears the overlay. @@ -97,20 +122,81 @@ impl OverlayState { self.wm_loaded = true; let buffer_scale = output_scale.ceil().max(1.0) as i32; + let pixels = rgba.into_vec(); self.render_buffer = Some(MemoryRenderBuffer::from_slice( - &rgba.into_vec(), + &pixels, Fourcc::Abgr8888, (self.wm_width as i32, self.wm_height as i32), buffer_scale, Transform::Normal, None, )); + self.wm_pixels = pixels; } else { self.wm_loaded = false; + self.wm_pixels = Vec::new(); self.render_buffer = None; } } + /// Alpha-blend the watermark into a BGRA frame (row `stride` in bytes) at its + /// current position. Clips per pixel at the frame bounds because the animated + /// position — or a watermark larger than the capture — can leave part of the + /// image off-frame, and only the in-frame portion may be written. + pub fn blend_bgra(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { + if !self.wm_loaded { + return; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + for y in 0..h { + let ty = self.wm_pos_y + y; + if ty < 0 || ty >= frame_h { + continue; + } + for x in 0..w { + let tx = self.wm_pos_x + x; + if tx < 0 || tx >= frame_w { + continue; + } + let src = ((y * w + x) * 4) as usize; + let (r, g, b, a) = ( + self.wm_pixels[src], + self.wm_pixels[src + 1], + self.wm_pixels[src + 2], + self.wm_pixels[src + 3], + ); + let off = ty as usize * stride + tx as usize * 4; + blend_pixel(&mut frame[off..off + 4], r, g, b, a); + } + } + } + + /// Frame-clipped union of the watermark's current and previous rectangles — + /// the region a damage-gated encoder must repaint after a bounce step moved + /// the image (the vacated area needs repainting as much as the new one). + pub fn damage_rect(&self, frame_w: i32, frame_h: i32) -> Option> { + if !self.wm_loaded { + return None; + } + let (w, h) = (self.wm_width as i32, self.wm_height as i32); + let (mut x0, mut y0) = (self.wm_pos_x, self.wm_pos_y); + let (mut x1, mut y1) = (x0 + w, y0 + h); + if let Some((px, py)) = self.wm_prev_pos { + x0 = x0.min(px); + y0 = y0.min(py); + x1 = x1.max(px + w); + y1 = y1.max(py + h); + } + x0 = x0.max(0); + y0 = y0.max(0); + x1 = x1.min(frame_w); + y1 = y1.min(frame_h); + if x1 <= x0 || y1 <= y0 { + return None; + } + Some(Rectangle::new((x0, y0).into(), (x1 - x0, y1 - y0).into())) + } + /// True once a watermark image has been loaded. pub fn is_active(&self) -> bool { self.wm_loaded @@ -135,6 +221,7 @@ impl OverlayState { let w = self.wm_width as i32; let h = self.wm_height as i32; + self.wm_prev_pos = Some((self.wm_pos_x, self.wm_pos_y)); self.is_animated = matches!(loc, WatermarkLocation::AN); match loc { @@ -211,12 +298,15 @@ impl OverlayState { } } - /// Render element for a software cursor `image` at `pos` (logical coords). + /// Render element for a software cursor `image` at logical `pos` on an output with + /// fractional `scale`; the position converts to physical here so the composited cursor + /// lands where the damage tracker and clients (which work in physical pixels) expect it. pub fn get_cursor_element( &self, renderer: &mut R, image: xcursor::parser::Image, - pos: Point, + pos: Point, + scale: f64, ) -> Option> where R: Renderer + ImportMem, @@ -233,7 +323,7 @@ impl OverlayState { let hot: Point = (image.xhot as i32, image.yhot as i32).into(); - let phys_pos: Point = (pos.x, pos.y).into(); + let phys_pos = pos.to_physical(smithay::utils::Scale::from(scale)).to_i32_round(); MemoryRenderBufferRenderElement::from_buffer( renderer, diff --git a/pixelflux/src/encoders/software.rs b/pixelflux/src/encoders/software.rs index 2bf9ed3..f25afd6 100644 --- a/pixelflux/src/encoders/software.rs +++ b/pixelflux/src/encoders/software.rs @@ -9,14 +9,14 @@ //! Frames are split into horizontal stripes processed in parallel via rayon. Each stripe is //! independently hashed against the previous frame for change detection, and only dirty stripes //! are encoded. The x264 path maintains per-stripe encoder state across frames for inter-prediction; -//! the JPEG path is stateless. An optional recording sink receives the H.264 NAL units for -//! muxing into a file or socket stream. +//! the JPEG path is stateless. use crate::RustCaptureSettings; use rayon::prelude::*; use smithay::utils::{Physical, Rectangle}; use std::ffi::CString; use std::ptr; +use std::sync::Arc; use yuv::{BufferStoreMut, YuvConversionMode, YuvPlanarImageMut, YuvRange, YuvStandardMatrix}; /// Upper bound on the horizontal stripes the CPU encoder splits a frame into, so the @@ -370,10 +370,9 @@ impl H264EncoderWrapper { /// /// The boolean return is load-bearing: `x264_encoder_encode` can legitimately produce nothing on /// a given call, and the caller must forward a stripe only when real bytes exist — never an empty - /// or header-only packet. The output also serves two consumers at once, which is why framing is - /// conditional: the transport needs the pipeline's small wire header to route the stripe, while - /// the optional recording sink needs the *bare* Annex-B elementary stream with no wire header so - /// it can be muxed directly. + /// or header-only packet. Framing is conditional because the transport needs the pipeline's small + /// wire header to route the stripe, while `omit_headers` consumers take the bare Annex-B + /// elementary stream. /// /// 1. **Picture setup**: wraps the borrowed Y/U/V planes and their strides in an /// `x264_picture_t` with the encoder's CSP, stamps the presentation timestamp with `frame_id`, @@ -386,9 +385,8 @@ impl H264EncoderWrapper { /// the client keys its decode-recovery on the frame type it truly received (IDR = `0x01`, /// I = `0x02`, else `0x00`), then the caller's `fixed_header` (frame number, y-start, width, /// height). With `omit_headers` the output is bare Annex-B. - /// 4. **Payload + recording**: every NAL payload is appended to `output_buf`, and each is also - /// forwarded to the recording sink when one is attached — giving the sink raw Annex-B without - /// the wire header. + /// 4. **Payload**: every NAL payload is appended to `output_buf` after the optional header, + /// so the bytes past the wire header are always a contiguous Annex-B access unit. #[allow(clippy::too_many_arguments)] pub fn encode_with_headers( &mut self, @@ -580,13 +578,14 @@ impl StripeState { /// /// # Fields /// -/// * `data` - Compressed payload (JPEG or H.264 NAL units). +/// * `data` - Compressed payload (JPEG or H.264 NAL units). `Arc`-shared so every +/// delivery-layer consumer can retain the frame without copying the bytes. /// * `data_type` - Codec tag: **1 = JPEG**, **2 = H.264**. /// * `stripe_y_start` - Y pixel coordinate of the stripe's top edge within the frame. /// * `stripe_height` - Height of the stripe in pixels. /// * `frame_id` - Frame sequence number this stripe belongs to. pub struct EncodedStripe { - pub data: Vec, + pub data: Arc>, pub data_type: i32, pub stripe_y_start: i32, pub stripe_height: i32, @@ -601,8 +600,7 @@ pub struct EncodedStripe { /// parallel stripes; bandwidth is precious, so unchanged stripes are skipped. Each stripe is /// independently hashed against the previous frame for change detection, and only dirty stripes /// are encoded. The x264 path maintains per-stripe encoder state across frames for -/// inter-prediction; the JPEG path is stateless. The recording sink is attached only in -/// single-stripe mode because several independent sub-frame bitstreams cannot be muxed. +/// inter-prediction; the JPEG path is stateless. /// /// # Arguments /// @@ -674,10 +672,10 @@ pub struct EncodedStripe { /// OpenH264 encoder — one fewer than the available cores, clamped to `[1, 4]`. The slice threads /// keep the in-frame encode latency inside the frame budget at high resolutions; the cap is four /// because `zerolatency` makes x264 slice-threaded and more than four slices trips decode -/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. Multiple stripes instead run across the -/// rayon pool with a single x264 thread and one conversion band each, since the parallelism there -/// already comes from encoding the stripes concurrently. The recording sink is attached only in single-stripe mode, because the several -/// independent sub-frame bitstreams of striped mode cannot be muxed into one recording. +/// glitches in some Chromium builds, and the minus-one leaves headroom for the capture thread. +/// Multiple stripes instead run across the rayon pool with a single x264 thread and one +/// conversion band each, since the parallelism there already comes from encoding the stripes +/// concurrently. #[allow(clippy::too_many_arguments)] pub fn encode_cpu( stripes: &mut Vec, @@ -932,7 +930,7 @@ pub fn encode_cpu( std::mem::take(&mut stripe_state.packet_buf) }; Some(EncodedStripe { - data, + data: Arc::new(data), data_type: 1, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -1025,7 +1023,7 @@ pub fn encode_cpu( &mut stripe_state.packet_buf, ) { Some(EncodedStripe { - data: std::mem::take(&mut stripe_state.packet_buf), + data: Arc::new(std::mem::take(&mut stripe_state.packet_buf)), data_type: 2, stripe_y_start: y_start as i32, stripe_height: actual_height as i32, @@ -1132,14 +1130,14 @@ mod tests { (w, h).into(), )]; let dirty = super::encode_cpu( - &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, None, false, + &mut stripes, &pixels, w, h, &full, &settings, 0, false, false, false, ); assert!(!dirty.is_empty(), "damaged frame must encode"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, None, false, + &mut stripes, &pixels, w, h, &[], &settings, frame, false, false, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once"); @@ -1177,14 +1175,14 @@ mod tests { }; let mut stripes = Vec::new(); let first = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, 0, false, true, false, ); assert!(!first.is_empty(), "first frame hashes as changed and encodes"); let mut fired_at = None; for frame in 1..=20u16 { let out = super::encode_cpu( - &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, None, false, + &mut stripes, &static_px, w, h, &[], &settings, frame, false, true, false, ); if !out.is_empty() { assert!(fired_at.is_none(), "paint-over must fire exactly once while static"); @@ -1194,7 +1192,7 @@ mod tests { assert_eq!(fired_at, Some(settings.paint_over_trigger_frames as u16)); let woke = super::encode_cpu( - &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, None, false, + &mut stripes, &changed_px, w, h, &[], &settings, 21, false, true, false, ); assert!(!woke.is_empty(), "content change after idle must encode"); } @@ -1313,7 +1311,7 @@ mod qp_bound_sweep { let mut out = Vec::new(); enc.encode_with_headers( &y, &u, &v, W as i32, (W / 2) as i32, (W / 2) as i32, - i as i64, i == 0, &[], true, &mut out, None, + i as i64, i == 0, &[], true, &mut out, ); out }) @@ -1336,7 +1334,7 @@ mod qp_bound_sweep { video_max_qp: max_qp, ..Default::default() }; - let mut enc = Openh264Encoder::new(&s, None).expect("oh264 init"); + let mut enc = Openh264Encoder::new(&s).expect("oh264 init"); (0..FRAMES) .map(|i| { let y = text_luma(i); diff --git a/pixelflux/src/encoders/vaapi.rs b/pixelflux/src/encoders/vaapi.rs index 2600e53..77e09bf 100644 --- a/pixelflux/src/encoders/vaapi.rs +++ b/pixelflux/src/encoders/vaapi.rs @@ -138,9 +138,8 @@ fn ff_err_str(err: i32) -> String { /// /// `current_qp` / `qp_hysteresis_counter` drive the CQP hysteresis in `update_qp`. `cbr_mode`, /// `current_bitrate_kbps`, `current_vbv_mult`, and `current_kf_s` cache the live rate-control state -/// so `reconfigure_rate` re-opens the codec only when a value actually changes. `recording_sink` is -/// the optional Unix-socket H.264 fan-out; `omit_stripe_headers` drops the 10-byte framing when the -/// consumer wants a bare Annex-B stream. +/// so `reconfigure_rate` re-opens the codec only when a value actually changes. +/// `omit_stripe_headers` drops the 10-byte framing when the consumer wants a bare Annex-B stream. pub struct VaapiEncoder { encoder_ctx: *mut ff::AVCodecContext, codec: *const ff::AVCodec, @@ -797,7 +796,7 @@ impl VaapiEncoder { /// described to the client as one full-height stripe: tag `0x04`, a keyframe flag (`0x01`/`0x00` /// from `AV_PKT_FLAG_KEY`), the frame number as a big-endian `u16`, a `0` y-start (a whole frame /// starts at the top), then width and height as big-endian `u16`s, followed by the raw Annex-B - /// payload. The recording sink intercepts frames at the delivery layer. + /// payload. /// `omit_stripe_headers` drops the header on the network path too, for a consumer that already /// wants raw Annex-B. The loop drains `avcodec_receive_packet` until the encoder is empty, /// unref'ing each packet before the next iteration. diff --git a/pixelflux/src/lib.rs b/pixelflux/src/lib.rs index 7ca8e2b..676c758 100644 --- a/pixelflux/src/lib.rs +++ b/pixelflux/src/lib.rs @@ -28,6 +28,7 @@ //! | [`x11`] | X11/XShm capture loop, XFixes out-of-band cursor monitor, and stripe dispatch | //! | [`pipeline`] | Frame-processing policy shared by both backends (send/QP/keyframe decisions) | //! | [`recording_sink`] | Unix-socket H.264 fan-out for external recording | +//! | [`recorder`] | Built-in MP4 recorder (fMP4 muxer + Python/env/REST control surfaces) | //! | [`computer_use`] | HTTP API for AI-agent desktop control (screenshots, input injection) | //! | [`nvgpufilter`] | Multi-GPU NVENC device filtering via ioctl | //! @@ -80,11 +81,13 @@ use smithay::{ element::{ memory::MemoryRenderBufferRenderElement, surface::WaylandSurfaceRenderElement, - AsRenderElements, Wrap, + AsRenderElements, Element, RenderElement, Wrap, }, gles::GlesRenderer, pixman::PixmanRenderer, - Bind, ImportAll, ImportEgl, ImportMem, + sync::SyncPoint, + Bind, ExportMem, Frame as _, ImportAll, ImportDma, ImportEgl, ImportMem, + Renderer as _, }, }, desktop::{space::SpaceRenderElements, Space}, @@ -96,7 +99,7 @@ use smithay::{ output::{Mode as OutputMode, Output, PhysicalProperties, Scale as OutputScale, Subpixel}, reexports::{ calloop::{ - channel::Event as CalloopEvent, generic::Generic, timer::{TimeoutAction, Timer}, + generic::Generic, timer::{TimeoutAction, Timer}, EventLoop, Interest, Mode, PostAction, }, pixman, @@ -113,7 +116,6 @@ use smithay::{ shell::xdg::XdgShellState, shm::ShmState, socket::ListeningSocketSource, - virtual_keyboard::VirtualKeyboardManagerState, pointer_warp::PointerWarpManager, relative_pointer::RelativePointerManagerState, pointer_constraints::PointerConstraintsState, @@ -173,6 +175,8 @@ pub mod encoders { pub mod wayland; /// Unix-socket H.264 recording fan-out for external capture tools. pub mod recording_sink; +/// Built-in MP4 recorder: independent capture-to-file with Python/env/REST control. +pub mod recorder; /// HTTP server implementing the Anthropic Computer Use spec for AI agent desktop control. pub mod computer_use; /// Frame-processing policy shared by the X11 and Wayland backends. @@ -226,7 +230,9 @@ use encoders::overlay::OverlayState; use encoders::software::MAX_STRIPE_CAPACITY; use encoders::vaapi::VaapiEncoder; -use wayland::cursor::Cursor; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1; + +use wayland::cursor::{Cursor, CursorJob}; use wayland::frontend::{AppState, ClientState, FocusTarget, GpuEncoder, next_serial, wayland_time, wayland_utime}; smithay::backend::renderer::element::render_elements! { @@ -244,7 +250,7 @@ smithay::backend::renderer::element::render_elements! { /// the buffer is described precisely enough (fd, stride, DRM modifier) for the importer to interpret /// it. One ARGB8888 plane is all that is carried because the compositor's offscreen target is exactly /// that single-plane format. -fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { +pub(crate) fn create_dmabuf_from_bo(bo: &BufferObject<()>) -> Dmabuf { let fd = bo.fd().expect("Failed to get FD from GBM BO"); let modifier = bo.modifier(); let stride = bo.stride(); @@ -304,6 +310,9 @@ pub struct RustCaptureSettings { pub debug_logging: bool, pub auto_adjust_screen_capture_size: bool, pub recording_socket: String, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode); + /// empty composites own clients as usual. + pub wayland_host_display: String, /// When true, encoders emit the raw payload without the per-stripe header byte block; /// stripe metadata is then carried only on the frame attributes. pub omit_stripe_headers: bool, @@ -314,7 +323,7 @@ pub struct RustCaptureSettings { /// infinite GOP, 3 when scheduled keyframes are enabled. pub video_vbv_multiplier: f64, /// Seconds between scheduled recovery keyframes; `<= 0` keeps the GOP infinite - /// (IDRs only on demand: client join / reset, recording cadence). + /// (IDRs only on demand: client join / reset, recorder connect). pub keyframe_interval_s: f64, /// Rate-controlled (CBR) QP clamp: `video_max_qp` bounds the quality FLOOR (screen text stays /// legible under motion at the cost of overshooting impossible targets), `video_min_qp` bounds @@ -407,6 +416,7 @@ impl Default for RustCaptureSettings { debug_logging: false, auto_adjust_screen_capture_size: false, recording_socket: String::new(), + wayland_host_display: String::new(), omit_stripe_headers: false, video_cbr_mode: false, video_bitrate_kbps: 4000, @@ -496,9 +506,11 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult().ok()) - .filter(|s| !s.is_empty()) - .or_else(|| std::env::var("PIXELFLUX_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) - .or_else(|| std::env::var("SELKIES_RECORDING_SOCKET").ok().filter(|s| !s.is_empty())) + .unwrap_or_default(), + wayland_host_display: settings + .getattr("wayland_host_display") + .ok() + .and_then(|v| v.extract::().ok()) .unwrap_or_default(), omit_stripe_headers: settings .getattr("omit_stripe_headers") @@ -537,20 +549,79 @@ pub(crate) fn extract_settings(settings: &Bound<'_, PyAny>) -> PyResult, RustCaptureSettings), - StopCapture, + /// Start (or in-place reconfigure) the capture bound to output `display_id`. + /// `callback` is the Python per-frame delivery target; `None` starts an internal + /// capture with no Python consumer (the built-in recorder taps the delivery layer). + StartCapture { display_id: u32, callback: Option>, settings: RustCaptureSettings }, + /// Stop the capture bound to output `display_id` (other displays keep running). + StopCapture { display_id: u32 }, + /// Create an additional output: `WxH` physical pixels at fractional `scale`, mapped + /// into the layout at offset `(x, y)`. Replies false when the id is taken/reserved or + /// the GPU render target cannot be allocated. + CreateOutput { + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + reply: std::sync::mpsc::Sender, + }, + /// Destroy a secondary output: its capture ends, its windows relocate to the primary + /// output. Replies false for the primary (id 0) or an unknown id. + DestroyOutput { id: u32, reply: std::sync::mpsc::Sender }, + /// Remap an existing output (the primary included) to layout offset `(x, y)`: the + /// Space mapping, the offsets used for absolute input injection and cursor + /// compositing, and the windows placed on it all follow, and the output is damaged so + /// the next frames render correctly. Replies false for an unknown id. + RepositionOutput { id: u32, x: i32, y: i32, reply: std::sync::mpsc::Sender }, + /// Reply with every live output as `(id, x, y, width, height, scale, capturing)`. + ListOutputs { reply: std::sync::mpsc::Sender> }, + /// Move the window with the given id onto output `output_id` (fullscreened there). + MoveWindowToOutput { window_id: u32, output_id: u32, reply: std::sync::mpsc::Sender }, + /// Reply with every mapped window as `(window_id, title, app_id, output_id)`. + ListWindows { reply: std::sync::mpsc::Sender> }, SetCursorCallback(Py), SetClipboardCallback(Py), /// Server-side clipboard offer: the compositor owns the selection and serves `data` as /// `mime` (plus text aliases) to pasting clients. SetClipboard { mime: String, data: Vec }, KeyboardKey { scancode: u32, state: u32 }, - /// Swap the seat keyboard's xkb keymap (XKB_KEYMAP_FORMAT_TEXT_V1 text); the caller owns - /// keysym-to-keycode policy and injects the keycodes it defined via `KeyboardKey`. + /// Set the seat's BASE keymap from a full XKB_KEYMAP_FORMAT_TEXT_V1 string. The + /// compositor's keymap policy rebuilds on top: overlay binds are re-spliced onto the new + /// base (same keycodes) and the combined keymap is applied in one swap. SetKeymapString(String), + /// Set the seat's BASE layout from RMLVO names (empty strings = xkbcommon defaults); + /// replies whether compilation succeeded. Overlay binds rebuild on top as for + /// `SetKeymapString`. + SetXkbLayout { + rules: String, + model: String, + layout: String, + variant: String, + options: String, + reply: std::sync::mpsc::Sender, + }, + /// Resolve keysyms to `(keycode, level)` against the seat keymap, overlay-binding every + /// keysym the base cannot produce — ONE keymap swap for the whole batch, and a keycode that + /// is currently pressed is never recycled. `(0, 0)` marks an unbindable keysym. + BindKeysyms { + keysyms: Vec, + reply: std::sync::mpsc::Sender>, + }, + /// Debug/verification readback: currently pressed xkb keycodes plus the modifier state + /// bitmask (1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5). + GetKeyboardState { + reply: std::sync::mpsc::Sender<(Vec, u32)>, + }, /// Reply with the smithay keyboard's keymap as an XKB_KEYMAP_FORMAT_TEXT_V1 string so a /// consumer (selkies) can build its reverse keysym map from the IDENTICAL keymap. GetXkbKeymap { reply: std::sync::mpsc::Sender }, + /// Ack once every previously queued command has been fully processed (the channel is + /// FIFO). The atexit sweep sends StopCapture + Barrier and waits, so the interpreter never + /// exits while the calloop thread is still mid-teardown (an NVENC/CUDA session drop racing + /// process exit segfaults). + Barrier { reply: std::sync::mpsc::Sender<()> }, PointerMotion { x: f64, y: f64 }, PointerRelativeMotion { dx: f64, dy: f64 }, /// `btn` is an evdev `BTN_` code by contract (e.g. 272 = BTN_LEFT, 273 = BTN_RIGHT, @@ -559,19 +630,29 @@ pub enum ThreadCommand { PointerButton { btn: u32, state: u32 }, PointerAxis { x: f64, y: f64 }, UpdateCursorConfig { render_on_framebuffer: bool }, - /// On-demand keyframe request (client reconnect / decoder reset): forces a send and an IDR - /// even on a static screen. - RequestIdr, - /// Live rate-control change for the Wayland calloop thread (parity with the X11 `rate_dirty` + /// Recreate the cursor theme handles at a new pixel size — the calloop's compositing + /// helper (the burned-in cursor) and, through its job channel, the `wl-cursor` worker's + /// (named-cursor PNG delivery). Replies false for a non-positive size. + SetCursorSize { size: i32, reply: std::sync::mpsc::Sender }, + /// On-demand keyframe request (client reconnect / decoder reset) for one display's + /// capture: forces a send and an IDR even on a static screen. + RequestIdr { display_id: u32 }, + /// Live rate-control change for one display's capture (parity with the X11 `rate_dirty` /// path). Each field is `None` when that dimension is unchanged. - UpdateRate { bitrate_kbps: Option, vbv_multiplier: Option, fps: Option }, - /// Live per-frame tunables (quality / paint-over / streaming / cursor), applied to the - /// calloop's settings and mirrored to the readback encode thread — no capture or encoder - /// restart. - UpdateTunables(LiveTunables), - CuScreenshot { resp: std::sync::mpsc::Sender> }, + UpdateRate { + display_id: u32, + bitrate_kbps: Option, + vbv_multiplier: Option, + fps: Option, + }, + /// Live per-frame tunables (quality / paint-over / streaming / cursor) for one + /// display's capture, mirrored to its readback encode thread — no restart. + UpdateTunables { display_id: u32, tunables: LiveTunables }, + /// One-shot PNG of one output's next rendered frame (0 = primary); an unknown + /// display id replies with an error immediately. + CuScreenshot { display_id: u32, resp: std::sync::mpsc::Sender, String>> }, CuCursorPosition { resp: std::sync::mpsc::Sender<(f64, f64)> }, - CuGetInfo { resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, + CuGetInfo { display_id: u32, resp: std::sync::mpsc::Sender<(i32, i32, f64)> }, } /// Read the kernel driver bound to a render node for encoder routing. @@ -950,6 +1031,8 @@ impl WlEncodeStats { /// flow through `controls`; damage and the IDR request arrive per-frame in `WlFrame`. struct WlEncodeConfig { settings: RustCaptureSettings, + /// Output/display id this encode loop serves; keys the recorder's delivery-layer tap. + display_id: u32, /// GLES readback is RGBA; the pixman framebuffer is BGRA. Selects CSC + encoder input kind. use_gpu: bool, /// Attempt a HW (NVENC/VAAPI) readback session before falling back to the CPU encoders. @@ -1118,7 +1201,14 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option = Vec::new(); if let Some(ref mut encoder) = video_encoder { @@ -1183,7 +1273,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1207,7 +1297,7 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option out.push(EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -1241,17 +1331,18 @@ fn wayland_encode_loop(pool: &WlFramePool, cfg: WlEncodeConfig) -> Option= 0 a device) -/// always wins, and only the unset `-2` sentinel is filled from the auto-picked render node. -/// H.264 output masks the dimensions even, because 4:2:0 needs even width and height. -/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this -/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback -/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its -/// encoders on the dedicated encode thread instead. A high-frequency timer composites the mapped -/// windows plus cursor and watermark, applies the shared paint-over / recovery-IDR policy, and -/// delivers the encoded stripes back to Python through the frame callback. The zero-copy encode -/// waits the GL render fence first, so a hardware encoder reading the dmabuf through CUDA/VA -/// never maps a half-rasterized (torn) frame. -/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its -/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on -/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the -/// delivery thread — it feeds the delivery sender and must be gone first — and the retained -/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing -/// interpreter. -fn run_wayland_thread( - command_rx: smithay::reexports::calloop::channel::Channel, - command_tx: smithay::reexports::calloop::channel::Sender, - initial_width: i32, - initial_height: i32, - explicit_dri_node: String, - auto_gpu_selected: bool, - cursor_size: i32, +/// Tear down a capture's encode/delivery threads and pools, returning any readback +/// hardware session for in-place reuse by a following start. The encode thread is joined +/// before the delivery sender drops (it feeds that sender), and the zero-copy session (if +/// any) is left on the capture for the caller to reuse or drop. +fn teardown_capture(cap: &mut wayland::frontend::WlCapture) -> Option { + if let Some(p) = cap.encode_pool.take() { + p.shutdown(); + } + let prior = cap.encode_join.take().and_then(|j| j.join().ok()).flatten(); + if let Some(tx) = cap.deliver_tx.take() { + drop(tx); + } + if let Some(j) = cap.deliver_join.take() { + let _ = j.join(); + } + cap.pending_hw_delivery = None; + cap.pending_hw_damage = false; + cap.recording_sink = None; + prior +} + +/// Stop the capture bound to `display_id`, leaving the output (and every other display's +/// capture) running. +fn stop_capture_on_display(state: &mut AppState, display_id: u32) { + let Some(idx) = state.node_idx_for_id(display_id) else { return }; + if let Some(mut cap) = state.output_nodes[idx].capture.take() { + println!("[Wayland] Capture loop stopped (display {display_id})."); + cap.video_encoder = None; + let _ = teardown_capture(&mut cap); + } + wayland_alive().lock().unwrap().remove(&display_id); +} + +/// Start (or in-place reconfigure) the capture bound to output `display_id`: reprogram the +/// output's mode/scale/refresh, size the render targets, fullscreen the display's windows at +/// the new logical size, resolve the encode path (zero-copy vs readback), and spawn the +/// delivery (and readback-mode encode) threads. The single-display behavior of the former +/// global StartCapture is preserved exactly for display 0. +fn start_capture_on_display( + state: &mut AppState, + display_id: u32, + cb: Option>, + mut settings: RustCaptureSettings, ) { - let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; - let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + use smithay::wayland::fractional_scale::with_fractional_scale; + + // Host-capture mode: connect on first use and point this display's capture + // thread at the requested size. The compositor keeps running (CU, clipboard + // callbacks, input fallbacks) but its renderer is bypassed. + if !settings.wayland_host_display.is_empty() { + if state.host.is_none() { + // Capture buffers come from the same render node the encoder imports + // from, resolved via its live fd (the path string is not retained in + // auto mode); each capture thread opens its own device handle. + let gbm_path = if state.use_gpu { + state.gbm_device.as_ref().and_then(|dev| { + use std::os::fd::{AsFd as _, AsRawFd as _}; + let fd = dev.as_fd().as_raw_fd(); + std::fs::read_link(format!("/proc/self/fd/{fd}")).ok() + }) + } else { + None + }; + match crate::wayland::host::HostSession::connect(&settings.wayland_host_display, gbm_path) + { + Ok(h) => { + println!( + "[HostCapture] capturing host compositor '{}' ({} outputs).", + settings.wayland_host_display, + h.output_count() + ); + state.host = Some(h); + } + Err(e) => eprintln!( + "[HostCapture] connect '{}' failed: {e}", + settings.wayland_host_display + ), + } + } + if let Some(host) = &state.host { + // The node's layout offset rides along so the host's heads mirror + // selkies' union layout (input coordinates already assume it). + if let Some(idx) = state.node_idx_for_id(display_id) { + let pos = state.output_nodes[idx].pos; + host.set_layout(display_id, pos.0, pos.1); + } + host.start_capture(display_id, settings.width, settings.height); + } + } - let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); - let display: Display = Display::new().unwrap(); - let dh: DisplayHandle = display.handle(); - unsafe { - if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { - if let Ok(set_max) = lib.get::( - b"wl_display_set_default_max_buffer_size\0", - ) { - set_max( - dh.backend_handle().display_ptr() as *mut std::ffi::c_void, - 10 * 1024 * 1024, - ); + let Some(node_idx) = state.node_idx_for_id(display_id) else { + eprintln!("[Wayland] StartCapture: no output with display id {display_id}."); + return; + }; + let mut node = state.output_nodes.remove(node_idx); + + if state.auto_gpu_selected && settings.encode_node_index < -1 { + if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { + if let Ok(idx) = idx_str.parse::() { + settings.encode_node_index = idx - 128; } - std::mem::forget(lib); } } - let dri_node = explicit_dri_node; + if settings.output_mode == 1 { + settings.width &= !1; + settings.height &= !1; + } - let mut use_gpu = !dri_node.is_empty(); - let render_node_path = dri_node.clone(); + // Tear down this display's previous capture first; its hardware sessions are the + // reuse candidates below (zero-copy inline, readback via the encode config). + let mut prior_zero_copy: Option = None; + let mut prior_readback_encoder: Option = None; + if let Some(mut old) = node.capture.take() { + prior_zero_copy = old.video_encoder.take(); + prior_readback_encoder = teardown_capture(&mut old); + } - let mut gles_renderer = None; - let mut pixman_renderer = None; - let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; - let mut dmabuf_global = None; - let mut gbm_device_raw = None; - let mut dmabuf_state = DmabufState::new(); + // Bind only after the old capture is gone: its sink unlinks the socket path in + // Drop, which would strip a fresh bind's filesystem name and leave every later + // recorder connect with ENOENT. + let recording_sink = crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); - let mut gpu_success = false; - if use_gpu { - println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); - let init_res: Result<(), String> = (|| { - let device_path = std::path::Path::new(&dri_node); - let file = File::options().read(true).write(true).open(device_path) - .map_err(|e| format!("Failed to open render device: {}", e))?; - let file_for_alloc = file.try_clone() - .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; - let gbm_allocator = RawGbmDevice::new(file_for_alloc) - .map_err(|_| "Failed to create Raw GBM Device")?; - let gbm = GbmDevice::new(file) - .map_err(|_| "Failed to create GBM device")?; - let egl = unsafe { EGLDisplay::new(gbm) } - .map_err(|_| "Failed to create EGL display")?; - let context = EGLContext::new(&egl) - .map_err(|_| "Failed to create EGL context")?; - let mut renderer = unsafe { GlesRenderer::new(context) } - .map_err(|_| "Failed to init GlesRenderer")?; - - if let Err(e) = renderer.bind_wl_display(&dh) { - println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + { + // Never panic the compositor thread: an output momentarily without a current + // mode falls back to the requested geometry so the reconfigure below is a + // no-op for size/refresh instead of unwrap-panicking. + let target_refresh = (settings.target_fps * 1000.0).round() as i32; + let (current_w, current_h, current_refresh) = match node.output.current_mode() { + Some(m) => (m.size.w, m.size.h, m.refresh), + None => (settings.width, settings.height, target_refresh), + }; + let current_scale = node.output.current_scale().fractional_scale(); + + if current_w != settings.width + || current_h != settings.height + || (current_scale - settings.scale).abs() > 0.001 + || current_refresh != target_refresh + { + // Allocate the GPU backing for the new dimensions BEFORE committing + // anything: if the driver refuses (VRAM exhaustion, dimensions it will + // not back), the whole reconfigure is skipped and the previous mode + + // buffers stay live. A failed resize must degrade to "no resize", never + // panic the compositor thread. + let mut new_offscreen = None; + let mut gbm_resize_failed = false; + if state.use_gpu { + if let Some(gbm) = state.gbm_device.as_mut() { + match gbm.create_buffer_object( + settings.width as u32, + settings.height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + new_offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!( + "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", + settings.width, settings.height, e + ); + gbm_resize_failed = true; + } + } + } } + if gbm_resize_failed { + // The mode commit below is skipped wholesale, so the rest of this + // StartCapture (encoder setup, stored settings) must see the + // dimensions actually live. + settings.width = current_w; + settings.height = current_h; + settings.scale = current_scale; + settings.target_fps = current_refresh as f64 / 1000.0; + } else { + println!( + "[Wayland] Configuring Output {} ({}): {}x{} @ {:.2} FPS (Scale {:.2})", + display_id, node.output.name(), + settings.width, settings.height, settings.target_fps, settings.scale + ); + let new_mode = OutputMode { + size: (settings.width, settings.height).into(), + refresh: target_refresh, + }; + node.output.change_current_state( + Some(new_mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(settings.scale)), + Some(Point::from(node.pos)), + ); + node.output.set_preferred(new_mode); - let formats = Bind::::supported_formats(&renderer) - .ok_or("Failed to query formats")? - .into_iter() - .collect::>(); + let pixel_count = + (settings.width.max(0) as usize) * (settings.height.max(0) as usize); + node.frame_buffer = vec![0u8; pixel_count * 4]; - let node = DrmNode::from_path(device_path) - .map_err(|_| "Failed to create DrmNode")?; - let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + if let Some(off) = new_offscreen.take() { + node.offscreen_buffer = Some(off); + } + } + } - dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { - dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) - } else { - dmabuf_state.create_global::(&dh, formats) - }); + let scale = settings.scale.max(0.1); + let logical_width = (settings.width as f64 / scale).round() as i32; + let logical_height = (settings.height as f64 / scale).round() as i32; - let bo = gbm_allocator.create_buffer_object( - width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING - ).map_err(|_| "Failed to allocate GBM buffer")?; + for window in state.space.elements() { + if wayland::frontend::window_output_id(window) != display_id { + continue; + } + if let Some(surface) = window.wl_surface() { + node.output.enter(&surface); + with_states(&surface, |states| { + with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + toplevel.with_pending_state(|state| { + use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; + state.states.set(State::Fullscreen); + state.states.set(State::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); + } + } + } - let dmabuf = create_dmabuf_from_bo(&bo); - offscreen_buffer = Some((bo, dmabuf)); - gbm_device_raw = Some(gbm_allocator); - gles_renderer = Some(renderer); - Ok(()) - })(); + let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; + let gpu_intent = settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; + if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { + println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); + } - match init_res { - Ok(_) => gpu_success = true, - Err(e) => { - println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); - use_gpu = false; - } + let mut different_gpu = false; + if gpu_intent { + let encode_node_idx = settings.encode_node_index.max(0); + if !state.render_node_path.is_empty() + && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) + { + different_gpu = true; } } - if !gpu_success { - if !dri_node.is_empty() && !use_gpu { + let mut video_encoder: Option = None; + if gpu_intent && state.use_gpu && !different_gpu { + let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); + println!( + "[Wayland] Encode Node Index: {} | Driver: {}", + settings.encode_node_index.max(0), encode_driver + ); + + if encode_driver.contains("nvidia") { + let reused = match prior_zero_copy.as_mut() { + Some(GpuEncoder::Nvenc(enc)) => match enc.reconfigure_resolution(&settings) { + Ok(()) => { + println!("[Wayland] NVENC session reconfigured in place."); + true + } + Err(e) => { + eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); + false + } + }, + _ => false, + }; + if reused { + video_encoder = prior_zero_copy.take(); + } else { + prior_zero_copy = None; + println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); + let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { + renderer.egl_context().display().get_display_handle().handle + } else { + std::ptr::null() + }; + + match NvencEncoder::new(&settings, egl_display) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Nvenc(encoder)); + println!("[Wayland] NVENC Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", + e + ), + } + } } else { - println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); + prior_zero_copy = None; + println!("[Wayland] Initializing Unified VAAPI Encoder..."); + if settings.video_fullcolor { + println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); + } else { + match VaapiEncoder::new(&settings) { + Ok(encoder) => { + video_encoder = Some(GpuEncoder::Vaapi(encoder)); + println!("[Wayland] VAAPI Encoder initialized successfully."); + } + Err(e) => eprintln!( + "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", + e + ), + } + } } - pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); - use_gpu = false; } + drop(prior_zero_copy); - let compositor_state = CompositorState::new::(&dh); - let fractional_scale_state = FractionalScaleManagerState::new::(&dh); - let shm_state = ShmState::new::(&dh, vec![]); - let output_state = OutputManagerState::new_with_xdg_output::(&dh); - let mut seat_state = SeatState::new(); - let shell_state = XdgShellState::new::(&dh); - let space = Space::default(); - let layer_shell_state = WlrLayerShellState::new::(&dh); - let data_device_state = DataDeviceState::new::(&dh); - let data_control_state = DataControlState::new::(&dh, None, |_| true); - let virtual_keyboard_state = VirtualKeyboardManagerState::new::(&dh, |_client| true); - let pointer_warp_state = PointerWarpManager::new::(&dh); - let relative_pointer_state = RelativePointerManagerState::new::(&dh); - let pointer_constraints_state = PointerConstraintsState::new::(&dh); + if different_gpu { + println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); + } + if video_encoder.is_none() { + println!("[Wayland] Decision: Readback path (encode thread) active."); + } else if !different_gpu { + println!("[Wayland] Decision: Zero-Copy path active."); + } - let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); - let xdg_decoration_state = XdgDecorationState::new::(&dh); - let single_pixel_buffer = SinglePixelBufferState::new::(&dh); - let viewporter_state = ViewporterState::new::(&dh); - let presentation_state = PresentationState::new::(&dh, 1); - let xdg_activation_state = XdgActivationState::new::(&dh); - let primary_selection_state = PrimarySelectionState::new::(&dh); - let popups = PopupManager::default(); + if recording_sink.is_some() && settings.output_mode == 0 { + eprintln!( + "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ + The recording socket requires a single H.264 stream. Please set output_mode=1 \ + on the Python CaptureSettings to produce a recordable output." + ); + } - let mut seat = seat_state.new_wl_seat(&dh, "seat0"); - seat.add_keyboard(XkbConfig::default(), 200, 25) - .expect("Failed to init keyboard"); - seat.add_pointer(); + // Every display's capture composites its own watermark, uploaded at this output's + // scale and placed against this output's frame dimensions. + let watermark_output_scale = node.output.current_scale().fractional_scale(); + node.overlay_state + .load_watermark(&settings.watermark_path, watermark_output_scale); + if display_id == 0 { + state.settings = settings.clone(); + if state.cursor_callback_set { + if let Some(icon) = state.current_cursor_icon.clone() { + state.send_cursor_image(&icon); + } + } + } + state.render_cursor_on_framebuffer = settings.capture_cursor; - let mut state = AppState { - compositor_state, - fractional_scale_state, - viewporter_state, - presentation_state, - shm_state, - single_pixel_buffer, - dmabuf_state, - dmabuf_global, - output_state, - seat_state, - shell_state, - layer_shell_state, - space, - data_device_state, - data_control_state, - dh: dh.clone(), - seat, - virtual_keyboard_state, - pointer_warp_state, - relative_pointer_state, - pointer_constraints_state, - outputs: Vec::new(), - pending_windows: Vec::new(), - foreign_toplevel_list, - xdg_decoration_state, - xdg_activation_state, - primary_selection_state, - popups, - frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], - gles_renderer, - pixman_renderer, - gbm_device: gbm_device_raw, - offscreen_buffer, - is_capturing: false, - settings: RustCaptureSettings { + let mut cap = wayland::frontend::WlCapture { + settings: settings.clone(), + video_encoder, + vaapi_state: StripeState::default(), + recording_sink, + deliver_tx: None, + deliver_join: None, + pending_hw_delivery: None, + pending_hw_damage: false, + encode_pool: None, + encode_join: None, + encode_controls: Arc::new(WlEncodeControls::new()), + encode_stats: Arc::new(WlEncodeStats::new()), + pool_last_render: Vec::new(), + render_seq: 0, + pool_content_gen: Vec::new(), + content_gen: 0, + frame_counter: 0, + pending_force_idr: false, + needs_full_render: true, + last_tick: None, + }; + + { + let (tx, rx) = std::sync::mpsc::sync_channel::>(1); + // With no Python callback (internal recorder-owned capture) the delivery thread + // only drains the channel: the recorder already consumed the frames at the + // delivery-layer tap, upstream of this per-consumer handoff. + let join = match cb { + Some(cb) => thread::spawn(move || { + crate::boost_thread_priority(-10); + while let Ok(stripes) = rx.recv() { + if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } + Python::attach(|py| { + for s in stripes { + match Py::new(py, StripeFrame::new_owned_meta( + s.data, s.data_type, s.stripe_y_start, + s.stripe_height, s.frame_id, + )) { + Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } + Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), + } + } + }); + } + }), + None => thread::spawn(move || while rx.recv().is_ok() {}), + }; + cap.deliver_tx = Some(tx); + cap.deliver_join = Some(join); + } + + if cap.video_encoder.is_none() { + if let Some(deliver_tx) = cap.deliver_tx.clone() { + let pool = Arc::new(WlFramePool::new( + WL_POOL_SURFACES, + (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, + )); + cap.pool_last_render = vec![0; WL_POOL_SURFACES]; + cap.render_seq = 0; + // u64::MAX marks every slot stale so each one is read back before its + // first publish, whatever the damage says. + cap.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; + cap.content_gen = 0; + let c = &cap.encode_controls; + c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + let cfg = WlEncodeConfig { + settings: settings.clone(), + display_id, + use_gpu: state.use_gpu, + try_gpu: gpu_intent && (!state.use_gpu || different_gpu), + prior: prior_readback_encoder.take(), + recording_sink: cap.recording_sink.clone(), + deliver_tx, + controls: cap.encode_controls.clone(), + stats: cap.encode_stats.clone(), + }; + let pool2 = pool.clone(); + cap.encode_join = Some( + thread::Builder::new() + .name(format!("wl-encode-{display_id}")) + .spawn(move || wayland_encode_loop(&pool2, cfg)) + .expect("failed to spawn wl-encode thread"), + ); + cap.encode_pool = Some(pool); + } + } else { + cap.encode_stats.n_stripes.store(1, Ordering::Relaxed); + *cap.encode_stats.desc.lock().unwrap() = + encoder_desc(&settings, cap.video_encoder.as_ref(), false, true); + log_stream_settings(&settings, 1, cap.video_encoder.as_ref(), false); + } + drop(prior_readback_encoder); + // Force the keyframe unconditionally: the damage tracker and offscreen buffer + // stay warm across stop/start, so a restarted capture on a static screen + // otherwise produces no damage, no first frame, and no IDR in either path. + cap.request_idr(); + + node.capture = Some(cap); + state.output_nodes.insert(node_idx, node); + wayland_alive().lock().unwrap().insert(display_id); +} + +/// One output's render + capture tick: composite the elements overlapping this output +/// (positions made output-local by subtracting its layout origin), track damage, feed the +/// display's own encode path, and answer a pending screenshot on the primary. Returns true +/// when the tick was skipped because this display's encode pool was still busy (the caller +/// then retries shortly instead of waiting a full frame interval). +/// Stamp the watermark element onto `target` in place — no clear, so the +/// captured content underneath stays — returning the draw's sync point for the +/// encoder to wait on. +fn draw_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +/// Re-compose `src` (the retained host frame) plus the watermark into `target`: +/// the path a moving watermark needs, since re-drawing over the same retained +/// buffer would leave trails. +fn compose_host_watermark( + renderer: &mut GlesRenderer, + overlay: &crate::encoders::overlay::OverlayState, + src: &Dmabuf, + target: &mut Dmabuf, + size: (i32, i32), +) -> Result { + let elem = overlay + .get_watermark_element(renderer) + .ok_or("watermark element unavailable")?; + let tex = renderer + .import_dmabuf(src, None) + .map_err(|e| format!("import: {e:?}"))?; + let mut fb = renderer.bind(target).map_err(|e| format!("bind: {e:?}"))?; + let full: Rectangle = Rectangle::from_size((size.0, size.1).into()); + let mut frame = renderer + .render(&mut fb, (size.0, size.1).into(), Transform::Normal) + .map_err(|e| format!("render: {e:?}"))?; + frame + .render_texture_from_to( + &tex, + Rectangle::from_size((size.0 as f64, size.1 as f64).into()), + full, + &[full], + // Opaque: the capture format's undefined alpha must not blend, it + // would leave the target's previous content (and stamped + // watermarks) underneath. + &[full], + Transform::Normal, + 1.0, + None, + &[], + ) + .map_err(|e| format!("texture: {e:?}"))?; + let dst = elem.geometry(1.0.into()); + let local = Rectangle::from_size(dst.size); + elem.draw(&mut frame, elem.src(), dst, &[local], &[]) + .map_err(|e| format!("draw: {e:?}"))?; + frame.finish().map_err(|e| format!("finish: {e:?}")) +} + +fn warn_once_host_watermark(e: &str) { + static WARNED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!("[HostCapture] watermark compositing failed: {e}"); + } +} + +fn render_node_tick( + state: &mut AppState, + node: &mut wayland::frontend::OutputNode, + is_memory_throttling: bool, +) -> bool { + let take_screenshot = state + .pending_screenshot + .as_ref() + .is_some_and(|(id, _)| *id == node.id); + if node.capture.is_none() && !take_screenshot { + return false; + } + + // Per-display frame pacing under the one shared timer (which fires at the fastest + // active capture's rate). + if let Some(cap) = node.capture.as_ref() { + if !take_screenshot { + let fps = (if is_memory_throttling { 5.0 } else { cap.settings.target_fps }).max(1.0); + if let Some(last) = cap.last_tick { + if last.elapsed().as_secs_f64() < (1.0 / fps) * 0.9 { + return false; + } + } + } + } + + let output = node.output.clone(); + let origin: Point = node.pos.into(); + let output_scale_val = output.current_scale().fractional_scale(); + let (width, height) = match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height), + None => output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)), + }; + if width <= 0 || height <= 0 { + return false; + } + if node.frame_buffer.len() < (width as usize) * (height as usize) * 4 { + node.frame_buffer = vec![0u8; (width as usize) * (height as usize) * 4]; + } + let logical_w = (width as f64 / output_scale_val).round(); + let logical_h = (height as f64 / output_scale_val).round(); + + // A recorder connecting counts as an IDR request, kept armed across skipped ticks. + if let Some(cap) = node.capture.as_mut() { + if cap + .recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false) + { + cap.request_idr(); + } + } + let requested_idr = node.capture.as_ref().map(|c| c.pending_force_idr).unwrap_or(false); + // A client keyframe request lands on the hardware path's atomic (RequestIdr sets + // it whenever an encode pool exists); host mode consults it — without consuming — + // to decide whether a static screen must re-encode its retained frame. + let hw_idr_pending = node + .capture + .as_ref() + .map(|c| c.encode_controls.force_idr.load(Ordering::Relaxed)) + .unwrap_or(false); + let want_idr_for_host = requested_idr || hw_idr_pending; + + let mut pool_slot: Option<(usize, Vec)> = None; + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + if !is_memory_throttling { + pool_slot = pool.try_begin(); + if pool_slot.is_none() { + return true; + } + } + } + } + + let loc_enum = node + .capture + .as_ref() + .map(|c| c.settings.watermark_location_enum) + .unwrap_or(state.settings.watermark_location_enum); + node.overlay_state.update_position(width, height, loc_enum); + + if let Some(cap) = node.capture.as_mut() { + cap.last_tick = Some(Instant::now()); + } + + // The cursor is composited only on the output the pointer is on, at that output's + // scale; its position is output-local. + let pointer_local: Option> = state + .seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| { + let rect = Rectangle::::new( + origin.to_f64(), + (logical_w, logical_h).into(), + ); + if rect.contains(pos) { + Some(pos - origin.to_f64()) + } else { + None + } + }); + + let mut render_success = false; + let mut render_sync = None; + let mut damage_rects: Vec> = Vec::new(); + let needs_full = node.capture.as_ref().map(|c| c.needs_full_render).unwrap_or(true); + + // Host-capture mode: the host compositor already blitted this display's frame + // into one of our buffers (screencopy); adopt it in place of compositing. + let host_mode = state.host.as_ref().map(|h| h.has_output_for(node.id)).unwrap_or(false); + if state.host.is_some() && !host_mode { + // No host output backs this display (start_capture already warned): + // produce nothing rather than the compositor's own empty content. + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } + // Dmabuf handed to the GPU encoder in host mode (from the new or retained frame). + let mut host_enc_dmabuf: Option = None; + if host_mode { + const RETAINED_OK: u8 = 0; + const RETAINED_NONE: u8 = 1; + const RETAINED_MISMATCH: u8 = 2; + let host_idx = node.id; + let gpu_encoder = node + .capture + .as_ref() + .map(|c| c.video_encoder.is_some()) + .unwrap_or(false); + // The session steps out of `state` while frames are adopted so the + // renderer can composite the watermark / serve screenshot readbacks. + let host = state.host.take().unwrap(); + let new_frame = host.try_take_frame(host_idx); + let have_new = new_frame.is_some(); + // Streaming mode wants a constant-rate stream (the client's decoder pipeline + // is built for it), so re-encode the retained frame every tick like the + // compositor path does. Outside streaming mode, stay damage-driven, waking + // only for a pending IDR (a viewer opening its keyframe gate), a screenshot + // request, or a bouncing watermark that must keep moving. + let streaming = node + .capture + .as_ref() + .map(|c| c.settings.video_streaming_mode) + .unwrap_or(false); + let wm_active = node.overlay_state.is_active(); + let wm_animated = wm_active && node.overlay_state.is_animated(); + if !have_new && !want_idr_for_host && !streaming && !take_screenshot && !wm_animated { + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + state.host = Some(host); + return false; + } + if let Some(f) = new_frame { + host.retain_frame(host_idx, f); + } + // Consume the (new or prior) retained frame's content into the pool slot / + // GPU dmabuf. The frame stays retained for the next IDR. The CPU path + // blends the watermark in place; the GPU path composites it below. Damage + // is the fresh blit's; a re-encode of retained content has none of its own. + let mut wm_drawn = false; + let outcome = host.with_retained(host_idx, |r| { + let Some(f) = r else { return RETAINED_NONE }; + damage_rects = if have_new { f.damage.clone() } else { Vec::new() }; + if let Some(cpu) = f.cpu.as_ref() { + if gpu_encoder { + return RETAINED_MISMATCH; // software frames, GPU encoder + } + if let Some((_, ref mut buf)) = pool_slot { + cpu.write_bgra(f.width, f.height, buf); + if wm_active { + node.overlay_state.blend_bgra(buf, (f.width as usize) * 4, f.width, f.height); + wm_drawn = true; + } + } + cpu.write_bgra(f.width, f.height, &mut node.frame_buffer); + if wm_active { + node.overlay_state + .blend_bgra(&mut node.frame_buffer, (f.width as usize) * 4, f.width, f.height); + } + } else if let Some(dmabuf) = f.dmabuf.as_ref() { + if !gpu_encoder { + return RETAINED_MISMATCH; // GPU frames, CPU encoder + } + host_enc_dmabuf = Some(dmabuf.clone()); + } + RETAINED_OK + }); + // GPU path: composite the watermark and serve screenshot readbacks with + // the renderer. A bouncing watermark re-composes retained content into + // this display's offscreen target every tick (drawing in place would + // trail); anchored watermarks are stamped once onto each fresh blit and + // ride along with retained re-encodes. + if let Some(src) = host_enc_dmabuf.clone() { + if wm_active { + if let Some(renderer) = state.gles_renderer.as_mut() { + if wm_animated { + if let Some((_, target)) = node.offscreen_buffer.as_mut() { + match compose_host_watermark( + renderer, + &node.overlay_state, + &src, + target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + host_enc_dmabuf = Some(target.clone()); + wm_drawn = true; + } + Err(e) => warn_once_host_watermark(&e), + } + } + } else if have_new { + let mut target = src.clone(); + match draw_host_watermark( + renderer, + &node.overlay_state, + &mut target, + (width, height), + ) { + Ok(sync) => { + render_sync = Some(sync); + wm_drawn = true; + } + Err(e) => warn_once_host_watermark(&e), + } + } + } + } + if take_screenshot { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut shot = host_enc_dmabuf.clone().unwrap_or(src); + match renderer.bind(&mut shot) { + Ok(fb) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&fb, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let n = data.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&data[..n]); + } + Err(e) => eprintln!("[HostCapture] screenshot map: {e:?}"), + }, + Err(e) => eprintln!("[HostCapture] screenshot copy: {e:?}"), + } + } + Err(e) => eprintln!("[HostCapture] screenshot bind: {e:?}"), + }; + } + } + } + if wm_drawn { + if let Some(rect) = node.overlay_state.damage_rect(width, height) { + damage_rects.push(rect); + } + } + state.host = Some(host); + if outcome != RETAINED_OK { + if outcome == RETAINED_MISMATCH { + static WARNED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + eprintln!( + "[HostCapture] host frame type and encoder mismatch \ + (GPU host needs a GPU encoder; software host needs a CPU encoder)." + ); + } + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + return false; + } + render_success = true; + } + + if !host_mode && state.use_gpu { + if let Some(renderer) = state.gles_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + if let Some((_bo, dmabuf)) = node.offscreen_buffer.as_mut() { + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { 1 }; + match renderer.bind(dmabuf) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } + } + + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers().rev() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); + } + + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let window_loc = state.space.element_location(window).unwrap_or_default() - origin; + + if let Some(surface) = window.wl_surface() { + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); + let popup_pos = window_loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + + elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(damage) = result.damage { + damage_rects = damage.clone(); + } + render_sync = Some(result.sync); + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; + } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap.as_deref_mut() { + if !damage_rects.is_empty() { + c.content_gen += 1; + } + if let Some((id, ref mut buf)) = pool_slot { + // No-damage ticks skip the readback, so a pooled buffer can + // lag the offscreen target whenever the encoder held the + // other slot across a tick; one catch-up readback keeps + // every published buffer current. + if render_success && c.pool_content_gen[id] != c.content_gen { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + buf.as_mut_ptr() as *mut std::ffi::c_void, + ); + }); + c.pool_content_gen[id] = c.content_gen; + } + } + } + if pool_slot.is_none() && take_screenshot { + let _ = renderer.with_context(|gl| unsafe { + gl.ReadPixels( + 0, + 0, + width, + height, + smithay::backend::renderer::gles::ffi::RGBA, + smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, + node.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, + ); + }); + } + }, + Err(e) => eprintln!("Failed to bind buffer: {:?}", e) + } + } + } + } else if !host_mode { + if let Some(renderer) = state.pixman_renderer.as_mut() { + let mut cap = node.capture.as_mut(); + let (ptr, buf_age) = match pool_slot { + Some((id, ref mut buf)) => { + let age = cap + .as_ref() + .map(|c| { + if c.pool_last_render[id] == 0 { + 0 + } else { + (c.render_seq + 1 - c.pool_last_render[id]) as usize + } + }) + .unwrap_or(0); + (buf.as_mut_ptr() as *mut u32, age) + } + None => (node.frame_buffer.as_mut_ptr() as *mut u32, 0), + }; + let mut image = unsafe { + pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") + }; + match renderer.bind(&mut image) { + Ok(mut frame) => { + let mut elements: Vec>> = Vec::new(); + + if state.render_cursor_on_framebuffer { + if let Some(pos) = pointer_local { + let scale = Scale::from(output_scale_val); + + if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { + let name = wayland::frontend::cursor_icon_to_str(icon); + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { + let phys_pos = pos.to_physical(scale); + let elem_result = with_states(surface, |states| { + WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) + }); + if let Ok(Some(cursor_elem)) = elem_result { + elements.push(CompositionElements::Surface(cursor_elem)); + } + } else if state.current_cursor_icon.is_none() { + let time = Duration::from_millis(state.clock.now().as_millis() as u64); + let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); + if let Some(elem) = node.overlay_state.get_cursor_element(renderer, image, pos, output_scale_val) { + elements.push(CompositionElements::Cursor(elem)); + } + } + } + } + + if let Some(elem) = node.overlay_state.get_watermark_element(renderer) { + elements.push(CompositionElements::Cursor(elem)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); + } + + for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { + let loc = state.space.element_location(window).unwrap_or_default() - origin; + + if let Some(surface) = window.wl_surface() { + let popups = PopupManager::popups_for_surface(&surface); + for (popup, location) in popups { + let popup_surface = popup.wl_surface(); { + let popup_pos = loc + location; + let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, + popup_surface, + states, + popup_pos.to_physical_precise_round(output_scale_val), + 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + + elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(output_scale_val), Scale::from(output_scale_val), 1.0).into_iter().map(CompositionElements::Space)); + } + + { + let layer_map = layer_map_for_output(&output); + + let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { + for surface in layer_map.layers() { + let current_layer = surface.layer(); + if current_layer == target_layer { + if let Some(geo) = layer_map.layer_geometry(surface) { + let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { + WaylandSurfaceRenderElement::from_surface( + renderer, surface.wl_surface(), states, + geo.loc.to_physical_precise_round(output_scale_val), 1.0, + smithay::backend::renderer::element::Kind::Unspecified + ) + }); + if let Ok(Some(e)) = elem { + elements.push(CompositionElements::Surface(e)); + } + } + } + } + }; + + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); + draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); + } + + let render_age = if node.overlay_state.is_animated() || needs_full { 0 } else { buf_age }; + match node.damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { + Ok(result) => { + render_success = true; + if let Some(c) = cap.as_deref_mut() { + c.needs_full_render = false; + } + if let Some(damage) = result.damage { damage_rects = damage.clone(); } + }, + Err(e) => eprintln!("Render error: {:?}", e) + } + if let Some(c) = cap.as_deref_mut() { + c.render_seq += 1; + if render_success { + if let Some((id, _)) = pool_slot { + c.pool_last_render[id] = c.render_seq; + } + } + } + }, + Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) + } + } + } + + if render_success { + let time = state.clock.now(); + for window in state.space.elements_for_output(&output).cloned().collect::>() { + window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); + } + + if let Some(cap) = node.capture.as_mut() { + if is_memory_throttling { + if cap.encode_pool.is_none() { + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if cap.encode_pool.is_some() { + if take_screenshot { + if let Some((_, ref buf)) = pool_slot { + let n = buf.len().min(node.frame_buffer.len()); + node.frame_buffer[..n].copy_from_slice(&buf[..n]); + } + } + if let Some((id, buf)) = pool_slot.take() { + let is_animated = node.overlay_state.is_animated(); + cap.encode_pool.as_ref().unwrap().publish(WlFrame { + id, + buf, + frame_id: cap.frame_counter, + damage: std::mem::take(&mut damage_rects), + is_animated, + }); + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } else if let Some(ref mut encoder) = cap.video_encoder { + // Deliver the parked frame (if any) first, WITHOUT blocking: this runs + // on the calloop thread, and a blocking send would freeze + // input/command/Wayland dispatch for as long as the Python consumer + // stalls. While a frame stays parked, no new frame is encoded — an + // encoded frame joins the H.264 reference chain and can never be + // dropped — and the tick's damage is latched so the pause never loses + // a change. + let slot_free = match cap.pending_hw_delivery.take() { + None => true, + Some(pending) => match cap.deliver_tx.as_ref() { + None => true, + Some(tx) => match tx.try_send(pending) { + Ok(()) => true, + Err(std::sync::mpsc::TrySendError::Full(p)) => { + cap.pending_hw_delivery = Some(p); + false + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, + }, + }, + }; + if !slot_free { + if !damage_rects.is_empty() { + cap.pending_hw_damage = true; + } + } else { + let is_animated = node.overlay_state.is_animated(); + let had_damage = !damage_rects.is_empty() + || std::mem::take(&mut cap.pending_hw_damage); + let decision = crate::pipeline::decide_hw_fullframe( + &mut cap.vaapi_state, + &cap.settings, + cap.frame_counter, + had_damage, + is_animated, + requested_idr, + ); + let send_frame = decision.send; + let force_idr = decision.force_idr; + let target_qp = decision.target_qp; + + let mut frame_out = false; + if send_frame { + if let Some(sync) = render_sync.take() { + let _ = sync.wait(); + } + // Host-capture frames encode from the buffer the host blitted + // into; otherwise from this display's own composited buffer. + let enc_dmabuf: Option = host_enc_dmabuf + .clone() + .or_else(|| node.offscreen_buffer.as_ref().map(|(_, d)| d.clone())); + let result = match encoder { + GpuEncoder::Nvenc(enc) => { + if let Some(ref dmabuf) = enc_dmabuf { + enc.encode(dmabuf, cap.frame_counter as u64, target_qp, force_idr) + } else { + Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + }, + GpuEncoder::Vaapi(enc) => { + if let Some(ref dmabuf) = enc_dmabuf { + enc.encode_dmabuf(dmabuf, cap.frame_counter as u64, target_qp, force_idr) + } else { + Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) + } + } + }; + + if let Ok(data) = result { + if !data.is_empty() { + frame_out = true; + cap.encode_stats.frames.fetch_add(1, Ordering::Relaxed); + cap.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); + if let Some(ref tx) = cap.deliver_tx { + let stripes = vec![EncodedStripe { + data: Arc::new(data), data_type: 2, stripe_y_start: 0, + stripe_height: height, frame_id: cap.frame_counter as i32, + }]; + if let Some(ref socket) = cap.recording_sink { + socket.write_frame(&stripes, height); + } + crate::recorder::wayland_tap(node.id, &stripes); + // Non-blocking: a full slot parks the frame (delivered + // ahead of any new encode above). + match tx.try_send(stripes) { + Ok(()) => {} + Err(std::sync::mpsc::TrySendError::Full(s)) => { + cap.pending_hw_delivery = Some(s); + } + Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} + } + } + } + } else if let Err(e) = result { + eprintln!("HW Encode Error: {}", e); + } + } + // An unserved request stays armed: on an infinite GOP an IDR lost to an + // encode error would never self-heal. + cap.pending_force_idr = requested_idr && !frame_out; + cap.frame_counter = cap.frame_counter.wrapping_add(1); + } + } + } + if take_screenshot { + if let Some((_, resp)) = state.pending_screenshot.take() { + if !node.frame_buffer.is_empty() { + let w = width as u32; + let h = height as u32; + let png = if state.use_gpu { + crate::computer_use::encode_png_rgba(&node.frame_buffer, w, h) + } else { + let mut rgba = node.frame_buffer.clone(); + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); + } + crate::computer_use::encode_png_rgba(&rgba, w, h) + }; + match png { + Ok(data) => { let _ = resp.send(Ok(data)); } + Err(e) => { + let _ = resp.send(Err(format!("PNG encode error: {e}"))); + eprintln!("[ComputerUse] PNG encode error: {}", e); + } + } + } else { + let _ = resp.send(Err("Screenshot render produced no pixels".to_string())); + } + } + } + } + if let Some((id, buf)) = pool_slot.take() { + if let Some(cap) = node.capture.as_ref() { + if let Some(ref pool) = cap.encode_pool { + pool.cancel(id, buf); + } + } + } + false +} + +/// True when the rectangles `(x, y, w, h)` overlap: strict interior intersection, so +/// touching edges do not count and empty (non-positive-dimension) rectangles never +/// overlap anything. Arithmetic is widened so extreme coordinates cannot wrap. +fn rects_overlap(a: (i32, i32, i32, i32), b: (i32, i32, i32, i32)) -> bool { + let (ax, ay, aw, ah) = (a.0 as i64, a.1 as i64, a.2 as i64, a.3 as i64); + let (bx, by, bw, bh) = (b.0 as i64, b.1 as i64, b.2 as i64, b.3 as i64); + aw > 0 && ah > 0 && bw > 0 && bh > 0 + && ax < bx + bw && bx < ax + aw + && ay < by + bh && by < ay + ah +} + +/// The first live output (excluding `skip_id`) whose rectangle overlaps a candidate +/// placement, as `(id, flavor, rect)`. Both rectangle flavors are checked — logical +/// (Space layout, scale-divided) and physical (mode pixels at the same origin) — because +/// input injection and cursor compositing key off the physical rects while window layout +/// keys off the logical ones, and neither may overlap. +fn find_output_overlap( + nodes: &[wayland::frontend::OutputNode], + skip_id: Option, + logical: (i32, i32, i32, i32), + physical: (i32, i32, i32, i32), +) -> Option<(u32, &'static str, (i32, i32, i32, i32))> { + for n in nodes { + if Some(n.id) == skip_id { + continue; + } + if let Some(geo) = n.logical_geometry() { + let other = (geo.loc.x, geo.loc.y, geo.size.w, geo.size.h); + if rects_overlap(logical, other) { + return Some((n.id, "logical", other)); + } + } + if let Some(mode) = n.output.current_mode() { + let other = (n.pos.0, n.pos.1, mode.size.w, mode.size.h); + if rects_overlap(physical, other) { + return Some((n.id, "physical", other)); + } + } + } + None +} + +/// Create an additional output mapped into the layout at `(x, y)`. Fails (false) on a +/// duplicate id, non-positive geometry/scale, a rectangle overlapping a live output, or a +/// GPU render-target allocation failure. Only Create/Reposition placements are validated: +/// a capture reconfigure (StartCapture on an existing output) resizes UNVALIDATED, so +/// keeping a multi-step relayout overlap-free at every step is the caller's ordering +/// responsibility. +fn create_output_on( + state: &mut AppState, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, +) -> bool { + if state.node_idx_for_id(id).is_some() || width <= 0 || height <= 0 || scale <= 0.0 { + return false; + } + let logical_size = ( + (width as f64 / scale).round() as i32, + (height as f64 / scale).round() as i32, + ); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + None, + (x, y, logical_size.0, logical_size.1), + (x, y, width, height), + ) { + eprintln!( + "[Wayland] CreateOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { width }, + if flavor == "logical" { logical_size.1 } else { height }, + other.2, other.3, other.0, other.1, + ); + return false; + } + let output = Output::new( + format!("HEADLESS-{}", id + 1), + PhysicalProperties { + size: (width, height).into(), + subpixel: Subpixel::Unknown, + make: "Pixelflux".into(), + model: "Virtual".into(), + serial_number: format!("{:03}", id + 1), + }, + ); + let mode = OutputMode { size: (width, height).into(), refresh: 60_000 }; + output.change_current_state( + Some(mode), + Some(Transform::Normal), + Some(OutputScale::Fractional(scale)), + Some((x, y).into()), + ); + output.set_preferred(mode); + let mut offscreen = None; + if state.use_gpu { + let Some(gbm) = state.gbm_device.as_mut() else { return false }; + match gbm.create_buffer_object( + width as u32, + height as u32, + GbmFormat::Argb8888, + BufferObjectFlags::RENDERING, + ) { + Ok(bo) => { + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen = Some((bo, dmabuf)); + } + Err(e) => { + eprintln!("[Wayland] CreateOutput {id}: GBM allocation {width}x{height} failed ({e:?})."); + return false; + } + } + } + state.space.map_output(&output, (x, y)); + let global = output.create_global::(&state.dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } + println!("[Wayland] Output {id} created: {width}x{height} @ ({x}, {y}) scale {scale:.2}."); + state.output_nodes.push(wayland::frontend::OutputNode { + id, + output, + global, + pos: (x, y), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer: offscreen, + overlay_state: OverlayState::default(), + capture: None, + }); + // A nested session opens one host toplevel per screen and stacks the extras + // on an existing output until a display exists for them: hand the newest + // stacked window to the new output. + let mut counts: Vec<(u32, usize)> = Vec::new(); + for w in state.space.elements() { + let oid = wayland::frontend::window_output_id(w); + match counts.iter_mut().find(|(o, _)| *o == oid) { + Some((_, c)) => *c += 1, + None => counts.push((oid, 1)), + } + } + let adopt = state + .space + .elements() + .filter(|w| { + let oid = wayland::frontend::window_output_id(w); + counts.iter().any(|(o, c)| *o == oid && *c >= 2) + }) + .max_by_key(|w| wayland::frontend::window_meta(w).map(|m| m.id).unwrap_or(0)) + .cloned(); + if let Some(window) = adopt { + state.place_window_on_output(&window, id); + println!( + "[Wayland] Output {id}: adopted stacked window {}.", + wayland::frontend::window_meta(&window).map(|m| m.id).unwrap_or(0) + ); + } + true +} + +/// Move an existing output (the primary included) to layout offset `(x, y)`. The output's +/// advertised position, its Space mapping, and the windows placed on it (mapped at the +/// output's origin — window positions are output-relative under forced fullscreen) all +/// follow, so absolute input injection and cursor compositing — both keyed off `node.pos` — +/// resolve against the new layout immediately. A destination overlapping another live +/// output is refused (false). As with `CreateOutput`, only the placement itself is +/// validated: a capture reconfigure (StartCapture on an existing output) resizes +/// UNVALIDATED, so keeping a multi-step relayout overlap-free at every step is the +/// caller's ordering responsibility. +fn reposition_output_on(state: &mut AppState, id: u32, x: i32, y: i32) -> bool { + let Some(idx) = state.node_idx_for_id(id) else { return false }; + let output = state.output_nodes[idx].output.clone(); + if state.output_nodes[idx].pos == (x, y) { + return true; + } + let logical_size = state.output_nodes[idx] + .logical_geometry() + .map(|g| (g.size.w, g.size.h)) + .unwrap_or((0, 0)); + let physical_size = output.current_mode().map(|m| (m.size.w, m.size.h)).unwrap_or((0, 0)); + if let Some((oid, flavor, other)) = find_output_overlap( + &state.output_nodes, + Some(id), + (x, y, logical_size.0, logical_size.1), + (x, y, physical_size.0, physical_size.1), + ) { + eprintln!( + "[Wayland] RepositionOutput {id}: rejected, {flavor} rect {}x{}+{x}+{y} overlaps output {oid} at {}x{}+{}+{}.", + if flavor == "logical" { logical_size.0 } else { physical_size.0 }, + if flavor == "logical" { logical_size.1 } else { physical_size.1 }, + other.2, other.3, other.0, other.1, + ); + return false; + } + state.output_nodes[idx].pos = (x, y); + if let Some(host) = state.host.as_ref() { + host.set_layout(id, x, y); + } + output.change_current_state(None, None, None, Some((x, y).into())); + state.space.map_output(&output, (x, y)); + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + state.space.map_element(window.clone(), (x, y), false); + } + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + cap.needs_full_render = true; + } + println!("[Wayland] Output {id} repositioned to ({x}, {y})."); + true +} + +/// Destroy a secondary output: end its capture, relocate its windows onto the primary +/// output, unmap it from the space, and retract its global. The primary (id 0) is refused. +fn destroy_output_on(state: &mut AppState, id: u32) -> bool { + if id == 0 { + return false; + } + let Some(_) = state.node_idx_for_id(id) else { return false }; + stop_capture_on_display(state, id); + if let Some(host) = state.host.as_ref() { + host.idle_output(id); + } + wayland_owners().lock().unwrap().remove(&id); + // Relocate while the node is still registered so output leave/enter both resolve. + let windows: Vec = state + .space + .elements() + .filter(|w| wayland::frontend::window_output_id(w) == id) + .cloned() + .collect(); + for window in &windows { + state.place_window_on_output(window, 0); + } + for w in &state.pending_windows { + if let Some(meta) = wayland::frontend::window_meta(w) { + if meta.output.load(Ordering::Relaxed) == id { + meta.output.store(0, Ordering::Relaxed); + } + } + } + let idx = state.node_idx_for_id(id).unwrap(); + let node = state.output_nodes.remove(idx); + state.space.unmap_output(&node.output); + state.dh.remove_global::(node.global); + println!( + "[Wayland] Output {id} destroyed; {} window(s) relocated to primary.", + windows.len() + ); + true +} + +/// The main execution loop of the Wayland backend. +/// +/// This function is the central nervous system of the backend. It runs on its own thread and owns +/// the entire lifecycle of the headless Wayland compositor: +/// +/// 1. **Initialization**: builds the `calloop` event loop and the Wayland display, raises +/// libwayland's per-client buffer limit when the newer setter is available (resolved at runtime +/// so the module still loads against older libwayland), and brings up the rendering pipeline — +/// GBM/EGL hardware acceleration on the resolved DRM render node, falling back to software +/// rendering (Pixman) when no node is usable. +/// 2. **State management**: constructs and holds the `AppState` — the Wayland globals (compositor, +/// seat, SHM, shell, dmabuf, selections, and the rest) plus the output registry: the primary +/// virtual `HEADLESS-1` output at layout (0, 0), extended at runtime by CreateOutput with +/// additional outputs at their layout offsets, each `OutputNode` owning its damage tracker, +/// render targets, and (at most one) capture pipeline. +/// 3. **Event dispatch**: +/// - **Command channel**: control messages from the Python thread — per-display start/stop, +/// output lifecycle (create/destroy/list/move-window), input injection routed across the +/// output layout, keymap and clipboard operations, live rate / tunable changes, and the +/// computer-use queries. +/// - **Wayland socket**: accepts client connections and drives the compositor protocol. +/// 4. **StartCapture reconfigure** (per display): reprograms that output's mode / scale / refresh, +/// resizes its framebuffer and offscreen GBM buffer, and fullscreens the toplevels placed on +/// it. The encode device is resolved here: an operator's explicit `encode_node_index` +/// (-1 software, >= 0 a device) always wins, and only the unset `-2` sentinel is filled from +/// the auto-picked render node. H.264 output masks the dimensions even, because 4:2:0 needs +/// even width and height. +/// 5. **Encode-path choice + render loop**: only a same-GPU GLES session encodes zero-copy on this +/// calloop thread, because the dmabuf and its EGL context are calloop-affine; every readback +/// flavor (OpenH264, striped x264/JPEG, Pixman, or a cross-GPU hardware encoder) builds its +/// encoders on that display's dedicated encode thread instead. A shared timer (paced at the +/// fastest active capture) renders each capturing output — its windows, popups and layers made +/// output-local, the cursor only on the pointer's output — applies the shared paint-over / +/// recovery-IDR policy per display, and delivers each display's encoded stripes through its own +/// frame callback. The zero-copy encode waits the GL render fence first, so a hardware encoder +/// reading the dmabuf through CUDA/VA never maps a half-rasterized (torn) frame. +/// 6. **Thread lifecycle**: the Python frame callback runs on a dedicated delivery thread so its +/// GIL never stalls calloop input / control dispatch, and in readback mode the encoders run on +/// the `wl-encode` thread. On a restart or stop the encode thread is torn down before the +/// delivery thread — it feeds the delivery sender and must be gone first — and the retained +/// callbacks are dropped and gated by a process-shutdown flag so nothing fires into a finalizing +/// interpreter. +fn run_wayland_thread( + command_rx: smithay::reexports::calloop::channel::Channel, + wake_rx: smithay::reexports::calloop::channel::Channel<()>, + command_tx: smithay::reexports::calloop::channel::Sender, + initial_width: i32, + initial_height: i32, + explicit_dri_node: String, + auto_gpu_selected: bool, + cursor_size: i32, +) { + let width: i32 = if initial_width > 0 { initial_width } else { 1024 }; + let height: i32 = if initial_height > 0 { initial_height } else { 768 }; + + let mut event_loop = EventLoop::::try_new().expect("Unable to create event_loop"); + let display: Display = Display::new().unwrap(); + let dh: DisplayHandle = display.handle(); + unsafe { + if let Ok(lib) = libloading::Library::new("libwayland-server.so.0") { + if let Ok(set_max) = lib.get::( + b"wl_display_set_default_max_buffer_size\0", + ) { + set_max( + dh.backend_handle().display_ptr() as *mut std::ffi::c_void, + 10 * 1024 * 1024, + ); + } + std::mem::forget(lib); + } + } + + let dri_node = explicit_dri_node; + + let mut use_gpu = !dri_node.is_empty(); + let render_node_path = dri_node.clone(); + + let mut gles_renderer = None; + let mut pixman_renderer = None; + let mut offscreen_buffer: Option<(BufferObject<()>, Dmabuf)> = None; + let mut dmabuf_global = None; + let mut gbm_device_raw = None; + let mut dmabuf_state = DmabufState::new(); + + let mut gpu_success = false; + if use_gpu { + println!("[Wayland] Initializing GL Renderer using device: {}", dri_node); + let init_res: Result<(), String> = (|| { + let device_path = std::path::Path::new(&dri_node); + let file = File::options().read(true).write(true).open(device_path) + .map_err(|e| format!("Failed to open render device: {}", e))?; + let file_for_alloc = file.try_clone() + .map_err(|e| format!("Failed to clone file for GBM Allocator: {}", e))?; + let gbm_allocator = RawGbmDevice::new(file_for_alloc) + .map_err(|_| "Failed to create Raw GBM Device")?; + let gbm = GbmDevice::new(file) + .map_err(|_| "Failed to create GBM device")?; + let egl = unsafe { EGLDisplay::new(gbm) } + .map_err(|_| "Failed to create EGL display")?; + let context = EGLContext::new(&egl) + .map_err(|_| "Failed to create EGL context")?; + let mut renderer = unsafe { GlesRenderer::new(context) } + .map_err(|_| "Failed to init GlesRenderer")?; + + if let Err(e) = renderer.bind_wl_display(&dh) { + println!("[Wayland] Warning: Failed to bind EGL to Wayland Display (Optional): {:?}", e); + } + + let formats = Bind::::supported_formats(&renderer) + .ok_or("Failed to query formats")? + .into_iter() + .collect::>(); + + let node = DrmNode::from_path(device_path) + .map_err(|_| "Failed to create DrmNode")?; + let dmabuf_default_feedback = DmabufFeedbackBuilder::new(node.dev_id(), formats.clone()).build(); + + dmabuf_global = Some(if let Ok(default_feedback) = dmabuf_default_feedback { + dmabuf_state.create_global_with_default_feedback::(&dh, &default_feedback) + } else { + dmabuf_state.create_global::(&dh, formats) + }); + + let bo = gbm_allocator.create_buffer_object( + width as u32, height as u32, GbmFormat::Argb8888, BufferObjectFlags::RENDERING + ).map_err(|_| "Failed to allocate GBM buffer")?; + + let dmabuf = create_dmabuf_from_bo(&bo); + offscreen_buffer = Some((bo, dmabuf)); + gbm_device_raw = Some(gbm_allocator); + gles_renderer = Some(renderer); + Ok(()) + })(); + + match init_res { + Ok(_) => gpu_success = true, + Err(e) => { + println!("[Wayland] GPU Initialization failed: {}. Falling back to Software Renderer (Pixman).", e); + use_gpu = false; + } + } + } + + if !gpu_success { + if !dri_node.is_empty() && !use_gpu { + } else { + println!("[Wayland] No render node. Initializing Software Renderer (Pixman)."); + } + pixman_renderer = Some(PixmanRenderer::new().expect("Failed to init PixmanRenderer")); + use_gpu = false; + } + + let compositor_state = CompositorState::new::(&dh); + let fractional_scale_state = FractionalScaleManagerState::new::(&dh); + let shm_state = ShmState::new::(&dh, vec![]); + let output_state = OutputManagerState::new_with_xdg_output::(&dh); + let mut seat_state = SeatState::new(); + let shell_state = XdgShellState::new::(&dh); + let space = Space::default(); + let layer_shell_state = WlrLayerShellState::new::(&dh); + let data_device_state = DataDeviceState::new::(&dh); + let data_control_state = DataControlState::new::(&dh, None, |_| true); + let _vk_global = dh.create_global::(1, ()); + let pointer_warp_state = PointerWarpManager::new::(&dh); + let relative_pointer_state = RelativePointerManagerState::new::(&dh); + let pointer_constraints_state = PointerConstraintsState::new::(&dh); + + let foreign_toplevel_list = ForeignToplevelListState::new::(&dh); + let xdg_decoration_state = XdgDecorationState::new::(&dh); + let single_pixel_buffer = SinglePixelBufferState::new::(&dh); + let viewporter_state = ViewporterState::new::(&dh); + let presentation_state = PresentationState::new::(&dh, 1); + let xdg_activation_state = XdgActivationState::new::(&dh); + let primary_selection_state = PrimarySelectionState::new::(&dh); + let popups = PopupManager::default(); + + let mut seat = seat_state.new_wl_seat(&dh, "seat0"); + seat.add_keyboard(XkbConfig::default(), 200, 25) + .expect("Failed to init keyboard"); + seat.add_pointer(); + + let mut state = AppState { + compositor_state, + fractional_scale_state, + viewporter_state, + presentation_state, + shm_state, + single_pixel_buffer, + dmabuf_state, + dmabuf_global, + output_state, + seat_state, + shell_state, + layer_shell_state, + space, + data_device_state, + data_control_state, + dh: dh.clone(), + seat, + pointer_warp_state, + relative_pointer_state, + pointer_constraints_state, + output_nodes: Vec::new(), + pending_windows: Vec::new(), + foreign_toplevel_list, + xdg_decoration_state, + xdg_activation_state, + primary_selection_state, + popups, + gles_renderer, + pixman_renderer, + gbm_device: gbm_device_raw, + settings: RustCaptureSettings { width, height, ..RustCaptureSettings::default() }, - callback: None, - cursor_callback: None, + cursor_callback_set: false, + cursor_tx: wayland::cursor::spawn_cursor_worker(cursor_size), clipboard_callback: None, pending_clipboard_read: None, + current_selection_mime: None, last_log_time: Instant::now(), start_time: Instant::now(), clock: Clock::new(), - frame_counter: 0, - pending_force_idr: false, - needs_full_render: true, use_gpu, - video_encoder: None, - vaapi_state: StripeState::default(), cursor_helper: Cursor::load(cursor_size), - overlay_state: OverlayState::default(), + keymap_policy: wayland::keymap::KeymapPolicy::empty(), + host: None, current_cursor_icon: None, cursor_buffer: None, - cursor_cache: std::collections::HashMap::new(), render_cursor_on_framebuffer: false, render_node_path, auto_gpu_selected, - recording_sink: None, - deliver_tx: None, - deliver_join: None, - pending_hw_delivery: None, - pending_hw_damage: false, - encode_pool: None, - encode_join: None, - encode_controls: Arc::new(WlEncodeControls::new()), - encode_stats: Arc::new(WlEncodeStats::new()), - pool_last_render: Vec::new(), - render_seq: 0, - pool_content_gen: Vec::new(), - content_gen: 0, pending_screenshot: None, + command_rx: None, }; + // Seed the keymap policy with the seat's initial keymap so overlay binds splice onto + // the exact text clients received. + { + let initial_keymap = if let Some(kb) = state.seat.get_keyboard() { + kb.with_xkb_state(&mut state, |context| match context.xkb().lock() { + Ok(guard) => { + let keymap = unsafe { guard.keymap() }; + keymap.get_as_string(smithay::input::keyboard::xkb::KEYMAP_FORMAT_TEXT_V1) + } + Err(_) => String::new(), + }) + } else { + String::new() + }; + state.keymap_policy.rebuild_base(initial_keymap); + } let output = Output::new( "HEADLESS-1".into(), @@ -1682,388 +3383,128 @@ fn run_wayland_thread( refresh: 60_000, }); state.space.map_output(&output, (0, 0)); - state.outputs.push(output.clone()); - let _global = output.create_global::(&dh); - let mut damage_tracker = OutputDamageTracker::from_output(&output); - - event_loop - .handle() - .insert_source(command_rx, move |event, _, state| { - match event { - CalloopEvent::Msg(ThreadCommand::StartCapture(cb, mut settings)) => { - if state.auto_gpu_selected && settings.encode_node_index < -1 { - if let Some(idx_str) = state.render_node_path.strip_prefix("/dev/dri/renderD") { - if let Ok(idx) = idx_str.parse::() { - settings.encode_node_index = idx - 128; - } - } - } - - if settings.output_mode == 1 { - settings.width &= !1; - settings.height &= !1; - } - - state.recording_sink = - crate::recording_sink::RecordingSink::try_bind(&settings.recording_socket); - - if let Some(output) = state.outputs.first() { - let current_mode = output.current_mode().unwrap(); - let current_w = current_mode.size.w; - let current_h = current_mode.size.h; - let current_scale = output.current_scale().fractional_scale(); - let current_refresh = current_mode.refresh; - let target_refresh = (settings.target_fps * 1000.0).round() as i32; - - let scale = settings.scale.max(0.1); - let logical_width = (settings.width as f64 / scale).round() as i32; - let logical_height = (settings.height as f64 / scale).round() as i32; - - if current_w != settings.width - || current_h != settings.height - || (current_scale - settings.scale).abs() > 0.001 - || current_refresh != target_refresh - { - // Allocate the GPU backing for the new dimensions BEFORE - // committing anything: if the driver refuses (VRAM - // exhaustion, dimensions it will not back), the whole - // reconfigure is skipped and the previous mode + buffers - // stay live. A failed resize must degrade to "no resize", - // never panic the compositor thread — that would drop - // every Wayland client with no recovery short of a - // process restart. (The INITIAL allocation path already - // degrades gracefully, to the software renderer.) - let mut new_offscreen = None; - let mut gbm_resize_failed = false; - if state.use_gpu { - if let Some(gbm) = state.gbm_device.as_mut() { - match gbm.create_buffer_object( - settings.width as u32, - settings.height as u32, - GbmFormat::Argb8888, - BufferObjectFlags::RENDERING, - ) { - Ok(bo) => { - let dmabuf = create_dmabuf_from_bo(&bo); - new_offscreen = Some((bo, dmabuf)); - } - Err(e) => { - eprintln!( - "[Wayland] GBM buffer resize to {}x{} failed ({:?}); keeping previous output mode.", - settings.width, settings.height, e - ); - gbm_resize_failed = true; - } - } - } - } - if gbm_resize_failed { - // The mode commit below is skipped wholesale, so the - // rest of this StartCapture (encoder setup, stored - // settings) must also see the dimensions that are - // actually live — otherwise the encoder and the - // still-old buffers disagree on frame geometry. - settings.width = current_w; - settings.height = current_h; - settings.scale = current_scale; - settings.target_fps = current_refresh as f64 / 1000.0; - } else { - println!( - "[Wayland] Configuring Output: {}x{} @ {:.2} FPS (Scale {:.2})", - settings.width, settings.height, settings.target_fps, settings.scale - ); - let new_mode = OutputMode { - size: (settings.width, settings.height).into(), - refresh: target_refresh, - }; - output.change_current_state( - Some(new_mode), - Some(Transform::Normal), - Some(OutputScale::Fractional(settings.scale)), - Some((0, 0).into()), - ); - output.set_preferred(new_mode); - - let pixel_count = - (settings.width.max(0) as usize) * (settings.height.max(0) as usize); - state.frame_buffer = vec![0u8; pixel_count * 4]; - - if let Some(off) = new_offscreen.take() { - state.offscreen_buffer = Some(off); - } - } - } - - for window in state.space.elements() { - if let Some(surface) = window.wl_surface() { - output.enter(&surface); - with_states(&surface, |states| { - smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { - fs.set_preferred_scale(scale); - }); - }); - } - if let Some(toplevel) = window.toplevel() { - toplevel.with_pending_state(|state| { - use smithay::reexports::wayland_protocols::xdg::shell::server::xdg_toplevel::State; - state.states.set(State::Fullscreen); - state.states.set(State::Activated); - state.size = Some((logical_width, logical_height).into()); - }); - toplevel.send_configure(); - } - } - } - - let use_cpu_explicit = settings.use_cpu || settings.encode_node_index == -1; - let gpu_intent = - settings.output_mode == 1 && !settings.use_openh264 && !use_cpu_explicit; - if use_cpu_explicit && !(settings.output_mode == 1 && settings.use_openh264) { - println!("[Wayland] CPU encoding selected (use_cpu=true or encode_node_index=-1)."); - } - - let mut different_gpu = false; - if gpu_intent { - let encode_node_idx = settings.encode_node_index.max(0); - if !state.render_node_path.is_empty() - && !state.render_node_path.contains(&format!("renderD{}", 128 + encode_node_idx)) - { - different_gpu = true; - } - } - - if gpu_intent && state.use_gpu && !different_gpu { - let encode_driver = get_gpu_driver(settings.encode_node_index.max(0)); - println!( - "[Wayland] Encode Node Index: {} | Driver: {}", - settings.encode_node_index.max(0), encode_driver - ); - - if encode_driver.contains("nvidia") { - let reused = match state.video_encoder.as_mut() { - Some(GpuEncoder::Nvenc(enc)) => { - match enc.reconfigure_resolution(&settings) { - Ok(()) => { - println!("[Wayland] NVENC session reconfigured in place."); - true - } - Err(e) => { - eprintln!("[Wayland] NVENC in-place reconfigure unavailable ({e}); rebuilding."); - false - } - } - } - _ => false, - }; - if !reused { - state.video_encoder = None; - println!("[Wayland] Nvidia Encoder detected. Initializing NVENC..."); - let egl_display = if let Some(renderer) = state.gles_renderer.as_ref() { - renderer.egl_context().display().get_display_handle().handle - } else { - std::ptr::null() - }; - - match NvencEncoder::new(&settings, egl_display) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Nvenc(encoder)); - println!("[Wayland] NVENC Encoder initialized successfully."); - } - Err(e) => eprintln!( - "[Wayland] Failed to init NVENC: {}. Falling back to CPU.", - e - ), - } - } - } else { - state.video_encoder = None; - println!("[Wayland] Initializing Unified VAAPI Encoder..."); - if settings.video_fullcolor { - println!("[Wayland] 4:4:4 Fullcolor requested. VAAPI does not support this profile reliably. Falling back to CPU."); - } else { - match VaapiEncoder::new(&settings) { - Ok(encoder) => { - state.video_encoder = Some(GpuEncoder::Vaapi(encoder)); - println!( - "[Wayland] VAAPI Encoder initialized successfully." - ); - } - Err(e) => eprintln!( - "[Wayland] Failed to init VAAPI: {}. Falling back to CPU.", - e - ), - } - } - } - } else { - state.video_encoder = None; - } - - if different_gpu { - println!("[Wayland] Decision: Rendering and Encoding GPUs differ -> Forcing Readback (CPU path for pixels)."); - } - if state.video_encoder.is_none() { - println!("[Wayland] Decision: Readback path (encode thread) active."); - } else if !different_gpu { - println!("[Wayland] Decision: Zero-Copy path active."); - } - - if state.recording_sink.is_some() && settings.output_mode == 0 { - eprintln!( - "[recording_sink] WARNING: recording_socket is set but output_mode is JPEG (0). \ - The recording socket requires a single H.264 stream. Please set output_mode=1 \ - on the Python CaptureSettings to produce a recordable output." - ); - } + let global = output.create_global::(&dh); + let damage_tracker = OutputDamageTracker::from_output(&output); + state.output_nodes.push(wayland::frontend::OutputNode { + id: 0, + output, + global, + pos: (0, 0), + damage_tracker, + frame_buffer: vec![0u8; (width.max(0) as usize) * (height.max(0) as usize) * 4], + offscreen_buffer, + overlay_state: OverlayState::default(), + capture: None, + }); - let watermark_output_scale = state - .outputs - .first() - .map(|o| o.current_scale().fractional_scale()) - .unwrap_or(1.0); - state - .overlay_state - .load_watermark(&settings.watermark_path, watermark_output_scale); - state.callback = Some(cb); - state.is_capturing = true; - WAYLAND_CAPTURE_ALIVE.store(true, Ordering::Release); - state.render_cursor_on_framebuffer = settings.capture_cursor; - state.settings = settings.clone(); - state.encode_stats.frames.store(0, Ordering::Relaxed); - state.encode_stats.stripes.store(0, Ordering::Relaxed); - state.last_log_time = Instant::now(); - state.frame_counter = 0; - state.pending_force_idr = false; - state.vaapi_state = StripeState::default(); - if state.cursor_callback.is_some() { - if let Some(icon) = state.current_cursor_icon.clone() { - state.send_cursor_image(&icon); - } - } - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - let prior_readback_encoder = state - .encode_join - .take() - .and_then(|j| j.join().ok()) - .flatten(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; - if let Some(cb) = state.callback.take() { - let (tx, rx) = std::sync::mpsc::sync_channel::>(1); - let join = thread::spawn(move || { - crate::boost_thread_priority(-10); - while let Ok(stripes) = rx.recv() { - if PY_SHUTDOWN.load(Ordering::Relaxed) { continue; } - Python::attach(|py| { - for s in stripes { - match Py::new(py, StripeFrame::new_owned_meta( - s.data, s.data_type, s.stripe_y_start, - s.stripe_height, s.frame_id, - )) { - Ok(f) => { if let Err(e) = cb.call1(py, (f,)) { e.print(py); } } - Err(e) => eprintln!("[wayland] frame alloc error: {e:?}"), - } - } - }); - } - }); - state.deliver_tx = Some(tx); - state.deliver_join = Some(join); - } + /// Apply every queued control command in FIFO order. Sends wake the loop through the + /// separate wake channel, and the render tick ALSO drains before starting its work, so + /// queued input is applied ahead of a long render/encode instead of waiting it out. + fn drain_thread_commands(state: &mut AppState) { + let Some(rx) = state.command_rx.take() else { return }; + while let Ok(cmd) = rx.try_recv() { + handle_thread_command(state, cmd); + } + state.command_rx = Some(rx); + } - if state.video_encoder.is_none() { - if let Some(ref deliver_tx) = state.deliver_tx { - let pool = Arc::new(WlFramePool::new( - WL_POOL_SURFACES, - (settings.width.max(0) as usize) * (settings.height.max(0) as usize) * 4, - )); - state.pool_last_render = vec![0; WL_POOL_SURFACES]; - state.render_seq = 0; - // u64::MAX marks every slot stale so each one is read back - // before its first publish, whatever the damage says. - state.pool_content_gen = vec![u64::MAX; WL_POOL_SURFACES]; - state.content_gen = 0; - let c = &state.encode_controls; - c.bitrate_kbps.store(settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(false, Ordering::Relaxed); - c.force_idr.store(false, Ordering::Relaxed); - // Also drop any leftover LiveTunables snapshot: UpdateTunables - // sets it even when no encode thread is consuming (zero-copy - // mode, or after StopCapture), and without this reset the new - // encode thread's first frame would apply that stale snapshot - // OVER the fresh StartCapture settings just established. - c.tunables_dirty.store(false, Ordering::Relaxed); - *c.tunables.lock().unwrap() = None; - let cfg = WlEncodeConfig { - settings: settings.clone(), - use_gpu: state.use_gpu, - try_gpu: gpu_intent && (!state.use_gpu || different_gpu), - prior: prior_readback_encoder, - recording_sink: state.recording_sink.clone(), - deliver_tx: deliver_tx.clone(), - controls: state.encode_controls.clone(), - stats: state.encode_stats.clone(), - }; - let pool2 = pool.clone(); - state.encode_join = Some( - thread::Builder::new() - .name("wl-encode".into()) - .spawn(move || wayland_encode_loop(&pool2, cfg)) - .expect("failed to spawn wl-encode thread"), - ); - state.encode_pool = Some(pool); - } - } else { - state.encode_stats.n_stripes.store(1, Ordering::Relaxed); - *state.encode_stats.desc.lock().unwrap() = - encoder_desc(&settings, state.video_encoder.as_ref(), false, true); - log_stream_settings(&settings, 1, state.video_encoder.as_ref(), false); - } - // Force the keyframe on the now-active path (as RequestIdr does), - // unconditionally: the damage tracker and offscreen buffer stay - // warm across StopCapture/StartCapture, so a restarted capture on - // a static screen otherwise produces no damage, no first frame, - // and no IDR in either the striped or the zero-copy path (a - // stop+start encoder switch froze until the next screen damage). - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); - } else { - state.pending_force_idr = true; - } - state.needs_full_render = true; + fn handle_thread_command(state: &mut AppState, cmd: ThreadCommand) { + match cmd { + ThreadCommand::StartCapture { display_id, callback, settings } => { + start_capture_on_display(state, display_id, callback, settings); + } + ThreadCommand::StopCapture { display_id } => { + // Cursor and clipboard callbacks deliberately SURVIVE StopCapture: + // captures cycle on client disconnects and setting restarts, and a + // copy or cursor change during that gap must still reach Python. + // PY_SHUTDOWN gates every use against a finalizing interpreter. + stop_capture_on_display(state, display_id); + } + ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply } => { + let _ = reply.send(create_output_on(state, id, width, height, x, y, scale)); + } + ThreadCommand::DestroyOutput { id, reply } => { + let _ = reply.send(destroy_output_on(state, id)); + } + ThreadCommand::RepositionOutput { id, x, y, reply } => { + let _ = reply.send(reposition_output_on(state, id, x, y)); + } + ThreadCommand::ListOutputs { reply } => { + let list = state + .output_nodes + .iter() + .map(|n| { + let (w, h) = n + .output + .current_mode() + .map(|m| (m.size.w, m.size.h)) + .unwrap_or((0, 0)); + ( + n.id, + n.pos.0, + n.pos.1, + w, + h, + n.output.current_scale().fractional_scale(), + n.capture.is_some(), + ) + }) + .collect(); + let _ = reply.send(list); } - CalloopEvent::Msg(ThreadCommand::StopCapture) => { - println!("[Wayland] Capture loop stopped."); - state.is_capturing = false; - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Release); - state.callback = None; - state.cursor_callback = None; - state.clipboard_callback = None; - state.video_encoder = None; - state.vaapi_state = StripeState::default(); - if let Some(p) = state.encode_pool.take() { p.shutdown(); } - if let Some(j) = state.encode_join.take() { let _ = j.join(); } - state.recording_sink = None; - *state.encode_stats.desc.lock().unwrap() = String::new(); - if let Some(tx) = state.deliver_tx.take() { drop(tx); } - if let Some(j) = state.deliver_join.take() { let _ = j.join(); } - state.pending_hw_delivery = None; - state.pending_hw_damage = false; + ThreadCommand::MoveWindowToOutput { window_id, output_id, reply } => { + let window = state + .space + .elements() + .find(|w| { + wayland::frontend::window_meta(w) + .map(|m| m.id == window_id) + .unwrap_or(false) + }) + .cloned(); + let ok = match window { + Some(w) => state.place_window_on_output(&w, output_id), + None => false, + }; + let _ = reply.send(ok); + } + ThreadCommand::ListWindows { reply } => { + use smithay::wayland::shell::xdg::XdgToplevelSurfaceData; + let mut list = Vec::new(); + for window in state.space.elements() { + let Some(meta) = wayland::frontend::window_meta(window) else { continue }; + let (title, app_id) = window + .toplevel() + .map(|tl| { + with_states(tl.wl_surface(), |states| { + states + .data_map + .get::() + .map(|d| { + let a = d.lock().unwrap(); + ( + a.title.clone().unwrap_or_default(), + a.app_id.clone().unwrap_or_default(), + ) + }) + .unwrap_or_default() + }) + }) + .unwrap_or_default(); + list.push((meta.id, title, app_id, meta.output.load(Ordering::Relaxed))); + } + let _ = reply.send(list); } - CalloopEvent::Msg(ThreadCommand::SetClipboardCallback(cb)) => { + ThreadCommand::SetClipboardCallback(cb) => { state.clipboard_callback = Some(cb); + // Re-stage a read of the CURRENT selection so a copy made before this + // callback was (re)armed is delivered rather than lost; the post-dispatch + // drain performs the read (a compositor-owned selection is skipped there). + if let Some(mime) = state.current_selection_mime.clone() { + state.pending_clipboard_read = Some(mime); + } } - CalloopEvent::Msg(ThreadCommand::SetClipboard { mime, data }) => { + ThreadCommand::SetClipboard { mime, data } => { let mimes: Vec = if mime.starts_with("text/plain") { ["text/plain;charset=utf-8", "UTF8_STRING", "text/plain", "STRING", "TEXT"].iter().map(|s| s.to_string()).collect() @@ -2076,14 +3517,22 @@ fn run_wayland_thread( mimes, std::sync::Arc::new((mime, data)), ); + // The selection is compositor-owned now; a later SetClipboardCallback + // must not try to re-read a client source that no longer holds it. + state.current_selection_mime = None; } - CalloopEvent::Msg(ThreadCommand::SetCursorCallback(cb)) => { - state.cursor_callback = Some(cb); + ThreadCommand::SetCursorCallback(cb) => { + let _ = state.cursor_tx.send(CursorJob::SetCallback(cb)); + state.cursor_callback_set = true; if let Some(icon) = state.current_cursor_icon.clone() { state.send_cursor_image(&icon); } } - CalloopEvent::Msg(ThreadCommand::KeyboardKey { scancode, state: key_state_val }) => { + ThreadCommand::KeyboardKey { scancode, state: key_state_val } => { + if let Some(host) = state.host.as_ref() { + host.key(scancode, key_state_val > 0); + return; + } let key_state = if key_state_val > 0 { KeyState::Pressed } else { KeyState::Released }; let serial = next_serial(); let time = wayland_time(); @@ -2093,14 +3542,57 @@ fn run_wayland_thread( }); } } - CalloopEvent::Msg(ThreadCommand::SetKeymapString(text)) => { - if let Some(keyboard) = state.seat.get_keyboard() { - if let Err(e) = keyboard.set_keymap_from_string(state, text) { - eprintln!("[Wayland] keymap swap failed: {e:?}"); + ThreadCommand::SetKeymapString(text) => { + // Validate before mutating the policy so a bad string leaves the seat + // keymap untouched. + if crate::wayland::keymap::compile_keymap(&text).is_none() { + eprintln!("[Wayland] set_keymap_string: keymap failed to compile; keeping current keymap."); + } else { + state.keymap_policy.rebuild_base(text); + state.apply_keymap_policy(); + } + } + ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply } => { + match crate::wayland::keymap::compile_rmlvo(&rules, &model, &layout, &variant, &options) { + Some(text) => { + state.keymap_policy.rebuild_base(text); + state.apply_keymap_policy(); + let _ = reply.send(true); + } + None => { + eprintln!("[Wayland] set_xkb_layout: RMLVO ({rules:?}, {model:?}, {layout:?}, {variant:?}, {options:?}) failed to compile."); + let _ = reply.send(false); } } } - CalloopEvent::Msg(ThreadCommand::GetXkbKeymap { reply }) => { + ThreadCommand::BindKeysyms { keysyms, reply } => { + let _ = reply.send(state.bind_keysyms(&keysyms)); + } + ThreadCommand::GetKeyboardState { reply } => { + let (pressed, mods) = state + .seat + .get_keyboard() + .map(|kb| { + let pressed: Vec = + kb.pressed_keys().iter().map(|c| c.raw()).collect(); + let m = kb.modifier_state(); + let mask = (m.ctrl as u32) + | (m.shift as u32) << 1 + | (m.alt as u32) << 2 + | (m.logo as u32) << 3 + | (m.caps_lock as u32) << 4 + | (m.num_lock as u32) << 5 + | (m.iso_level3_shift as u32) << 6 + | (m.iso_level5_shift as u32) << 7; + (pressed, mask) + }) + .unwrap_or_default(); + let _ = reply.send((pressed, mods)); + } + ThreadCommand::Barrier { reply } => { + let _ = reply.send(()); + } + ThreadCommand::GetXkbKeymap { reply } => { let mut keymap_str = String::new(); if let Some(keyboard) = state.seat.get_keyboard() { keymap_str = keyboard.with_xkb_state(state, |context| { @@ -2117,35 +3609,48 @@ fn run_wayland_thread( } let _ = reply.send(keymap_str); } - CalloopEvent::Msg(ThreadCommand::PointerMotion { x, y }) => { + ThreadCommand::PointerMotion { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_motion_abs(x, y); + return; + } let serial = next_serial(); let time = wayland_time(); - let scale = state.settings.scale; - let logical_w = (state.settings.width as f64 / scale).floor(); - let logical_h = (state.settings.height as f64 / scale).floor(); - let logical_x = (x / scale).max(0.0).min(logical_w - 1.0); - let logical_y = (y / scale).max(0.0).min(logical_h - 1.0); + // (x, y) are physical union-layout coordinates: each output occupies + // the physical rectangle at its layout offset, and the point maps + // through the CONTAINING output's scale (clamped into the nearest + // output when outside all of them). + let p = state.layout_physical_to_logical(x, y); if let Some(pointer) = state.seat.get_pointer() { - let p = Point::::from((logical_x, logical_y)); - let mut under = None; - - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - + // Layer surfaces live on the output under the point; their + // geometry is output-local, so hit-test with the local point and + // report the global location. + let layer_hit = |state: &AppState, layers: &[smithay::wayland::shell::wlr_layer::Layer]| { + let idx = state.node_idx_under(p)?; + let node = &state.output_nodes[idx]; + let origin = Point::::from(node.pos); + let local = (p - origin.to_f64()).to_i32_round(); + let layer_map = layer_map_for_output(&node.output); for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Overlay || state_layer == smithay::wayland::shell::wlr_layer::Layer::Top { + if layers.contains(&layer.layer()) { if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; + if bbox.contains(local) { + return Some(( + FocusTarget::LayerSurface(layer.clone()), + (bbox.loc + origin).to_f64(), + )); } } } } - } + None + }; + + let mut under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Overlay, + smithay::wayland::shell::wlr_layer::Layer::Top, + ]); if under.is_none() { under = state.space.element_under(p).map(|(window, loc)| { @@ -2154,44 +3659,26 @@ fn run_wayland_thread( } if under.is_none() { - if let Some(output) = state.outputs.first() { - let layer_map = layer_map_for_output(output); - let pointer_pos = p.to_i32_round(); - - for layer in layer_map.layers().rev() { - let state_layer = layer.layer(); - if state_layer == smithay::wayland::shell::wlr_layer::Layer::Bottom || state_layer == smithay::wayland::shell::wlr_layer::Layer::Background { - if let Some(bbox) = layer_map.layer_geometry(layer) { - if bbox.contains(pointer_pos) { - under = Some((FocusTarget::LayerSurface(layer.clone()), bbox.loc.to_f64())); - break; - } - } - } - } - } + under = layer_hit(state, &[ + smithay::wayland::shell::wlr_layer::Layer::Bottom, + smithay::wayland::shell::wlr_layer::Layer::Background, + ]); } pointer.motion(state, under, &MotionEvent { location: p, serial, time }); pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerRelativeMotion { dx, dy }) => { + ThreadCommand::PointerRelativeMotion { dx, dy } => { let utime = wayland_utime(); let time = wayland_time(); let serial = next_serial(); if let Some(pointer) = state.seat.get_pointer() { let current_pos = pointer.current_location(); - - let scale = state.settings.scale; - let max_w = (state.settings.width as f64 / scale).floor() - 1.0; - let max_h = (state.settings.height as f64 / scale).floor() - 1.0; - - let new_x = (current_pos.x + dx).max(0.0).min(max_w); - let new_y = (current_pos.y + dy).max(0.0).min(max_h); - - let new_pos = Point::::from((new_x, new_y)); + let new_pos = state.clamp_logical( + (current_pos.x + dx, current_pos.y + dy).into(), + ); let under = state.space.element_under(new_pos).map(|(window, loc)| { (FocusTarget::Window(window.clone()), loc.to_f64()) @@ -2217,7 +3704,11 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerButton { btn, state: btn_state_val }) => { + ThreadCommand::PointerButton { btn, state: btn_state_val } => { + if let Some(host) = state.host.as_ref() { + host.pointer_button(btn, btn_state_val > 0); + return; + } let serial = next_serial(); let time = wayland_time(); let button_state = if btn_state_val > 0 { smithay::backend::input::ButtonState::Pressed } else { smithay::backend::input::ButtonState::Released }; @@ -2239,7 +3730,11 @@ fn run_wayland_thread( pointer.frame(state); } } - CalloopEvent::Msg(ThreadCommand::PointerAxis { x, y }) => { + ThreadCommand::PointerAxis { x, y } => { + if let Some(host) = state.host.as_ref() { + host.pointer_axis(x, y); + return; + } let time = wayland_time(); if let Some(pointer) = state.seat.get_pointer() { @@ -2265,66 +3760,120 @@ fn run_wayland_thread( } } } - CalloopEvent::Msg(ThreadCommand::UpdateCursorConfig { render_on_framebuffer }) => { + ThreadCommand::UpdateCursorConfig { render_on_framebuffer } => { state.render_cursor_on_framebuffer = render_on_framebuffer; } - CalloopEvent::Msg(ThreadCommand::RequestIdr) => { - if state.encode_pool.is_some() { - state.encode_controls.force_idr.store(true, Ordering::Relaxed); + ThreadCommand::SetCursorSize { size, reply } => { + if size <= 0 { + let _ = reply.send(false); } else { - state.pending_force_idr = true; + state.cursor_helper = Cursor::load(size); + let _ = state.cursor_tx.send(CursorJob::SetSize(size)); + // The burned-in cursor changed size; force a repaint everywhere so + // a static screen doesn't keep showing the old sprite. + for node in state.output_nodes.iter_mut() { + if let Some(cap) = node.capture.as_mut() { + cap.needs_full_render = true; + } + } + let _ = reply.send(true); } - state.needs_full_render = true; } - CalloopEvent::Msg(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) => { - if let Some(b) = bitrate_kbps { state.settings.video_bitrate_kbps = b; } - if let Some(v) = vbv_multiplier { state.settings.video_vbv_multiplier = v; } - if let Some(f) = fps { if f > 0.0 { state.settings.target_fps = f; } } - if let Some(GpuEncoder::Nvenc(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); + ThreadCommand::RequestIdr { display_id } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + cap.request_idr(); + } } - if let Some(GpuEncoder::Vaapi(enc)) = state.video_encoder.as_mut() { - enc.reconfigure_rate(&state.settings); + } + ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps } => { + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + if let Some(b) = bitrate_kbps { cap.settings.video_bitrate_kbps = b; } + if let Some(v) = vbv_multiplier { cap.settings.video_vbv_multiplier = v; } + if let Some(f) = fps { if f > 0.0 { cap.settings.target_fps = f; } } + if let Some(GpuEncoder::Nvenc(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); + } + if let Some(GpuEncoder::Vaapi(enc)) = cap.video_encoder.as_mut() { + enc.reconfigure_rate(&cap.settings); + } + let c = &cap.encode_controls; + c.bitrate_kbps.store(cap.settings.video_bitrate_kbps, Ordering::Relaxed); + c.vbv_mult_milli.store( + (cap.settings.video_vbv_multiplier * 1000.0).round() as i32, + Ordering::Relaxed, + ); + c.fps_milli.store( + (cap.settings.target_fps.max(1.0) * 1000.0) as u64, + Ordering::Relaxed, + ); + c.rate_dirty.store(true, Ordering::Release); + if display_id == 0 { + state.settings.video_bitrate_kbps = cap.settings.video_bitrate_kbps; + state.settings.video_vbv_multiplier = cap.settings.video_vbv_multiplier; + state.settings.target_fps = cap.settings.target_fps; + } + } } - let c = &state.encode_controls; - c.bitrate_kbps.store(state.settings.video_bitrate_kbps, Ordering::Relaxed); - c.vbv_mult_milli.store( - (state.settings.video_vbv_multiplier * 1000.0).round() as i32, - Ordering::Relaxed, - ); - c.fps_milli.store( - (state.settings.target_fps.max(1.0) * 1000.0) as u64, - Ordering::Relaxed, - ); - c.rate_dirty.store(true, Ordering::Release); } - CalloopEvent::Msg(ThreadCommand::UpdateTunables(t)) => { - t.apply_to(&mut state.settings); + ThreadCommand::UpdateTunables { display_id, tunables: t } => { state.render_cursor_on_framebuffer = t.capture_cursor; - *state.encode_controls.tunables.lock().unwrap() = Some(t); - state.encode_controls.tunables_dirty.store(true, Ordering::Release); + if display_id == 0 { + t.apply_to(&mut state.settings); + } + if let Some(idx) = state.node_idx_for_id(display_id) { + if let Some(cap) = state.output_nodes[idx].capture.as_mut() { + t.apply_to(&mut cap.settings); + *cap.encode_controls.tunables.lock().unwrap() = Some(t); + cap.encode_controls.tunables_dirty.store(true, Ordering::Release); + } + } } - CalloopEvent::Msg(ThreadCommand::CuScreenshot { resp }) => { - state.pending_screenshot = Some(resp); + ThreadCommand::CuScreenshot { display_id, resp } => { + if state.node_idx_for_id(display_id).is_some() { + state.pending_screenshot = Some((display_id, resp)); + } else { + let _ = resp.send(Err(format!("Unknown display: {display_id}"))); + } } - CalloopEvent::Msg(ThreadCommand::CuCursorPosition { resp }) => { + ThreadCommand::CuCursorPosition { resp } => { let pos = state.seat.get_pointer() .map(|p| p.current_location()) .unwrap_or_else(|| (0.0f64, 0.0f64).into()); - let scale = state.settings.scale; - let fb_x = pos.x * scale; - let fb_y = pos.y * scale; - let _ = resp.send((fb_x, fb_y)); + let _ = resp.send(state.layout_logical_to_physical(pos)); } - CalloopEvent::Msg(ThreadCommand::CuGetInfo { resp }) => { - let _ = resp.send(( - state.settings.width, - state.settings.height, - state.settings.scale, - )); + ThreadCommand::CuGetInfo { display_id, resp } => { + let info = state + .node_idx_for_id(display_id) + .map(|idx| { + let node = &state.output_nodes[idx]; + match node.capture.as_ref() { + Some(c) => (c.settings.width, c.settings.height, c.settings.scale), + None => node + .output + .current_mode() + .map(|m| { + ( + m.size.w, + m.size.h, + node.output.current_scale().fractional_scale(), + ) + }) + .unwrap_or((0, 0, 0.0)), + } + }) + .unwrap_or((0, 0, 0.0)); + let _ = resp.send(info); } - CalloopEvent::Closed => {} } + } + + state.command_rx = Some(command_rx); + event_loop + .handle() + .insert_source(wake_rx, |_, _, state| { + drain_thread_commands(state); }) .unwrap(); @@ -2332,6 +3881,7 @@ fn run_wayland_thread( let socket_name = source.socket_name().to_string_lossy().into_owned(); println!("[Wayland] Socket listening on: {:?}", socket_name); std::env::set_var("WAYLAND_DISPLAY", &socket_name); + publish_socket_name(&socket_name); event_loop .handle() @@ -2347,574 +3897,112 @@ fn run_wayland_thread( let timer = Timer::immediate(); let mut is_memory_throttling = false; - event_loop - .handle() - .insert_source(timer, move |_, _, state| { - let loop_start_time = Instant::now(); - state.space.refresh(); - - let current_rss = get_process_rss_bytes(); - let shm_usage = get_shm_usage_bytes(); - let memory_threshold = calculate_memory_threshold(state.settings.width, state.settings.height); - - if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { - if !is_memory_throttling { - is_memory_throttling = true; - } - } else if is_memory_throttling - && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { - is_memory_throttling = false; - } - - let now = Instant::now(); - let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); - if elapsed >= 1.0 { - let frames = state.encode_stats.frames.swap(0, Ordering::Relaxed); - let stripes = state.encode_stats.stripes.swap(0, Ordering::Relaxed); - if state.is_capturing && state.settings.debug_logging { - let actual_fps = frames as f64 / elapsed; - let stripes_per_sec = stripes as f64 / elapsed; - let mode_str = state.encode_stats.desc.lock().unwrap().clone(); - let n_stripes = state.encode_stats.n_stripes.load(Ordering::Relaxed); - - let rss_mb = current_rss / 1024 / 1024; - let shm_mb = shm_usage / 1024 / 1024; - let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; - - println!("Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", - state.settings.width, state.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); - } - state.last_log_time = now; - } - - if !state.is_capturing && state.pending_screenshot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(16)); - } - - let requested_idr = state.pending_force_idr; - - let mut pool_slot: Option<(usize, Vec)> = None; - if let Some(ref pool) = state.encode_pool { - if !is_memory_throttling { - pool_slot = pool.try_begin(); - if pool_slot.is_none() { - return TimeoutAction::ToDuration(Duration::from_millis(1)); - } - } - } - - state.overlay_state.update_position( - state.settings.width, - state.settings.height, - state.settings.watermark_location_enum, - ); - - let mut render_success = false; - let mut render_sync = None; - let mut damage_rects: Vec> = Vec::new(); - let width = state.settings.width; - let height = state.settings.height; - - if let Some(output) = state.outputs.first().cloned() { - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { 1 }; - if state.use_gpu { - if let Some(renderer) = state.gles_renderer.as_mut() { - if let Some((_bo, dmabuf)) = state.offscreen_buffer.as_mut() { - match renderer.bind(dmabuf) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers().rev() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let window_loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); - let popup_pos = window_loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - - elements.extend(window.render_elements(renderer, window_loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut GlesRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - if let Some(damage) = result.damage { - damage_rects = damage.clone(); - } - render_sync = Some(result.sync); - state.needs_full_render = false; - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - if !damage_rects.is_empty() { - state.content_gen += 1; - } - if let Some((id, ref mut buf)) = pool_slot { - // No-damage ticks skip the readback, so a pooled - // buffer can lag the offscreen target whenever the - // encoder held the other slot across a tick. Publishing - // a lagging buffer would let a paint-over / burst / - // recovery send re-encode pre-damage pixels; one - // catch-up readback keeps every published buffer - // current while idle readbacks stay skipped. - if render_success && state.pool_content_gen[id] != state.content_gen { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - buf.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - state.pool_content_gen[id] = state.content_gen; - } - } else if state.pending_screenshot.is_some() { - let _ = renderer.with_context(|gl| unsafe { - gl.ReadPixels( - 0, - 0, - width, - height, - smithay::backend::renderer::gles::ffi::RGBA, - smithay::backend::renderer::gles::ffi::UNSIGNED_BYTE, - state.frame_buffer.as_mut_ptr() as *mut std::ffi::c_void, - ); - }); - } - }, - Err(e) => eprintln!("Failed to bind buffer: {:?}", e) - } - } - } - } else { - if let Some(renderer) = state.pixman_renderer.as_mut() { - let (ptr, buf_age) = match pool_slot { - Some((id, ref mut buf)) => { - let age = if state.pool_last_render[id] == 0 { - 0 - } else { - (state.render_seq + 1 - state.pool_last_render[id]) as usize - }; - (buf.as_mut_ptr() as *mut u32, age) - } - None => (state.frame_buffer.as_mut_ptr() as *mut u32, 0), - }; - let mut image = unsafe { - pixman::Image::from_raw_mut(pixman::FormatCode::A8R8G8B8, width as usize, height as usize, ptr, (width as usize) * 4, false).expect("Failed to create pixman image") - }; - match renderer.bind(&mut image) { - Ok(mut frame) => { - let mut elements: Vec>> = Vec::new(); - - if state.render_cursor_on_framebuffer { - if let Some(pointer) = state.seat.get_pointer() { - let pos = pointer.current_location(); - let output_scale_val = output.current_scale().fractional_scale(); - let scale = Scale::from(output_scale_val); - - if let Some(CursorImageStatus::Named(icon)) = &state.current_cursor_icon { - let name = wayland::frontend::cursor_icon_to_str(icon); - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - if let Some(image) = state.cursor_helper.get_image_by_name(name, output_scale_val.round() as u32, time) { - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } else if let Some(CursorImageStatus::Surface(surface)) = &state.current_cursor_icon { - let phys_pos = pos.to_physical(scale); - let elem_result = with_states(surface, |states| { - WaylandSurfaceRenderElement::from_surface(renderer, surface, states, phys_pos, 1.0, smithay::backend::renderer::element::Kind::Cursor) - }); - if let Ok(Some(cursor_elem)) = elem_result { - elements.push(CompositionElements::Surface(cursor_elem)); - } - } else if state.current_cursor_icon.is_none() { - let time = Duration::from_millis(state.clock.now().as_millis() as u64); - let image = state.cursor_helper.get_image(output_scale_val.round() as u32, time); - if let Some(elem) = state.overlay_state.get_cursor_element(renderer, image, pos.to_i32_round()) { - elements.push(CompositionElements::Cursor(elem)); - } - } - } - } - - if let Some(elem) = state.overlay_state.get_watermark_element(renderer) { - elements.push(CompositionElements::Cursor(elem)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; - - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Overlay); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Top); - } - - for window in state.space.elements_for_output(&output).collect::>().into_iter().rev() { - let loc = state.space.element_location(window).unwrap_or_default(); - - if let Some(surface) = window.wl_surface() { - let popups = PopupManager::popups_for_surface(&surface); - for (popup, location) in popups { - let popup_surface = popup.wl_surface(); { - let popup_pos = loc + location; - let elem = smithay::wayland::compositor::with_states(popup_surface, |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, - popup_surface, - states, - popup_pos.to_physical_precise_round(1), - 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - - elements.extend(window.render_elements(renderer, loc.to_physical_precise_round(1), Scale::from(1.0), 1.0).into_iter().map(CompositionElements::Space)); - } - - { - let layer_map = layer_map_for_output(&output); - - let draw_layer = |renderer: &mut PixmanRenderer, elements: &mut Vec>>, target_layer: smithay::wayland::shell::wlr_layer::Layer| { - for surface in layer_map.layers() { - let current_layer = surface.layer(); - if current_layer == target_layer { - if let Some(geo) = layer_map.layer_geometry(surface) { - let elem = smithay::wayland::compositor::with_states(surface.wl_surface(), |states| { - WaylandSurfaceRenderElement::from_surface( - renderer, surface.wl_surface(), states, - geo.loc.to_physical_precise_round(1), 1.0, - smithay::backend::renderer::element::Kind::Unspecified - ) - }); - if let Ok(Some(e)) = elem { - elements.push(CompositionElements::Surface(e)); - } - } - } - } - }; + event_loop + .handle() + .insert_source(timer, move |_, _, state| { + // Apply queued commands (input above all) BEFORE the render/encode work: a + // command that raced the timer wakeup would otherwise wait out the whole tick. + drain_thread_commands(state); + let loop_start_time = Instant::now(); + state.space.refresh(); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Bottom); - draw_layer(renderer, &mut elements, smithay::wayland::shell::wlr_layer::Layer::Background); - } + let current_rss = get_process_rss_bytes(); + let shm_usage = get_shm_usage_bytes(); + // The threshold follows the largest active capture (fallback: primary settings). + let (max_w, max_h) = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| (c.settings.width, c.settings.height))) + .fold((state.settings.width, state.settings.height), |acc, (w, h)| { + (acc.0.max(w), acc.1.max(h)) + }); + let memory_threshold = calculate_memory_threshold(max_w, max_h); - let render_age = if state.overlay_state.is_animated() || state.needs_full_render { 0 } else { buf_age }; - match damage_tracker.render_output(renderer, &mut frame, render_age, &elements, [0.1, 0.1, 0.1, 1.0]) { - Ok(result) => { - render_success = true; - state.needs_full_render = false; - if let Some(damage) = result.damage { damage_rects = damage.clone(); } - }, - Err(e) => eprintln!("Render error: {:?}", e) - } - state.render_seq += 1; - if render_success { - if let Some((id, _)) = pool_slot { - state.pool_last_render[id] = state.render_seq; - } - } - }, - Err(e) => eprintln!("Failed to bind pixman image: {:?}", e) - } - } + if current_rss > memory_threshold || shm_usage > (4 * 1024 * 1024 * 1024) { + if !is_memory_throttling { + is_memory_throttling = true; + } + } else if is_memory_throttling + && current_rss < (memory_threshold as f64 * 0.75) as usize && shm_usage < (3 * 1024 * 1024 * 1024) { + is_memory_throttling = false; } - if render_success { - let time = state.clock.now(); - for window in state.space.elements() { - window.send_frame(&output, time, Some(Duration::ZERO), |_, _| Some(output.clone())); + let now = Instant::now(); + let elapsed = now.duration_since(state.last_log_time).as_secs_f64(); + if elapsed >= 1.0 { + for node in &state.output_nodes { + let Some(cap) = node.capture.as_ref() else { continue }; + let frames = cap.encode_stats.frames.swap(0, Ordering::Relaxed); + let stripes = cap.encode_stats.stripes.swap(0, Ordering::Relaxed); + if cap.settings.debug_logging { + let actual_fps = frames as f64 / elapsed; + let stripes_per_sec = stripes as f64 / elapsed; + let mode_str = cap.encode_stats.desc.lock().unwrap().clone(); + let n_stripes = cap.encode_stats.n_stripes.load(Ordering::Relaxed); + + let rss_mb = current_rss / 1024 / 1024; + let shm_mb = shm_usage / 1024 / 1024; + let throttle_warn = if is_memory_throttling { " [THROTTLED]" } else { "" }; + + println!("Display: {} Res: {}x{} Mode: {} Stripes: {} EncFPS: {:.2} EncStripes/s: {:.2} Mem: {}MB SHM: {}MB{}", + node.id, cap.settings.width, cap.settings.height, mode_str, n_stripes, actual_fps, stripes_per_sec, rss_mb, shm_mb, throttle_warn); } + } + state.last_log_time = now; + } - if is_memory_throttling { - if state.encode_pool.is_none() { - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if state.encode_pool.is_some() { - if state.pending_screenshot.is_some() { - if let Some((_, ref buf)) = pool_slot { - let n = buf.len().min(state.frame_buffer.len()); - state.frame_buffer[..n].copy_from_slice(&buf[..n]); - } - } - if let Some((id, buf)) = pool_slot.take() { - let is_animated = state.overlay_state.is_animated(); - state.encode_pool.as_ref().unwrap().publish(WlFrame { - id, - buf, - frame_id: state.frame_counter, - damage: std::mem::take(&mut damage_rects), - is_animated, - }); - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } else if let Some(ref mut encoder) = state.video_encoder { - // Deliver the parked frame (if any) first, WITHOUT blocking: - // this runs on the calloop thread, and a blocking send would - // freeze input/command/Wayland dispatch for as long as the - // Python consumer stalls (the readback path offloads its - // sends to the wl-encode thread for exactly this reason). - // While a frame stays parked, no new frame is encoded — an - // encoded frame joins the H.264 reference chain and can - // never be dropped — and the tick's damage is latched so - // the pause never loses a change. - let slot_free = match state.pending_hw_delivery.take() { - None => true, - Some(pending) => match state.deliver_tx.as_ref() { - None => true, - Some(tx) => match tx.try_send(pending) { - Ok(()) => true, - Err(std::sync::mpsc::TrySendError::Full(p)) => { - state.pending_hw_delivery = Some(p); - false - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => true, - }, - }, - }; - if !slot_free { - if !damage_rects.is_empty() { - state.pending_hw_damage = true; - } - } else { - let is_animated = state.overlay_state.is_animated(); - let had_damage = !damage_rects.is_empty() - || std::mem::take(&mut state.pending_hw_damage); - let decision = crate::pipeline::decide_hw_fullframe( - &mut state.vaapi_state, - &state.settings, - state.frame_counter, - had_damage, - is_animated, - requested_idr, - ); - let send_frame = decision.send; - let force_idr = decision.force_idr; - let target_qp = decision.target_qp; - - if send_frame { - if let Some(sync) = render_sync.take() { - let _ = sync.wait(); - } - let force_idr_for_recording = state - .recording_sink - .as_ref() - .map(|s| s.should_force_idr()) - .unwrap_or(false); - let force_idr = force_idr || force_idr_for_recording; - let result = match encoder { - GpuEncoder::Nvenc(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("NVENC ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - }, - GpuEncoder::Vaapi(enc) => { - if let Some((_, ref dmabuf)) = state.offscreen_buffer { - enc.encode_dmabuf(dmabuf, state.frame_counter as u64, target_qp, force_idr) - } else { - Err("Vaapi ZeroCopy requires offscreen buffer (GPU context)".to_string()) - } - } - }; - - if let Ok(data) = result { - if !data.is_empty() { - state.encode_stats.frames.fetch_add(1, Ordering::Relaxed); - state.encode_stats.stripes.fetch_add(1, Ordering::Relaxed); - if let Some(ref tx) = state.deliver_tx { - let stripes = vec![EncodedStripe { - data, data_type: 2, stripe_y_start: 0, - stripe_height: height, frame_id: state.frame_counter as i32, - }]; - if let Some(ref socket) = state.recording_sink { - for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); - } - } - // Non-blocking: a full slot parks the frame - // (delivered ahead of any new encode above). - match tx.try_send(stripes) { - Ok(()) => {} - Err(std::sync::mpsc::TrySendError::Full(s)) => { - state.pending_hw_delivery = Some(s); - } - Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {} - } - } - } - } else if let Err(e) = result { - eprintln!("HW Encode Error: {}", e); - } - } - state.pending_force_idr = false; - state.frame_counter = state.frame_counter.wrapping_add(1); - } - } - if let Some(resp) = state.pending_screenshot.take() { - if !state.frame_buffer.is_empty() { - let w = state.settings.width as u32; - let h = state.settings.height as u32; - let png = if state.use_gpu { - crate::computer_use::encode_png_rgba(&state.frame_buffer, w, h) - } else { - let mut rgba = state.frame_buffer.clone(); - for px in rgba.chunks_exact_mut(4) { - px.swap(0, 2); - } - crate::computer_use::encode_png_rgba(&rgba, w, h) - }; - match png { - Ok(data) => { let _ = resp.send(data); } - Err(e) => { let _ = resp.send(Vec::new()); eprintln!("[ComputerUse] PNG encode error: {}", e); } - } - } else { - let _ = resp.send(Vec::new()); - } + let any_capturing = state.output_nodes.iter().any(|n| n.capture.is_some()); + if !any_capturing && state.pending_screenshot.is_none() { + // No render/encode work, but committed clients still need their + // frame callbacks: a vsynced client (FIFO Vulkan present, games, a + // nested compositor's own clients) otherwise blocks in its swap + // until a viewer attaches — apps appear frozen whenever nobody is + // watching. + let time = state.clock.now(); + for node in &state.output_nodes { + for window in state + .space + .elements_for_output(&node.output) + .cloned() + .collect::>() + { + window.send_frame(&node.output, time, Some(Duration::ZERO), |_, _| { + Some(node.output.clone()) + }); } } + return TimeoutAction::ToDuration(Duration::from_millis(16)); } - if let Some((id, buf)) = pool_slot.take() { - if let Some(ref pool) = state.encode_pool { - pool.cancel(id, buf); + + // Render every output that needs it. The nodes are taken out of the state so + // each per-output render can borrow the shared renderer/space alongside its + // own damage tracker and buffers. + let mut nodes = std::mem::take(&mut state.output_nodes); + let mut any_pool_busy = false; + for node in nodes.iter_mut() { + if render_node_tick(state, node, is_memory_throttling) { + any_pool_busy = true; } } + state.output_nodes = nodes; + + if any_pool_busy { + return TimeoutAction::ToDuration(Duration::from_millis(1)); + } let work_elapsed = loop_start_time.elapsed(); - let fps = (if is_memory_throttling { 5.0 } else { state.settings.target_fps }).max(1.0); + let max_fps = state + .output_nodes + .iter() + .filter_map(|n| n.capture.as_ref().map(|c| c.settings.target_fps)) + .fold(0.0f64, f64::max); + let fps = (if is_memory_throttling { + 5.0 + } else if max_fps > 0.0 { + max_fps + } else { + state.settings.target_fps + }) + .max(1.0); let target_frame_duration = Duration::from_secs_f64(1.0 / fps); let wait_duration = target_frame_duration.saturating_sub(work_elapsed); let final_wait = if wait_duration.as_millis() < 1 { Duration::from_millis(1) } else { wait_duration }; @@ -2930,17 +4018,8 @@ fn run_wayland_thread( }) .unwrap(); - if let Ok(port_str) = std::env::var("PIXELFLUX_CU") { - if let Ok(port) = port_str.parse::() { - println!("[ComputerUse] Starting server on port {} (PIXELFLUX_CU={})", port, port); - let cu_tx = command_tx.clone(); - thread::spawn(move || { - crate::computer_use::run_cu_server(cu_tx, port); - }); - } else { - println!("[ComputerUse] Invalid PIXELFLUX_CU value: '{}' - expected a port number", port_str); - } - } + crate::computer_use::register_wayland_backend(command_tx.clone()); + crate::computer_use::spawn_cu_from_env(); event_loop.run(None, &mut state, |state| { state.process_pending_clipboard_read(); @@ -2954,7 +4033,7 @@ fn run_wayland_thread( /// four stripe-metadata ints as Python attributes. #[pyclass] struct StripeFrame { - data: Vec, + data: Arc>, #[pyo3(get, set)] data_type: i32, #[pyo3(get, set)] @@ -2966,10 +4045,10 @@ struct StripeFrame { } impl StripeFrame { - /// Hot-path constructor: MOVES the encoded buffer in (no copy) and carries stripe + /// Hot-path constructor: shares the encoder's buffer by `Arc` (no copy) and carries stripe /// metadata as attributes, so the consumer can read it without parsing a header /// (required for omit_stripe_headers). - fn new_owned_meta(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { + fn new_owned_meta(data: Arc>, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { Self { data, data_type, stripe_y_start, stripe_height, frame_id } } } @@ -2981,7 +4060,7 @@ impl StripeFrame { #[new] #[pyo3(signature = (data, data_type = 0, stripe_y_start = 0, stripe_height = 0, frame_id = 0))] fn new(data: Vec, data_type: i32, stripe_y_start: i32, stripe_height: i32, frame_id: i32) -> Self { - Self { data, data_type, stripe_y_start, stripe_height, frame_id } + Self { data: Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id } } fn __len__(&self) -> usize { @@ -3022,6 +4101,18 @@ impl StripeFrame { #[pyclass] struct WaylandBackend { tx: smithay::reexports::calloop::channel::Sender, + /// Wakes the calloop after each command send: the command channel itself is drained in + /// place by the compositor thread (render tick and wake handler), not registered as its + /// own source, so input never waits behind an in-flight render tick's timer wakeup. + wake_tx: smithay::reexports::calloop::channel::Sender<()>, +} + +impl WaylandBackend { + fn send(&self, cmd: ThreadCommand) -> Result<(), String> { + self.tx.send(cmd).map_err(|e| e.to_string())?; + let _ = self.wake_tx.send(()); + Ok(()) + } } #[pymethods] @@ -3040,61 +4131,155 @@ impl WaylandBackend { cursor_size: i32, ) -> Self { let (tx, rx) = smithay::reexports::calloop::channel::channel(); + let (wake_tx, wake_rx) = smithay::reexports::calloop::channel::channel(); let cu_tx = tx.clone(); thread::spawn(move || { crate::boost_thread_priority(-15); - run_wayland_thread(rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); + run_wayland_thread(rx, wake_rx, cu_tx, width, height, dri_node, auto_gpu_selected, cursor_size); }); - WaylandBackend { tx } + WaylandBackend { tx, wake_tx } } - /// Begin a capture with the given frame callback and settings. + /// Begin a capture with the given frame callback and settings. The target display is + /// the settings' `display_id` attribute (absent = 0, the primary); each display id runs + /// at most one capture, independent of every other display's. /// /// Issuing the start also clears the interpreter-teardown gate: starting from Python proves the /// interpreter is live again after a manual atexit sweep. fn start_capture(&self, callback: Py, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rust_settings = extract_settings(settings)?; + let display_id = read_display_id(settings); PY_SHUTDOWN.store(false, Ordering::Relaxed); - self.tx - .send(ThreadCommand::StartCapture(callback, rust_settings)) + self.send(ThreadCommand::StartCapture { display_id, callback: Some(callback), settings: rust_settings }) .map_err(|e| PyErr::new::(format!("Failed to send start command: {}", e)))?; Ok(()) } - fn stop_capture(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::StopCapture) + /// Stop the capture bound to `display_id` (default: the primary display). + #[pyo3(signature = (display_id = 0))] + fn stop_capture(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::StopCapture { display_id }) .map_err(|e| PyErr::new::(format!("Failed to send stop command: {}", e)))?; Ok(()) } + /// Create an additional output (`WxH` physical pixels at fractional `scale`) mapped into + /// the layout at offset `(x, y)`; `id` is the display key used by every per-display API. + /// False when the id is taken, the geometry/scale is invalid, the rectangle overlaps a + /// live output, or the GPU render target cannot be allocated. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::CreateOutput { id, width, height, x, y, scale, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to create output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Destroy a secondary output: its capture ends cleanly and its windows relocate to the + /// primary output. False for the primary (id 0) or an unknown id. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::DestroyOutput { id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to destroy output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Move an existing output (the primary, id 0, included) to layout offset `(x, y)`; + /// its windows, absolute input injection, and cursor compositing follow. False for an + /// unknown id or a destination overlapping a live output. Capture reconfigures + /// (`start_capture` on an existing output) resize without this validation, so a + /// multi-step relayout must stay overlap-free at every step by caller ordering. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::RepositionOutput { id, x, y, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to reposition output: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every live output as `(id, x, y, width, height, scale, capturing)` — width/height in + /// physical pixels, `(x, y)` the layout offset. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list outputs: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// Move the window with the given id onto output `output_id`, fullscreened at that + /// output's logical size. False for an unknown window or output id. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::MoveWindowToOutput { window_id, output_id, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to move window: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Every mapped window as `(window_id, title, app_id, output_id)`. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel(); + self.send(ThreadCommand::ListWindows { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to list windows: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + fn set_cursor_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetCursorCallback(callback)) + self.send(ThreadCommand::SetCursorCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set cursor callback: {}", e)))?; Ok(()) } + /// Recreate the cursor theme at `size` pixels — no restart: subsequent named-cursor + /// callbacks and the burned-in cursor overlay render at the new size. False for a + /// non-positive size. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetCursorSize { size, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set cursor size: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + /// cb(mime: str, data: bytes) fires when a client app copies to the clipboard. fn set_clipboard_callback(&self, callback: Py) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboardCallback(callback)) + self.send(ThreadCommand::SetClipboardCallback(callback)) .map_err(|e| PyErr::new::(format!("Failed to set clipboard callback: {}", e)))?; Ok(()) } /// Compositor-side clipboard offer: serve `data` as `mime` to pasting clients. fn set_clipboard(&self, mime: String, data: Vec) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetClipboard { mime, data }) + self.send(ThreadCommand::SetClipboard { mime, data }) .map_err(|e| PyErr::new::(format!("Failed to set clipboard: {}", e)))?; Ok(()) } fn inject_key(&self, scancode: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::KeyboardKey { scancode, state }) + self.send(ThreadCommand::KeyboardKey { scancode, state }) .map_err(|e| PyErr::new::(format!("Failed to inject key: {}", e)))?; Ok(()) } @@ -3103,8 +4288,7 @@ impl WaylandBackend { /// owns keysym-to-keycode policy: define keycodes here, then press them via /// `inject_key`. Ordered with key events on the one compositor channel. fn set_keymap_string(&self, text: String) -> PyResult<()> { - self.tx - .send(ThreadCommand::SetKeymapString(text)) + self.send(ThreadCommand::SetKeymapString(text)) .map_err(|e| PyErr::new::(format!("Failed to set keymap: {}", e)))?; Ok(()) } @@ -3117,8 +4301,7 @@ impl WaylandBackend { /// and an empty string is returned when the keymap cannot be read in time. fn get_xkb_keymap_string(&self, py: Python<'_>) -> PyResult { let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); - self.tx - .send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) + self.send(ThreadCommand::GetXkbKeymap { reply: reply_tx }) .map_err(|e| PyErr::new::(format!("Failed to request keymap: {}", e)))?; let result = py.detach(move || reply_rx.recv_timeout(Duration::from_secs(2))); match result { @@ -3128,36 +4311,31 @@ impl WaylandBackend { } fn inject_mouse_move(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerMotion { x, y }) + self.send(ThreadCommand::PointerMotion { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject motion: {}", e)))?; Ok(()) } fn inject_relative_mouse_move(&self, dx: f64, dy: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerRelativeMotion { dx, dy }) + self.send(ThreadCommand::PointerRelativeMotion { dx, dy }) .map_err(|e| PyErr::new::(format!("Failed to inject relative motion: {}", e)))?; Ok(()) } fn inject_mouse_button(&self, btn: u32, state: u32) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerButton { btn, state }) + self.send(ThreadCommand::PointerButton { btn, state }) .map_err(|e| PyErr::new::(format!("Failed to inject button: {}", e)))?; Ok(()) } fn inject_mouse_scroll(&self, x: f64, y: f64) -> PyResult<()> { - self.tx - .send(ThreadCommand::PointerAxis { x, y }) + self.send(ThreadCommand::PointerAxis { x, y }) .map_err(|e| PyErr::new::(format!("Failed to inject axis: {}", e)))?; Ok(()) } fn set_cursor_rendering(&self, enabled: bool) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) + self.send(ThreadCommand::UpdateCursorConfig { render_on_framebuffer: enabled }) .map_err(|e| PyErr::new::(format!("Failed to set cursor config: {}", e)))?; Ok(()) } @@ -3166,31 +4344,100 @@ impl WaylandBackend { /// or a decoder reset can resume immediately. With the default infinite GOP this /// is the only recovery path, so every consumer that can lose decoder state must /// call it. No-op cost on the JPEG/software path (keyframes are N/A). - fn request_idr_frame(&self) -> PyResult<()> { - self.tx - .send(ThreadCommand::RequestIdr) + #[pyo3(signature = (display_id = 0))] + fn request_idr_frame(&self, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::RequestIdr { display_id }) .map_err(|e| PyErr::new::(format!("Failed to request IDR: {}", e)))?; Ok(()) } - /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the running capture. - fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateRate { bitrate_kbps, vbv_multiplier, fps }) + /// Apply a live bitrate (kbps) / VBV (kb) / framerate change to the given display's + /// running capture. + #[pyo3(signature = (bitrate_kbps = None, vbv_multiplier = None, fps = None, display_id = 0))] + fn update_rate(&self, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, display_id: u32) -> PyResult<()> { + self.send(ThreadCommand::UpdateRate { display_id, bitrate_kbps, vbv_multiplier, fps }) .map_err(|e| PyErr::new::(format!("Failed to update rate: {}", e)))?; Ok(()) } + + /// Set the seat's BASE xkb layout from RMLVO names at runtime (empty strings select the + /// xkbcommon defaults). Returns whether the layout compiled and was applied; overlay binds + /// rebuild on top with their keycodes unchanged. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::(); + self.send(ThreadCommand::SetXkbLayout { rules, model, layout, variant, options, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to set layout: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or(false)) + } + + /// Resolve `keysyms` to `(keycode, level)` pairs against the seat keymap, overlay-binding + /// every keysym the base cannot produce. Binding N new keysyms costs ONE keymap swap, and a + /// keycode currently held down is never rebound. `(0, 0)` marks an unbindable keysym. + fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { + let n = keysyms.len(); + let (reply_tx, reply_rx) = std::sync::mpsc::channel::>(); + self.send(ThreadCommand::BindKeysyms { keysyms, reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to bind keysyms: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_else(|_| vec![(0, 0); n])) + } + + /// Debug/verification readback of the seat keyboard: `(pressed_keycodes, modifier_mask)` + /// with mask bits 1 ctrl, 2 shift, 4 alt, 8 logo, 16 caps, 32 num, 64 altgr, 128 level5. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(Vec, u32)>(); + self.send(ThreadCommand::GetKeyboardState { reply: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read keyboard state: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or_default()) + } + + /// The capture geometry actually live on the given display: `(width, height, scale)` in + /// physical pixels. Reflects any degrade a `start_capture` performed (H.264 even-masking, + /// GBM allocation failure keeping the previous mode), and the command channel is FIFO, so + /// calling this after `start_capture` returns what that start realized. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + let (reply_tx, reply_rx) = std::sync::mpsc::channel::<(i32, i32, f64)>(); + self.send(ThreadCommand::CuGetInfo { display_id, resp: reply_tx }) + .map_err(|e| PyErr::new::(format!("Failed to read geometry: {}", e)))?; + Ok(py + .detach(move || reply_rx.recv_timeout(Duration::from_secs(2))) + .unwrap_or((0, 0, 0.0))) + } } impl WaylandBackend { - fn update_tunables(&self, t: LiveTunables) -> PyResult<()> { - self.tx - .send(ThreadCommand::UpdateTunables(t)) + fn update_tunables(&self, display_id: u32, t: LiveTunables) -> PyResult<()> { + self.send(ThreadCommand::UpdateTunables { display_id, tunables: t }) .map_err(|e| PyErr::new::(format!("Failed to update tunables: {}", e)))?; Ok(()) } } +/// The optional `display_id` attribute on a settings object (absent/invalid = 0, the +/// primary display). +fn read_display_id(settings: &Bound<'_, PyAny>) -> u32 { + settings + .getattr("display_id") + .ok() + .and_then(|v| v.extract::().ok()) + .unwrap_or(0) +} + use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; use std::sync::{Condvar, Mutex, OnceLock}; @@ -3210,7 +4457,7 @@ fn stripe_frame_from_buffer( stripe_height: i32, frame_id: i32, ) -> StripeFrame { - StripeFrame::new_owned_meta(data, data_type, stripe_y_start, stripe_height, frame_id) + StripeFrame::new_owned_meta(Arc::new(data), data_type, stripe_y_start, stripe_height, frame_id) } /// Capture configuration read by `start_capture` (each field by attribute name via @@ -3218,6 +4465,8 @@ fn stripe_frame_from_buffer( /// can stash extra attributes not listed here. #[pyclass(dict)] struct CaptureSettings { + /// Wayland display key this capture binds to (0 = primary output); ignored on X11. + #[pyo3(get, set)] display_id: u32, #[pyo3(get, set)] capture_width: i32, #[pyo3(get, set)] capture_height: i32, #[pyo3(get, set)] scale: f64, @@ -3263,6 +4512,8 @@ struct CaptureSettings { #[pyo3(get, set)] use_wayland: Py, /// H.264 recording tap: a Unix socket path to bind, or empty for none. #[pyo3(get, set)] recording_socket: Py, + /// Wayland display of an EXTERNAL compositor to capture (host-capture mode). + #[pyo3(get, set)] wayland_host_display: Py, /// Compositor cursor-theme size in pixels; <=0 keeps the theme default (24). #[pyo3(get, set)] cursor_size: i32, /// Longest cursor edge the X11 out-of-band cursor callback delivers; larger @@ -3275,6 +4526,7 @@ impl CaptureSettings { #[new] fn new(py: Python<'_>) -> Self { Self { + display_id: 0, capture_width: 1920, capture_height: 1080, scale: 1.0, capture_x: 0, capture_y: 0, target_fps: 60.0, jpeg_quality: 85, paint_over_jpeg_quality: 95, use_paint_over_quality: false, paint_over_trigger_frames: 10, @@ -3289,13 +4541,42 @@ impl CaptureSettings { auto_adjust_screen_capture_size: false, omit_stripe_headers: false, encode_node_path: py.None(), render_node_path: py.None(), auto_gpu: py.None(), use_wayland: py.None(), - recording_socket: py.None(), cursor_size: -1, cursor_size_cap: 32, + recording_socket: py.None(), wayland_host_display: py.None(), + cursor_size: -1, cursor_size_cap: 32, } } } /// Process-wide Wayland backend: input and capture share ONE compositor (constructed lazily). static WAYLAND_BACKEND: OnceLock>>> = OnceLock::new(); +/// The compositor's auto-picked socket name (e.g. "wayland-1"), published by the compositor +/// thread once its listening socket exists. `ListeningSocketSource::new_auto` binds the first +/// FREE wayland-N, which need not match any configured index — consumers must read the real +/// name from here instead of assuming one. +static WAYLAND_SOCKET_NAME: Mutex> = Mutex::new(None); +static WAYLAND_SOCKET_CV: Condvar = Condvar::new(); + +fn publish_socket_name(name: &str) { + *WAYLAND_SOCKET_NAME.lock().unwrap() = Some(name.to_string()); + WAYLAND_SOCKET_CV.notify_all(); +} + +/// Wait (bounded) for the compositor thread to publish its socket name. +fn wait_socket_name(timeout: Duration) -> Option { + let deadline = Instant::now() + timeout; + let mut g = WAYLAND_SOCKET_NAME.lock().unwrap(); + loop { + if let Some(name) = g.as_ref() { + return Some(name.clone()); + } + let now = Instant::now(); + if now >= deadline { + return None; + } + let (gg, _) = WAYLAND_SOCKET_CV.wait_timeout(g, deadline - now).unwrap(); + g = gg; + } +} /// Cursor callback registered before the backend exists (selkies registers it pre-start); /// applied when the backend is created, which is deferred to capture start so the real /// render node (not a placeholder) reaches the compositor. @@ -3304,16 +4585,25 @@ static PENDING_CURSOR_CALLBACK: Mutex>> = Mutex::new(None); /// threads must never attach to a finalizing interpreter (aborts the process pre-3.13). /// Cleared by a fresh capture start (only a live interpreter can start one). pub(crate) static PY_SHUTDOWN: AtomicBool = AtomicBool::new(false); -/// ScreenCapture id that owns the active shared Wayland capture (0 = none). Only the owner may -/// stop it, so an input-only or stale instance can't tear down a live capture. -static WAYLAND_OWNER: AtomicU64 = AtomicU64::new(0); -/// Real Wayland capture liveness (StartCapture -> true, StopCapture -> false), mirrored -/// from AppState.is_capturing so the Python-facing is_capturing() reports whether the -/// pipeline is actually running, not merely which ScreenCapture owns the backend. -static WAYLAND_CAPTURE_ALIVE: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNER` compares to -/// decide which instance is allowed to stop the one shared compositor capture. Starts at 1 so 0 can +/// Per-display capture ownership: display id -> the ScreenCapture id that owns that +/// display's capture. Only the owner may stop it, so an input-only or stale instance can't +/// tear down a live capture. +static WAYLAND_OWNERS: OnceLock>> = OnceLock::new(); + +fn wayland_owners() -> &'static Mutex> { + WAYLAND_OWNERS.get_or_init(|| Mutex::new(std::collections::HashMap::new())) +} + +/// Display ids whose capture pipeline is actually running (StartCapture inserts, +/// StopCapture/DestroyOutput remove), so the Python-facing is_capturing() reports pipeline +/// liveness, not merely which ScreenCapture owns the backend. +static WAYLAND_ALIVE_DISPLAYS: OnceLock>> = OnceLock::new(); + +fn wayland_alive() -> &'static Mutex> { + WAYLAND_ALIVE_DISPLAYS.get_or_init(|| Mutex::new(std::collections::HashSet::new())) +} +/// Hands each `ScreenCapture` a unique, monotonic id — the token `WAYLAND_OWNERS` compares to +/// decide which instance is allowed to stop a display's capture. Starts at 1 so 0 can /// mean "no owner". static NEXT_CAPTURE_ID: AtomicU64 = AtomicU64::new(1); /// Registry of every live X11 capture's `Controls`, so the atexit sweep can flag them all to @@ -3357,22 +4647,23 @@ pub(crate) fn boost_thread_priority(nice: libc::c_int) { /// Forward a live rate change to the shared Wayland backend (no-op if none is running). fn wayland_update_rate( py: Python<'_>, + display_id: u32, bitrate_kbps: Option, vbv_multiplier: Option, fps: Option, ) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps); + let _ = be.bind(py).borrow().update_rate(bitrate_kbps, vbv_multiplier, fps, display_id); } } } /// Forward live per-frame tunables to the shared Wayland backend (no-op if none is running). -fn wayland_update_tunables(py: Python<'_>, t: LiveTunables) { +fn wayland_update_tunables(py: Python<'_>, display_id: u32, t: LiveTunables) { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().update_tunables(t); + let _ = be.bind(py).borrow().update_tunables(display_id, t); } } } @@ -3457,6 +4748,14 @@ fn want_wayland(settings: &Bound<'_, PyAny>) -> bool { struct ScState { /// 0 = idle, 1 = X11, 2 = Wayland. backend: u8, + /// This capture holds one reference on the shared X11 cursor monitor. Set in + /// the same locked section as `backend = 1` and TAKEN in the same locked + /// section a stop reads the backend, so acquire/release pair exactly per + /// capture — inferring the reference from `backend` alone would let a stop + /// that interleaves with a start release a reference not yet taken (leaking + /// the monitor once the acquire lands). The GIL happens to serialize that + /// window today; the pairing must not depend on it. + cursor_ref: bool, controls: Option>, handle: Option>, cap_thread_id: Option, @@ -3477,6 +4776,8 @@ struct ScState { /// self-joining. deliver_handle: Option>, deliver_thread_id: Option, + /// The Wayland display id this instance's capture is bound to (backend == 2). + wl_display: u32, } /// Unified capture handle exposed to Python. Drives the X11 capture directly or delegates to the @@ -3499,7 +4800,7 @@ impl ScreenCapture { /// GIL across the joins would deadlock. A re-entrant stop arriving on the capture, encode, or /// deliver thread cannot join itself, so it detaches and lets the threads exit on the stop flag. fn stop_internal(&self, py: Python<'_>) -> PyResult<()> { - let (handle, deliver_handle, same_thread, backend, controls) = { + let (handle, deliver_handle, same_thread, backend, controls, wl_display, cursor_ref) = { let mut st = self.inner.lock().unwrap(); if let Some(c) = &st.controls { c.stop.store(true, Ordering::Relaxed); @@ -3519,27 +4820,37 @@ impl ScreenCapture { let handle = st.handle.take(); let deliver_handle = st.deliver_handle.take(); let backend = st.backend; + let cursor_ref = std::mem::take(&mut st.cursor_ref); + let wl_display = st.wl_display; st.backend = 0; st.cap_thread_id = None; st.encode_thread_id = None; st.encode_tid_rx = None; st.deliver_thread_id = None; - (handle, deliver_handle, same, backend, controls) + st.wl_display = 0; + (handle, deliver_handle, same, backend, controls, wl_display, cursor_ref) }; if let Some(c) = &controls { live_x11().lock().unwrap().retain(|x| !Arc::ptr_eq(x, c)); } - if backend == 1 { + if cursor_ref { crate::x11::cursor::release(py); } if backend == 2 { - if WAYLAND_OWNER - .compare_exchange(self.id, 0, Ordering::AcqRel, Ordering::Relaxed) - .is_ok() - { + let did = wl_display; + let owned = { + let mut owners = wayland_owners().lock().unwrap(); + if owners.get(&did) == Some(&self.id) { + owners.remove(&did); + true + } else { + false + } + }; + if owned { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let _ = be.bind(py).borrow().stop_capture(did); } } } @@ -3574,6 +4885,7 @@ impl ScreenCapture { id: NEXT_CAPTURE_ID.fetch_add(1, Ordering::Relaxed), inner: Mutex::new(ScState { backend: 0, + cursor_ref: false, controls: None, handle: None, cap_thread_id: None, @@ -3581,6 +4893,7 @@ impl ScreenCapture { encode_tid_rx: None, deliver_handle: None, deliver_thread_id: None, + wl_display: 0, }), } } @@ -3602,10 +4915,14 @@ impl ScreenCapture { callback: Py, settings: &Bound<'_, PyAny>, ) -> PyResult<()> { + let display_id = read_display_id(settings); let live_wayland_restart = want_wayland(settings) - && self.inner.lock().unwrap().backend == 2 - && WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire); + && { + let st = self.inner.lock().unwrap(); + st.backend == 2 && st.wl_display == display_id + } + && wayland_owners().lock().unwrap().get(&display_id) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&display_id); if !live_wayland_restart { self.stop_internal(py)?; } @@ -3637,8 +4954,12 @@ impl ScreenCapture { cursor_size, )?; be.bind(py).borrow().start_capture(callback, settings)?; - WAYLAND_OWNER.store(self.id, Ordering::Relaxed); - self.inner.lock().unwrap().backend = 2; + wayland_owners().lock().unwrap().insert(display_id, self.id); + { + let mut st = self.inner.lock().unwrap(); + st.backend = 2; + st.wl_display = display_id; + } return Ok(()); } @@ -3767,8 +5088,12 @@ impl ScreenCapture { None } }; + // Take the monitor reference BEFORE publishing the capture: a stop that + // observes this capture must always find the reference it is to release. + crate::x11::cursor::acquire(cursor_cap); let mut st = self.inner.lock().unwrap(); st.backend = 1; + st.cursor_ref = true; st.controls = Some(controls); st.handle = Some(handle); st.cap_thread_id = tid; @@ -3777,7 +5102,6 @@ impl ScreenCapture { st.deliver_handle = Some(deliver_handle); st.deliver_thread_id = Some(deliver_thread_id); drop(st); - crate::x11::cursor::acquire(cursor_cap); Ok(()) } @@ -3786,9 +5110,9 @@ impl ScreenCapture { } fn request_idr_frame(&self, py: Python<'_>) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3799,7 +5123,7 @@ impl ScreenCapture { 2 => { if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().request_idr_frame(); + let _ = be.bind(py).borrow().request_idr_frame(did); } } } @@ -3813,9 +5137,9 @@ impl ScreenCapture { /// On the X11 path the dirty flag is Release-published after the payload store, so the encode /// thread's Acquire read can never observe the flag set against a stale bitrate. fn update_video_bitrate(&self, py: Python<'_>, kbps: i32) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3824,16 +5148,16 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, Some(kbps), None, None), + 2 => wayland_update_rate(py, did, Some(kbps), None, None), _ => {} } Ok(()) } fn update_framerate(&self, py: Python<'_>, fps: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3842,7 +5166,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, None, Some(fps)), + 2 => wayland_update_rate(py, did, None, None, Some(fps)), _ => {} } Ok(()) @@ -3850,9 +5174,9 @@ impl ScreenCapture { /// Live CBR VBV change, as a multiple of one frame's bit budget (<= 0 = policy default). fn update_vbv_multiplier(&self, py: Python<'_>, multiplier: f64) -> PyResult<()> { - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3862,7 +5186,7 @@ impl ScreenCapture { c.rate_dirty.store(true, Ordering::Release); } } - 2 => wayland_update_rate(py, None, Some(multiplier), None), + 2 => wayland_update_rate(py, did, None, Some(multiplier), None), _ => {} } Ok(()) @@ -3874,9 +5198,9 @@ impl ScreenCapture { fn update_tunables(&self, py: Python<'_>, settings: &Bound<'_, PyAny>) -> PyResult<()> { let rs = extract_settings(settings)?; let t = LiveTunables::from_settings(&rs); - let (backend, controls) = { + let (backend, controls, did) = { let st = self.inner.lock().unwrap(); - (st.backend, st.controls.clone()) + (st.backend, st.controls.clone(), st.wl_display) }; match backend { 1 => { @@ -3887,7 +5211,7 @@ impl ScreenCapture { } crate::x11::cursor::set_size_cap(rs.cursor_size_cap); } - 2 => wayland_update_tunables(py, t), + 2 => wayland_update_tunables(py, did, t), _ => {} } Ok(()) @@ -3924,8 +5248,8 @@ impl ScreenCapture { .as_ref() .map(|c| !c.stop.load(Ordering::Relaxed)) .unwrap_or(false), - 2 => WAYLAND_OWNER.load(Ordering::Relaxed) == self.id - && WAYLAND_CAPTURE_ALIVE.load(Ordering::Acquire), + 2 => wayland_owners().lock().unwrap().get(&st.wl_display) == Some(&self.id) + && wayland_alive().lock().unwrap().contains(&st.wl_display), _ => false, } } @@ -3936,6 +5260,88 @@ impl ScreenCapture { fn set_keymap_string(&self, py: Python<'_>, text: String) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().set_keymap_string(text)) } + /// Compositor apps run under (a nested labwc/kwin session pixelflux captures), + /// the target for Computer-Use text injection; selkies resolves it and hands it + /// over. Empty clears it. Stored process-wide since the CU server is per-process. + fn set_app_wayland_display(&self, display: String) { + crate::computer_use::set_app_wayland_display( + if display.is_empty() { None } else { Some(display) }, + ); + } + /// Type `text` through `display`'s zwp_virtual_keyboard_manager_v1 as a one-shot + /// client: selkies' text-injection path, targeting whichever compositor the apps + /// live under (the nested session's, or pixelflux's own in a direct session). + /// Blocking; releases the GIL for the duration. Raises + /// [`VirtualKeyboardUnavailable`] when the compositor lacks the protocol. + fn type_text_wayland(&self, py: Python<'_>, display: String, text: String) -> PyResult<()> { + py.detach(move || { + let path = crate::wayland::wlclient::socket_path(&display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string())?; + crate::wayland::vkclient::type_text_to(&path, &text) + }) + .map_err(|e: String| { + if e.contains("zwp_virtual_keyboard_manager_v1") { + VirtualKeyboardUnavailable::new_err(e) + } else { + pyo3::exceptions::PyRuntimeError::new_err(e) + } + }) + } + /// Mimes the app compositor's current selection offers (empty = nothing copied). + fn clipboard_types_app(&self, py: Python<'_>, display: String) -> PyResult> { + py.detach(move || { + crate::wayland::dcclient::list_types(&app_socket_path(&display)?) + }) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// The app compositor selection's payload for `mime`, or None when nothing is + /// copied or the selection does not offer that mime. + fn clipboard_read_app( + &self, + py: Python<'_>, + display: String, + mime: String, + ) -> PyResult>> { + let data = py + .detach(move || crate::wayland::dcclient::read(&app_socket_path(&display)?, &mime)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err)?; + Ok(data.map(|d| pyo3::types::PyBytes::new(py, &d).unbind())) + } + /// Take the app compositor's selection, serving `entries` (mime, bytes) to + /// every paster from a background thread until another client copies. + fn clipboard_write_app( + &self, + py: Python<'_>, + display: String, + entries: Vec<(String, Vec)>, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::write(&app_socket_path(&display)?, entries)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Drop the app compositor's selection. + fn clipboard_clear_app(&self, py: Python<'_>, display: String) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::clear(&app_socket_path(&display)?)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Invoke `callback(mimes: list[str])` from a background thread on every + /// selection change in the app compositor (including the one current at call + /// time). A second watch for the same display replaces the first. + fn clipboard_watch_app( + &self, + py: Python<'_>, + display: String, + callback: Py, + ) -> PyResult<()> { + py.detach(move || crate::wayland::dcclient::watch(&app_socket_path(&display)?, callback)) + .map_err(pyo3::exceptions::PyRuntimeError::new_err) + } + /// Stop the selection watch for `display` (no-op without one). + fn clipboard_unwatch_app(&self, py: Python<'_>, display: String) { + let _ = py.detach(move || { + crate::wayland::dcclient::unwatch(&app_socket_path(&display)?); + Ok::<(), String>(()) + }); + } fn inject_mouse_move(&self, py: Python<'_>, x: f64, y: f64) -> PyResult<()> { wayland_backend_running(py).map_or(Ok(()), |be| be.bind(py).borrow().inject_mouse_move(x, y)) } @@ -4004,6 +5410,93 @@ impl ScreenCapture { )), } } + /// Set the seat's BASE xkb layout from RMLVO names; false when no backend runs or the + /// layout fails to compile. + #[pyo3(signature = (layout, variant = String::new(), options = String::new(), model = String::new(), rules = String::new()))] + fn set_xkb_layout( + &self, + py: Python<'_>, + layout: String, + variant: String, + options: String, + model: String, + rules: String, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().set_xkb_layout(py, layout, variant, options, model, rules) + }) + } + /// Resolve keysyms to `(keycode, level)` with batched overlay binding (one keymap swap + /// per call, held keycodes never rebound); all `(0, 0)` when no backend runs. + fn bind_keysyms(&self, py: Python<'_>, keysyms: Vec) -> PyResult> { + let n = keysyms.len(); + wayland_backend_running(py) + .map_or(Ok(vec![(0, 0); n]), |be| be.bind(py).borrow().bind_keysyms(py, keysyms)) + } + /// Seat keyboard readback: `(pressed_keycodes, modifier_mask)`; empty when no backend runs. + fn get_keyboard_state(&self, py: Python<'_>) -> PyResult<(Vec, u32)> { + wayland_backend_running(py) + .map_or(Ok((Vec::new(), 0)), |be| be.bind(py).borrow().get_keyboard_state(py)) + } + /// The capture geometry actually live on the given display `(width, height, scale)`; + /// `(0, 0, 0.0)` when no backend runs. + #[pyo3(signature = (display_id = 0))] + fn get_realized_geometry(&self, py: Python<'_>, display_id: u32) -> PyResult<(i32, i32, f64)> { + wayland_backend_running(py) + .map_or(Ok((0, 0, 0.0)), |be| be.bind(py).borrow().get_realized_geometry(py, display_id)) + } + /// Create an additional Wayland output (see `WaylandBackend.create_output`); false when + /// no backend runs. + #[pyo3(signature = (id, width, height, x = 0, y = 0, scale = 1.0))] + fn create_output( + &self, + py: Python<'_>, + id: u32, + width: i32, + height: i32, + x: i32, + y: i32, + scale: f64, + ) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().create_output(py, id, width, height, x, y, scale) + }) + } + /// Destroy a secondary Wayland output; false when no backend runs. + fn destroy_output(&self, py: Python<'_>, id: u32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().destroy_output(py, id)) + } + /// Move a Wayland output (the primary included) to layout offset `(x, y)`; false when + /// no backend runs or the id is unknown. + fn reposition_output(&self, py: Python<'_>, id: u32, x: i32, y: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().reposition_output(py, id, x, y)) + } + /// Recreate the Wayland cursor theme at `size` pixels (named-cursor callbacks and the + /// burned-in overlay); false when no backend runs or the size is non-positive. + fn set_cursor_size(&self, py: Python<'_>, size: i32) -> PyResult { + wayland_backend_running(py) + .map_or(Ok(false), |be| be.bind(py).borrow().set_cursor_size(py, size)) + } + /// Every live Wayland output as `(id, x, y, width, height, scale, capturing)`; empty + /// when no backend runs. + fn list_outputs(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_outputs(py)) + } + /// Move a window onto an output (fullscreened there); false when no backend runs. + fn move_window_to_output(&self, py: Python<'_>, window_id: u32, output_id: u32) -> PyResult { + wayland_backend_running(py).map_or(Ok(false), |be| { + be.bind(py).borrow().move_window_to_output(py, window_id, output_id) + }) + } + /// Every mapped window as `(window_id, title, app_id, output_id)`; empty when no + /// backend runs. + fn list_windows(&self, py: Python<'_>) -> PyResult> { + wayland_backend_running(py) + .map_or(Ok(Vec::new()), |be| be.bind(py).borrow().list_windows(py)) + } } /// Best-effort teardown: flag the capture thread to exit without joining. @@ -4021,11 +5514,96 @@ impl Drop for ScreenCapture { } } +/// Build a Python dict from a recorder status snapshot (one shape for status and stop). +fn recording_status_dict(py: Python<'_>, s: &crate::recorder::RecordingStatus) -> PyResult> { + let d = pyo3::types::PyDict::new(py); + d.set_item("active", s.active)?; + d.set_item("path", &s.path)?; + d.set_item("backend", s.backend)?; + d.set_item("mode", s.mode)?; + d.set_item("frames", s.frames)?; + d.set_item("sync_frames", s.sync_frames)?; + d.set_item("dropped", s.dropped)?; + d.set_item("skipped_non_h264", s.skipped_non_h264)?; + d.set_item("bytes", s.bytes)?; + d.set_item("duration_s", s.duration_s)?; + d.set_item("width", s.width)?; + d.set_item("height", s.height)?; + d.set_item("error", s.error.as_deref())?; + Ok(d.into_any().unbind()) +} + +/// Start the built-in MP4 recorder. Works with no capture and no client running: the +/// recorder owns an independent capture (X11 root, or a Wayland output of the in-process +/// compositor) and taps a live streaming session instead of restarting it. `settings` is an +/// optional `CaptureSettings` for a recorder-owned capture (H.264 only; `display_id` +/// selects the Wayland output); when omitted, `PIXELFLUX_RECORD_*` environment variables +/// and full-screen defaults apply. +#[pyfunction] +#[pyo3(signature = (path, settings = None))] +fn start_recording( + py: Python<'_>, + path: String, + settings: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let mut opts = crate::recorder::RecordOptions::from_env(path); + if let Some(s) = settings { + let rs = extract_settings(s)?; + if rs.output_mode != 1 { + return Err(PyErr::new::( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded", + )); + } + opts.display_id = read_display_id(s); + if let Some(explicit) = s + .getattr("use_wayland") + .ok() + .and_then(|v| v.extract::().ok()) + { + opts.backend = Some(if explicit { + crate::recorder::PreferredBackend::Wayland + } else { + crate::recorder::PreferredBackend::X11 + }); + } + // Explicit settings are authoritative over the env knobs they subsume. + opts.fps = 0.0; + opts.bitrate_kbps = 0; + opts.capture = Some(rs); + } + let status = py + .detach(|| crate::recorder::start(opts)) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Stop the active recording, finalize the MP4, and return the final status dict. Raises +/// when no recording is active or nothing recordable was captured. +#[pyfunction] +fn stop_recording(py: Python<'_>) -> PyResult> { + let status = py + .detach(crate::recorder::stop) + .map_err(PyErr::new::)?; + recording_status_dict(py, &status) +} + +/// Status of the live recording (or the last finished one); `None` if this process has +/// never recorded. +#[pyfunction] +fn recording_status(py: Python<'_>) -> PyResult> { + match crate::recorder::status() { + Some(s) => recording_status_dict(py, &s), + None => Ok(py.None()), + } +} + /// Bring the Wayland compositor socket up before any capture so apps launched early can /// connect (sets WAYLAND_DISPLAY for children of this process). Idempotent; the running /// backend keeps its node/dimensions on later calls. `render_node` is an explicit /// /dev/dri/renderD* path; `auto_gpu` is a truthy string or vendor/driver token; empty -/// values mean software rendering. +/// values mean software rendering. Returns the compositor's actual socket name (the +/// auto-picked `wayland-N`, which need not match any configured index); empty string only +/// if the socket did not come up in time. #[pyfunction] #[pyo3(signature = (width = 0, height = 0, render_node = String::new(), auto_gpu = String::new(), cursor_size = -1))] fn ensure_wayland_display( @@ -4035,9 +5613,19 @@ fn ensure_wayland_display( render_node: String, auto_gpu: String, cursor_size: i32, -) -> PyResult<()> { - ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size) - .map(|_| ()) +) -> PyResult { + ensure_wayland_backend(py, width, height, render_node, auto_gpu, String::new(), cursor_size)?; + Ok(py + .detach(|| wait_socket_name(Duration::from_secs(5))) + .unwrap_or_default()) +} + +/// The running compositor's Wayland socket name (e.g. "wayland-1"), or None when no +/// compositor thread has been started (this never creates one). +#[pyfunction] +fn get_wayland_display_name(py: Python<'_>) -> Option { + wayland_backend_running(py)?; + py.detach(|| wait_socket_name(Duration::from_secs(2))) } /// Stop every live capture (registered with atexit) before interpreter finalization. @@ -4051,6 +5639,9 @@ fn ensure_wayland_display( #[pyfunction] fn _stop_all_captures(py: Python<'_>) { PY_SHUTDOWN.store(true, Ordering::Relaxed); + // Finalize any active recording first so its last buffered MP4 sample is flushed and + // its own capture (if any) is stopped through the normal path. + py.detach(crate::recorder::finalize_on_exit); *PENDING_CURSOR_CALLBACK.lock().unwrap() = None; crate::x11::cursor::shutdown(); if let Some(slot) = LIVE_X11.get() { @@ -4060,16 +5651,52 @@ fn _stop_all_captures(py: Python<'_>) { } if let Some(slot) = WAYLAND_BACKEND.get() { if let Some(be) = slot.lock().unwrap().as_ref() { - let _ = be.bind(py).borrow().stop_capture(); + let be = be.bind(py).borrow(); + let mut displays: Vec = + wayland_alive().lock().unwrap().iter().copied().collect(); + if !displays.contains(&0) { + displays.push(0); + } + for did in displays { + let _ = be.stop_capture(did); + } + // Wait (bounded) until the calloop finished processing the stops: dropping a + // hardware encoder session (NVENC/CUDA) mid-process-exit segfaults, so the + // interpreter must not finalize while that teardown is still running. + let (ack_tx, ack_rx) = std::sync::mpsc::channel::<()>(); + if be.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = py.detach(move || ack_rx.recv_timeout(Duration::from_secs(2))); + } } } - WAYLAND_OWNER.store(0, Ordering::Relaxed); - WAYLAND_CAPTURE_ALIVE.store(false, Ordering::Relaxed); + wayland_owners().lock().unwrap().clear(); + wayland_alive().lock().unwrap().clear(); py.detach(|| std::thread::sleep(Duration::from_millis(50))); } /// The `pixelflux` Python module: registers the exported classes and functions, and hooks /// `_stop_all_captures` into `atexit` so every live capture is stopped before interpreter shutdown. +/// Socket path for a Wayland display name, with the ABI methods' error shape. +fn app_socket_path(display: &str) -> Result { + crate::wayland::wlclient::socket_path(display) + .ok_or_else(|| "XDG_RUNTIME_DIR is unset".to_string()) +} + +pyo3::create_exception!( + pixelflux, + VirtualKeyboardUnavailable, + pyo3::exceptions::PyRuntimeError, + "The target compositor does not advertise zwp_virtual_keyboard_manager_v1." +); + +/// Start the Computer-Use HTTP server: a bare port listens on all interfaces, +/// `host:port` scopes it. Idempotent; the PIXELFLUX_CU env var remains the +/// standalone fallback. +#[pyfunction] +fn start_computer_use(bind: String) { + crate::computer_use::start_cu_server(&bind); +} + #[pymodule] fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -4080,15 +5707,86 @@ fn pixelflux(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("X11_CURSOR_CALLBACK", true)?; m.add_function(wrap_pyfunction!(stripe_frame_from_buffer, m)?)?; m.add_function(wrap_pyfunction!(ensure_wayland_display, m)?)?; + m.add_function(wrap_pyfunction!(get_wayland_display_name, m)?)?; + m.add_function(wrap_pyfunction!(start_recording, m)?)?; + m.add_function(wrap_pyfunction!(stop_recording, m)?)?; + m.add_function(wrap_pyfunction!(recording_status, m)?)?; + m.add_function(wrap_pyfunction!(start_computer_use, m)?)?; + m.add( + "VirtualKeyboardUnavailable", + m.py().get_type::(), + )?; m.add_function(wrap_pyfunction!(_stop_all_captures, m)?)?; if let Ok(atexit) = m.py().import("atexit") { let _ = atexit.call_method1("register", (m.getattr("_stop_all_captures")?,)); } + // Standalone CU entry point: with PIXELFLUX_CU set the server binds at import, serving + // X11 (via DISPLAY) until a Wayland compositor registers itself as the backend. + crate::computer_use::spawn_cu_from_env(); + // PIXELFLUX_RECORD=: start recording from process start (X11 immediately, or as + // soon as the in-process Wayland compositor comes up). + crate::recorder::autostart_from_env(); Ok(()) } +#[cfg(test)] +mod output_overlap_tests { + //! Invariants of the output-placement intersection predicate: strict interior + //! intersection (touching edges never overlap), containment and identity overlap, + //! empty/negative rectangles never overlap, and extreme coordinates do not wrap. + use super::rects_overlap; + + #[test] + fn disjoint_rects_do_not_overlap() { + assert!(!rects_overlap((0, 0, 100, 100), (200, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 200, 100, 100))); + } + + #[test] + fn touching_edges_do_not_overlap() { + // Right edge of a meets left edge of b, and bottom meets top. + assert!(!rects_overlap((0, 0, 100, 100), (100, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (0, 100, 100, 100))); + // Corner touch only. + assert!(!rects_overlap((0, 0, 100, 100), (100, 100, 50, 50))); + } + + #[test] + fn one_pixel_intrusion_overlaps() { + assert!(rects_overlap((0, 0, 100, 100), (99, 0, 100, 100))); + assert!(rects_overlap((0, 0, 100, 100), (0, 99, 100, 100))); + } + + #[test] + fn containment_and_identity_overlap() { + assert!(rects_overlap((0, 0, 100, 100), (25, 25, 10, 10))); + assert!(rects_overlap((25, 25, 10, 10), (0, 0, 100, 100))); + assert!(rects_overlap((5, 5, 50, 50), (5, 5, 50, 50))); + } + + #[test] + fn empty_or_negative_rects_never_overlap() { + assert!(!rects_overlap((10, 10, 0, 50), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, 50, 0), (0, 0, 100, 100))); + assert!(!rects_overlap((10, 10, -5, 5), (0, 0, 100, 100))); + assert!(!rects_overlap((0, 0, 100, 100), (10, 10, 0, 0))); + } + + #[test] + fn negative_origins_overlap_correctly() { + assert!(rects_overlap((-50, -50, 100, 100), (0, 0, 100, 100))); + assert!(!rects_overlap((-100, -100, 100, 100), (0, 0, 100, 100))); + } + + #[test] + fn extreme_coordinates_do_not_wrap() { + assert!(!rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MIN, 0, 10, 10))); + assert!(rects_overlap((i32::MAX - 10, 0, 10, 10), (i32::MAX - 5, 0, 10, 10))); + } +} + #[cfg(test)] mod wl_frame_pool_tests { //! Invariants under test: try_begin is non-blocking and hands out a buffer ONLY while the diff --git a/pixelflux/src/pipeline.rs b/pixelflux/src/pipeline.rs index 6c504c5..26afaee 100644 --- a/pixelflux/src/pipeline.rs +++ b/pixelflux/src/pipeline.rs @@ -14,7 +14,6 @@ use crate::encoders::nvenc::NvencEncoder; use crate::encoders::oh264::Openh264Encoder; use crate::encoders::software::{encode_cpu, EncodedStripe, StripeState}; use crate::encoders::vaapi::VaapiEncoder; -use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; use std::sync::Arc; @@ -164,7 +163,8 @@ enum X11Encoder { /// content frame-to-frame. Full-frame H.264 runs through `decide_hw_fullframe`; striped JPEG/x264 /// runs through `encode_cpu` with `hash_damage=true`. /// -/// Recording sink fan-out is handled at the delivery layer. +/// Recording fan-out (socket sink and MP4 recorder) is handled at the delivery layer; a +/// consumer needing a keyframe goes through [`X11Pipeline::request_idr`] like everyone else. pub struct X11Pipeline { settings: RustCaptureSettings, stripes: Vec, @@ -172,7 +172,6 @@ pub struct X11Pipeline { hw_state: StripeState, frame_counter: u16, pending_force_idr: bool, - recording_sink: Option>, } impl X11Pipeline { @@ -182,8 +181,6 @@ impl X11Pipeline { /// /// * `settings` - Capture configuration. The `output_mode`, `use_openh264`, `use_cpu`, /// `encode_node_index`, and `video_fullcolor` fields drive encoder selection. - /// * `recording_sink` - Optional Unix-socket H.264 fan-out. Owned by the caller; not - /// rebound on auto-adjust resizes to avoid tearing down the socket listener. /// /// # Encoder selection /// @@ -191,7 +188,7 @@ impl X11Pipeline { /// 2. **NVENC** — on an NVIDIA driver (or no detectable GPU, since the attempt is cheap). /// 3. **VA-API** — on any other GPU driver (except 4:4:4 full-color, which falls to x264). /// 4. **Software** (`X11Encoder::None`) — on any hardware init failure or explicit software. - pub fn new(settings: RustCaptureSettings, recording_sink: Option>) -> Self { + pub fn new(settings: RustCaptureSettings) -> Self { let hw = if settings.output_mode == 1 && settings.use_openh264 { println!("[x11] OpenH264 software encoder selected."); match Openh264Encoder::new(&settings) { @@ -248,7 +245,6 @@ impl X11Pipeline { hw_state: StripeState::default(), frame_counter: 0, pending_force_idr: false, - recording_sink, } } @@ -360,8 +356,7 @@ impl X11Pipeline { ); if d.send { let fc = self.frame_counter as u64; - let force_idr = d.force_idr - || self.recording_sink.as_ref().map(|s| s.should_force_idr()).unwrap_or(false); + let force_idr = d.force_idr; let res = match &mut self.hw { X11Encoder::Nvenc(enc) => { enc.encode_cpu_argb(argb, stride, fc, d.target_qp, force_idr) @@ -377,7 +372,7 @@ impl X11Pipeline { match res { Ok(data) if !data.is_empty() => { vec![EncodedStripe { - data, + data: Arc::new(data), data_type: 2, stripe_y_start: 0, stripe_height: height, @@ -416,7 +411,10 @@ impl X11Pipeline { ) }; - self.pending_force_idr = false; + // An unserved request stays armed: on an infinite GOP an IDR lost to an encode + // error or skip would never self-heal, leaving a joining consumer with an + // undecodable stream. + self.pending_force_idr = requested && out.is_empty(); self.frame_counter = self.frame_counter.wrapping_add(1); out } @@ -441,7 +439,7 @@ mod tests { use_paint_over_quality: false, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame_a = vec![10u8; stride * 128]; let mut frame_b = frame_a.clone(); @@ -474,7 +472,7 @@ mod tests { target_fps: 60.0, ..Default::default() }; - let mut p = X11Pipeline::new(s, None); + let mut p = X11Pipeline::new(s); let stride = 128 * 4; let frame = vec![10u8; stride * 128]; assert!(!p.process(&frame, stride).is_empty(), "first frame emits"); diff --git a/pixelflux/src/recorder/mod.rs b/pixelflux/src/recorder/mod.rs new file mode 100644 index 0000000..8b22581 --- /dev/null +++ b/pixelflux/src/recorder/mod.rs @@ -0,0 +1,747 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Built-in MP4 recorder: an independent pipeline that captures the desktop to a +//! ready-to-play fragmented MP4 without needing any connected client. +//! +//! One implementation serves three control surfaces — the Python module functions +//! (`start_recording` / `stop_recording` / `recording_status`), the `PIXELFLUX_RECORD*` +//! environment variables (record from process start), and the `record_start` / +//! `record_stop` / `record_status` endpoints on the Computer Use HTTP server — so their +//! behavior cannot drift. +//! +//! The recorder is a consumer of the existing capture machinery, never a hook inside an +//! encoder: +//! +//! * **X11**: it always runs its own [`crate::x11::run_capture`] instance against the root +//! window (a second capture of the same root is independent of any streaming session), with +//! the encoded frames delivered straight into the recorder's queue. +//! * **Wayland**: the compositor lives in this process and allows one capture per output, so +//! the recorder attaches at the delivery layer. When the output is already being captured +//! for a streaming client it taps that stream (and paces recovery keyframes through the +//! standard `RequestIdr` command); when the output is idle it starts its own capture with no +//! Python callback and taps the identical delivery point. +//! +//! Frames cross into the recorder through one bounded queue, mirroring the Unix-socket sink's +//! isolation contract: the delivery thread only clones an `Arc` and `try_send`s, an overflowing +//! queue drops frames (never blocks the pipeline), and all muxing happens on the recorder's own +//! writer thread. Timestamps are wall-clock, so the damage-driven, variable-rate frame flow +//! lands at its true times in the MP4. + +pub mod mp4; + +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +use crossbeam_channel::{bounded, Receiver, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; +use crate::RustCaptureSettings; +use crate::ThreadCommand; + +/// Recorder queue bound, matching the socket sink's stalled-consumer policy: a writer that +/// falls this far behind loses frames instead of growing memory or blocking the encoder. +const QUEUE_CAP: usize = 256; + +/// A queued frame: `Arc`-shared encoded payload, byte offset where the Annex-B stream +/// starts (past the wire header when present), and the wall-clock capture time. +type TapFrame = (Arc>, usize, u64); + +/// Which capture system feeds the recording. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PreferredBackend { + X11, + Wayland, +} + +/// Resolved recording parameters. Environment variables provide the defaults; explicit +/// Python/REST arguments override them. +#[derive(Clone)] +pub struct RecordOptions { + pub path: String, + pub display_id: u32, + /// Capture fps cap for a recorder-owned capture (`<= 0` = default 30). An attached + /// recording follows the live session's rate. + pub fps: f64, + /// Bitrate override in kbps for a recorder-owned capture (`<= 0` = settings default). + pub bitrate_kbps: i32, + /// Recovery-keyframe cadence in seconds (`<= 0` = default 2.0). Recorder-owned captures + /// carry it in their settings (scheduled IDRs); attached recordings pace standard + /// request-IDR commands at this interval. + pub keyframe_interval_s: f64, + pub backend: Option, + /// Full capture settings for a recorder-owned capture; `None` derives them (full root / + /// current output geometry, H.264 full-frame). + pub capture: Option, +} + +impl RecordOptions { + /// Options seeded entirely from `PIXELFLUX_RECORD_*` environment variables. + pub fn from_env(path: String) -> Self { + let f = |k: &str| std::env::var(k).ok().and_then(|v| v.parse::().ok()); + let backend = match std::env::var("PIXELFLUX_RECORD_BACKEND").ok().as_deref() { + Some("x11") => Some(PreferredBackend::X11), + Some("wayland") => Some(PreferredBackend::Wayland), + _ => None, + }; + Self { + path, + display_id: f("PIXELFLUX_RECORD_DISPLAY").map(|v| v as u32).unwrap_or(0), + fps: f("PIXELFLUX_RECORD_FPS").unwrap_or(0.0), + bitrate_kbps: f("PIXELFLUX_RECORD_BITRATE").map(|v| v as i32).unwrap_or(0), + keyframe_interval_s: f("PIXELFLUX_RECORD_KEYFRAME_S").unwrap_or(0.0), + backend, + capture: None, + } + } + + fn effective_fps(&self) -> f64 { + if self.fps > 0.0 { self.fps } else { 30.0 } + } + + fn effective_keyframe_s(&self) -> f64 { + if self.keyframe_interval_s > 0.0 { self.keyframe_interval_s } else { 2.0 } + } +} + +/// Live counters shared between the feeding side (delivery threads), the writer thread and +/// the status surfaces. +struct RecShared { + start: Instant, + enqueued: AtomicU64, + dropped: AtomicU64, + skipped_non_h264: AtomicU64, + muxed: AtomicU64, + sync_frames: AtomicU64, + bytes: AtomicU64, + width: AtomicU32, + height: AtomicU32, + last_idr_req_us: AtomicU64, + error: Mutex>, +} + +impl RecShared { + fn new() -> Arc { + Arc::new(Self { + start: Instant::now(), + enqueued: AtomicU64::new(0), + dropped: AtomicU64::new(0), + skipped_non_h264: AtomicU64::new(0), + muxed: AtomicU64::new(0), + sync_frames: AtomicU64::new(0), + bytes: AtomicU64::new(0), + width: AtomicU32::new(0), + height: AtomicU32::new(0), + last_idr_req_us: AtomicU64::new(0), + error: Mutex::new(None), + }) + } + + fn set_error(&self, msg: String) { + let mut g = self.error.lock().unwrap(); + if g.is_none() { + eprintln!("[recorder] {msg}"); + *g = Some(msg); + } + } +} + +/// Point-in-time view of the recorder, identical across the Python, env and REST surfaces. +#[derive(Clone, Debug)] +pub struct RecordingStatus { + pub active: bool, + pub path: String, + pub backend: &'static str, + pub mode: &'static str, + pub frames: u64, + pub sync_frames: u64, + pub dropped: u64, + pub skipped_non_h264: u64, + pub bytes: u64, + pub duration_s: f64, + pub width: u32, + pub height: u32, + pub error: Option, +} + +/// The wayland-side tap handle consulted by [`wayland_tap`]; cloneable so the delivery +/// thread can drop the registry lock before doing any work. +#[derive(Clone)] +struct WlTap { + display_id: u32, + tx: Sender, + shared: Arc, + /// Standard request-IDR path for attached recordings (`None` when the recorder owns the + /// capture and scheduled keyframes ride in its settings). + idr: Option, + keyframe_interval_us: u64, +} + +/// Sends `ThreadCommand::RequestIdr` for one display over the compositor's command channel. +#[derive(Clone)] +struct IdrRequester { + tx: smithay::reexports::calloop::channel::Sender, + display_id: u32, +} + +impl IdrRequester { + fn request(&self) { + let _ = self.tx.send(ThreadCommand::RequestIdr { display_id: self.display_id }); + } +} + +/// How the active recording is fed and what must be torn down on stop. +enum RecordingMode { + /// Recorder-owned X11 capture: its controls and capture thread. + X11Own { + controls: Arc, + join: thread::JoinHandle<()>, + }, + /// Recorder-owned Wayland capture on `display_id` (stopped unless a streaming client has + /// since taken ownership of the display). + WaylandOwn { display_id: u32 }, + /// Tap on a streaming client's live Wayland capture; nothing to stop. + WaylandAttached, +} + +struct ActiveRecording { + path: String, + backend: &'static str, + mode_name: &'static str, + mode: RecordingMode, + tx: Sender, + shared: Arc, + writer: thread::JoinHandle<()>, +} + +static ACTIVE: Mutex> = Mutex::new(None); +static LAST_FINISHED: Mutex> = Mutex::new(None); +/// Fast idle guard for [`wayland_tap`]: the delivery threads pay one relaxed atomic load +/// per frame while no recording is armed. +static WL_TAP_ARMED: AtomicBool = AtomicBool::new(false); +static WL_TAP: Mutex> = Mutex::new(None); + +/// Delivery-layer feed from the Wayland pipelines (readback encode loop and zero-copy +/// tick). Near-zero cost when idle; while recording it clones an `Arc` into the bounded +/// queue and never blocks. +pub(crate) fn wayland_tap(display_id: u32, stripes: &[EncodedStripe]) { + if !WL_TAP_ARMED.load(Ordering::Relaxed) { + return; + } + let Some(tap) = WL_TAP.lock().unwrap().clone() else { return }; + if tap.display_id != display_id { + return; + } + offer_frame(&tap.shared, &tap.tx, stripes); + if let Some(ref idr) = tap.idr { + pace_idr_requests(&tap.shared, idr, tap.keyframe_interval_us); + } +} + +/// Issue a standard request-IDR when the cadence interval has elapsed. Compare-and-swap on +/// the last-request stamp so concurrent delivery threads emit one request per interval. +fn pace_idr_requests(shared: &RecShared, idr: &IdrRequester, interval_us: u64) { + if interval_us == 0 { + return; + } + let now = shared.start.elapsed().as_micros() as u64; + let last = shared.last_idr_req_us.load(Ordering::Relaxed); + if now.saturating_sub(last) >= interval_us + && shared + .last_idr_req_us + .compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + idr.request(); + } +} + +/// Enqueue one delivered frame for muxing. Only a single full-frame H.264 stripe is +/// recordable; JPEG or striped output is counted and skipped so the surfaces can report a +/// clear "nothing recordable" error instead of writing a corrupt file. +fn offer_frame(shared: &RecShared, tx: &Sender, stripes: &[EncodedStripe]) { + if stripes.is_empty() { + return; + } + if stripes.len() != 1 || stripes[0].data_type != 2 || stripes[0].stripe_y_start != 0 { + shared.skipped_non_h264.fetch_add(1, Ordering::Relaxed); + return; + } + let s = &stripes[0]; + if s.data.is_empty() { + return; + } + let offset = if s.data.len() >= 10 && s.data[0] == 0x04 { 10 } else { 0 }; + if s.data.len() == offset { + return; + } + let pts_us = shared.start.elapsed().as_micros() as u64; + match tx.try_send((s.data.clone(), offset, pts_us)) { + Ok(()) => { + shared.enqueued.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Full(_)) => { + shared.dropped.fetch_add(1, Ordering::Relaxed); + } + Err(TrySendError::Disconnected(_)) => {} + } +} + +/// Writer-thread body: drain the queue, convert Annex-B to AVCC, and mux. Output starts at +/// the first IDR with parameter sets; earlier frames are discarded. Runs until every sender +/// is dropped (stop) or a write error occurs, then finalizes the file and publishes the +/// final status. +fn writer_thread( + rx: Receiver, + file: std::fs::File, + path: String, + backend: &'static str, + mode_name: &'static str, + shared: Arc, +) { + let mut writer = mp4::FragmentWriter::new(std::io::BufWriter::new(file)); + let mut builder = mp4::H264SampleBuilder::new(); + + 'recv: for (buf, offset, pts_us) in rx.iter() { + let Some(sample) = builder.build_sample(&buf[offset..]) else { continue }; + if !writer.init_written() { + if !sample.sync || !builder.have_parameter_sets() { + continue; + } + let Some(cfg) = builder.track_config() else { + shared.set_error("failed to parse H.264 SPS for MP4 init".to_string()); + break 'recv; + }; + shared.width.store(cfg.width, Ordering::Relaxed); + shared.height.store(cfg.height, Ordering::Relaxed); + if let Err(e) = writer.write_init(&cfg) { + shared.set_error(format!("MP4 init write failed: {e}")); + break 'recv; + } + } + if let Err(e) = writer.push_sample(sample.data, sample.sync, pts_us) { + shared.set_error(format!("MP4 write failed: {e}")); + break 'recv; + } + let st = writer.stats(); + shared.muxed.store(st.samples + 1, Ordering::Relaxed); // +1: one sample is buffered + shared.sync_frames.store(st.sync_samples, Ordering::Relaxed); + shared.bytes.store(st.bytes, Ordering::Relaxed); + } + + let final_stats = match writer.finish() { + Ok(st) => st, + Err(e) => { + shared.set_error(format!("MP4 finalize failed: {e}")); + mp4::Mp4Stats::default() + } + }; + if final_stats.samples == 0 { + shared.set_error( + "no recordable H.264 frames received (the session must produce a single \ + full-frame H.264 stream; JPEG and striped modes cannot be recorded)" + .to_string(), + ); + let _ = std::fs::remove_file(&path); + } + let status = RecordingStatus { + active: false, + path, + backend, + mode: mode_name, + frames: final_stats.samples, + sync_frames: final_stats.sync_samples, + dropped: shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: final_stats.bytes, + duration_s: final_stats.duration_us as f64 / 1e6, + width: shared.width.load(Ordering::Relaxed), + height: shared.height.load(Ordering::Relaxed), + error: shared.error.lock().unwrap().clone(), + }; + println!( + "[recorder] finished {}: {} frames ({} sync), {:.2}s, {} bytes", + status.path, status.frames, status.sync_frames, status.duration_s, status.bytes + ); + *LAST_FINISHED.lock().unwrap() = Some(status); +} + +/// Capture settings for a recorder-owned capture: explicit settings when given (validated +/// H.264), otherwise derived defaults, always forced to the one recordable shape (full-frame +/// H.264, no socket rebind, scheduled recovery keyframes). +fn own_capture_settings(opts: &RecordOptions) -> Result { + let mut s = match &opts.capture { + Some(explicit) => { + if explicit.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + explicit.clone() + } + None => RustCaptureSettings { + width: 0, + height: 0, + output_mode: 1, + capture_cursor: true, + target_fps: opts.effective_fps(), + ..Default::default() + }, + }; + s.video_fullframe = true; + s.recording_socket = String::new(); + if opts.fps > 0.0 { + s.target_fps = opts.fps; + } + if opts.bitrate_kbps > 0 { + s.video_bitrate_kbps = opts.bitrate_kbps; + } + if s.keyframe_interval_s <= 0.0 { + s.keyframe_interval_s = opts.effective_keyframe_s(); + } + Ok(s) +} + +/// Start a recording. Exactly one may be active per process; returns the initial status. +pub fn start(opts: RecordOptions) -> Result { + let mut guard = ACTIVE.lock().unwrap(); + if guard.is_some() { + return Err("a recording is already active".to_string()); + } + if opts.path.is_empty() { + return Err("recording path is empty".to_string()); + } + if let Some(cap) = &opts.capture { + if cap.output_mode != 1 { + return Err( + "recording requires H.264 capture settings (output_mode=1); JPEG cannot be recorded" + .to_string(), + ); + } + } + + let wl_tx = crate::computer_use::wayland_command_sender(); + let backend = match opts.backend { + Some(PreferredBackend::X11) => PreferredBackend::X11, + Some(PreferredBackend::Wayland) => { + if wl_tx.is_none() { + return Err("no Wayland compositor is running in this process".to_string()); + } + PreferredBackend::Wayland + } + None => { + if wl_tx.is_some() { + PreferredBackend::Wayland + } else if std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false) { + PreferredBackend::X11 + } else { + return Err( + "no capture backend available: no in-process Wayland compositor and DISPLAY is unset" + .to_string(), + ); + } + } + }; + + let file = std::fs::File::create(&opts.path) + .map_err(|e| format!("cannot create {}: {e}", opts.path))?; + let shared = RecShared::new(); + let (tx, rx) = bounded::(QUEUE_CAP); + + let (mode, mode_name, backend_name) = match backend { + PreferredBackend::Wayland => { + let cmd_tx = wl_tx.unwrap(); + let display_id = opts.display_id; + let attached = crate::wayland_alive().lock().unwrap().contains(&display_id); + if attached { + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: Some(IdrRequester { tx: cmd_tx.clone(), display_id }), + keyframe_interval_us: (opts.effective_keyframe_s() * 1e6) as u64, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + // The stream is mid-GOP: a standard request-IDR gives the file its first + // decodable frame promptly. + let _ = cmd_tx.send(ThreadCommand::RequestIdr { display_id }); + (RecordingMode::WaylandAttached, "attached", "wayland") + } else { + let mut settings = own_capture_settings(&opts)?; + if settings.width <= 0 || settings.height <= 0 { + let (reply_tx, reply_rx) = mpsc::channel(); + cmd_tx + .send(ThreadCommand::ListOutputs { reply: reply_tx }) + .map_err(|_| "wayland compositor is not accepting commands".to_string())?; + let outputs = reply_rx + .recv_timeout(Duration::from_secs(5)) + .map_err(|_| "wayland compositor did not report its outputs".to_string())?; + let out = outputs + .iter() + .find(|o| o.0 == display_id) + .ok_or_else(|| format!("no wayland output with display id {display_id}"))?; + settings.width = out.3; + settings.height = out.4; + settings.scale = out.5; + } + *WL_TAP.lock().unwrap() = Some(WlTap { + display_id, + tx: tx.clone(), + shared: shared.clone(), + idr: None, + keyframe_interval_us: 0, + }); + WL_TAP_ARMED.store(true, Ordering::Relaxed); + cmd_tx + .send(ThreadCommand::StartCapture { display_id, callback: None, settings }) + .map_err(|_| { + disarm_tap(); + "wayland compositor is not accepting commands".to_string() + })?; + // StartCapture has no reply; liveness shows up in the alive set. + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if crate::wayland_alive().lock().unwrap().contains(&display_id) { + break; + } + if Instant::now() >= deadline { + disarm_tap(); + return Err(format!( + "wayland capture on display {display_id} failed to start" + )); + } + thread::sleep(Duration::from_millis(20)); + } + (RecordingMode::WaylandOwn { display_id }, "own-capture", "wayland") + } + } + PreferredBackend::X11 => { + let settings = own_capture_settings(&opts)?; + let controls = Arc::new(crate::x11::Controls::new(&settings)); + crate::live_x11().lock().unwrap().push(controls.clone()); + let err_slot: Arc>> = Arc::new(Mutex::new(None)); + let err_slot2 = err_slot.clone(); + let (etid_tx, etid_rx) = mpsc::channel(); + let cap_controls = controls.clone(); + let feed_tx = tx.clone(); + let feed_shared = shared.clone(); + let join = thread::spawn(move || { + let on_frame = move |stripes: Vec| { + offer_frame(&feed_shared, &feed_tx, &stripes); + }; + if let Err(e) = crate::x11::run_capture(settings, cap_controls, etid_tx, on_frame) { + eprintln!("[recorder] x11 capture error: {e}"); + *err_slot2.lock().unwrap() = Some(e); + } + }); + match etid_rx.recv_timeout(Duration::from_secs(3)) { + Ok(_) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + let msg = err_slot + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "X11 capture exited during start".to_string()); + return Err(msg); + } + // Slow X server setup: the capture is still coming up; proceed. + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + (RecordingMode::X11Own { controls, join }, "own-capture", "x11") + } + }; + + let writer = { + let shared = shared.clone(); + let path = opts.path.clone(); + thread::Builder::new() + .name("pf-recorder".to_string()) + .spawn(move || writer_thread(rx, file, path, backend_name, mode_name, shared)) + .map_err(|e| format!("failed to spawn recorder writer thread: {e}"))? + }; + + println!( + "[recorder] recording to {} ({} {})", + opts.path, backend_name, mode_name + ); + let status = RecordingStatus { + active: true, + path: opts.path.clone(), + backend: backend_name, + mode: mode_name, + frames: 0, + sync_frames: 0, + dropped: 0, + skipped_non_h264: 0, + bytes: 0, + duration_s: 0.0, + width: 0, + height: 0, + error: None, + }; + *guard = Some(ActiveRecording { + path: opts.path, + backend: backend_name, + mode_name, + mode, + tx, + shared, + writer, + }); + Ok(status) +} + +fn disarm_tap() { + WL_TAP_ARMED.store(false, Ordering::Relaxed); + *WL_TAP.lock().unwrap() = None; +} + +/// Stop the active recording, finalize the MP4, and return the final status. Errors when no +/// recording is active or when nothing recordable was ever received. +pub fn stop() -> Result { + let active = ACTIVE + .lock() + .unwrap() + .take() + .ok_or_else(|| "no recording is active".to_string())?; + disarm_tap(); + match active.mode { + RecordingMode::X11Own { controls, join } => { + controls.stop.store(true, Ordering::Relaxed); + let _ = join.join(); + crate::live_x11().lock().unwrap().retain(|c| !Arc::ptr_eq(c, &controls)); + } + RecordingMode::WaylandOwn { display_id } => { + // A streaming client that reconfigured this display now owns it; the capture + // must survive the recorder's exit. + let client_owns = crate::wayland_owners().lock().unwrap().contains_key(&display_id); + if !client_owns { + if let Some(tx) = crate::computer_use::wayland_command_sender() { + let _ = tx.send(ThreadCommand::StopCapture { display_id }); + // The next start classifies attached-vs-own against wayland_alive: + // wait for the compositor to drain the stop, or an immediate restart + // attaches to the dying capture and records nothing. + let (ack_tx, ack_rx) = mpsc::channel(); + if tx.send(ThreadCommand::Barrier { reply: ack_tx }).is_ok() { + let _ = ack_rx.recv_timeout(Duration::from_secs(2)); + } + } + } + } + RecordingMode::WaylandAttached => {} + } + drop(active.tx); + let _ = active.writer.join(); + let finished = LAST_FINISHED.lock().unwrap().clone().ok_or_else(|| { + format!("recording of {} produced no final status", active.path) + })?; + match finished.error { + Some(ref e) => Err(e.clone()), + None => Ok(finished), + } +} + +/// The current status: the live recording when one is active, otherwise the last finished +/// one; `None` when the process has never recorded. +pub fn status() -> Option { + let guard = ACTIVE.lock().unwrap(); + if let Some(a) = guard.as_ref() { + return Some(RecordingStatus { + active: true, + path: a.path.clone(), + backend: a.backend, + mode: a.mode_name, + frames: a.shared.muxed.load(Ordering::Relaxed), + sync_frames: a.shared.sync_frames.load(Ordering::Relaxed), + dropped: a.shared.dropped.load(Ordering::Relaxed), + skipped_non_h264: a.shared.skipped_non_h264.load(Ordering::Relaxed), + bytes: a.shared.bytes.load(Ordering::Relaxed), + duration_s: a.shared.start.elapsed().as_secs_f64(), + width: a.shared.width.load(Ordering::Relaxed), + height: a.shared.height.load(Ordering::Relaxed), + error: a.shared.error.lock().unwrap().clone(), + }); + } + drop(guard); + LAST_FINISHED.lock().unwrap().clone() +} + +/// Finalize any active recording at interpreter shutdown so the MP4's last buffered sample +/// is flushed. Best-effort; the fragmented layout keeps even an unflushed file playable. +pub fn finalize_on_exit() { + if ACTIVE.lock().unwrap().is_some() { + let _ = stop(); + } +} + +/// `PIXELFLUX_RECORD=`: record from process start. X11 (via `DISPLAY`) starts +/// immediately; otherwise a background thread waits for the in-process Wayland compositor to +/// come up and then starts. Retries transient failures until the deadline. +pub fn autostart_from_env() { + use std::sync::OnceLock; + static STARTED: OnceLock<()> = OnceLock::new(); + let Ok(path) = std::env::var("PIXELFLUX_RECORD") else { return }; + if path.is_empty() { + return; + } + let mut first = false; + STARTED.get_or_init(|| first = true); + if !first { + return; + } + thread::spawn(move || { + let opts = RecordOptions::from_env(path); + let deadline = Instant::now() + Duration::from_secs(300); + let mut last_err = String::new(); + loop { + let wayland_up = crate::computer_use::wayland_command_sender().is_some(); + let x11_up = std::env::var("DISPLAY").map(|v| !v.is_empty()).unwrap_or(false); + let ready = match opts.backend { + Some(PreferredBackend::X11) => x11_up, + Some(PreferredBackend::Wayland) => wayland_up, + None => wayland_up || x11_up, + }; + if ready { + match start(opts.clone()) { + Ok(_) => return, + Err(e) => last_err = e, + } + } + if Instant::now() >= deadline { + eprintln!( + "[recorder] PIXELFLUX_RECORD autostart gave up: {}", + if last_err.is_empty() { "no capture backend appeared" } else { &last_err } + ); + return; + } + thread::sleep(Duration::from_millis(250)); + } + }); +} + +/// Serialize a status into the JSON shape shared by the REST endpoints. +pub fn status_to_json(s: &RecordingStatus) -> serde_json::Value { + serde_json::json!({ + "active": s.active, + "path": s.path, + "backend": s.backend, + "mode": s.mode, + "frames": s.frames, + "sync_frames": s.sync_frames, + "dropped": s.dropped, + "skipped_non_h264": s.skipped_non_h264, + "bytes": s.bytes, + "duration_s": s.duration_s, + "width": s.width, + "height": s.height, + "error": s.error, + }) +} diff --git a/pixelflux/src/recorder/mp4.rs b/pixelflux/src/recorder/mp4.rs new file mode 100644 index 0000000..5e96c13 --- /dev/null +++ b/pixelflux/src/recorder/mp4.rs @@ -0,0 +1,806 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Pure-Rust fragmented-MP4 (fMP4) muxer for the built-in recorder. +//! +//! Hand-rolled rather than pulled in as a dependency for two reasons: `ffmpeg-sys-next`'s +//! `avformat` feature would add libavformat as a hard runtime dependency of every build, and the +//! pure-Rust mp4 crates only write moov-trailing progressive files, which lose everything on a +//! crash. Fragmented MP4 needs no trailer and no seeking — each `moof`+`mdat` pair is +//! self-contained — so a file truncated by a crash or SIGKILL stays playable up to the last +//! fragment, and the writer works on any `Write` sink. +//! +//! Timestamps are caller-supplied wall-clock microseconds (damage-driven capture emits sparse, +//! irregular frames), carried at a 90 kHz track timescale with one sample per fragment: `tfdt` +//! anchors every sample at its true capture time, so variable framerate needs no constant-rate +//! lie. The sample duration is only known once the NEXT frame arrives, so one sample is always +//! buffered and flushed a frame behind (a clean stop closes it with the median observed +//! duration); on SIGKILL at most that one buffered frame is lost — every fragment already +//! written remains playable. +//! +//! Codec support is deliberately split: [`annexb_to_sample`] and the H.264-specific parameter-set +//! capture live in [`H264SampleBuilder`], while the fragment/box writer below is codec-agnostic +//! (bytes + sync flag + timestamps + a ready-made `stsd` sample entry). HEVC or AV1 recording +//! later means a new sample builder emitting an `hvc1`/`av01` entry, not a new muxer. + +use std::io::Write; + +/// 90 kHz: the conventional H.264 track timescale, exactly representing common frame intervals. +const TIMESCALE: u32 = 90_000; + +/// Fallback duration for the final buffered sample when only one frame was ever written +/// (no observed inter-frame delta to take a median of): 1/30 s. +const DEFAULT_LAST_DURATION: u32 = TIMESCALE / 30; + +/// Split an Annex-B elementary stream into NAL payloads (start codes removed, emulation +/// prevention bytes kept — the RBSP layer is only unescaped where a parser needs it). +pub fn split_annexb(data: &[u8]) -> Vec<&[u8]> { + let mut nals = Vec::new(); + let mut i = 0usize; + let mut nal_start: Option = None; + while i + 2 < data.len() { + if data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 { + let code_start = if i > 0 && data[i - 1] == 0 { i - 1 } else { i }; + if let Some(s) = nal_start { + if code_start > s { + nals.push(&data[s..code_start]); + } + } + i += 3; + nal_start = Some(i); + } else if data[i + 2] == 0 { + // A zero at i+2 can begin the next start code; only advance one byte. + i += 1; + } else { + i += 3; + } + } + if let Some(s) = nal_start { + if data.len() > s { + nals.push(&data[s..]); + } + } + nals +} + +/// Unescape an H.264 RBSP: drop the emulation-prevention byte from every `00 00 03` run. +fn unescape_rbsp(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len()); + let mut zeros = 0u32; + for &b in data { + if zeros >= 2 && b == 3 { + zeros = 0; + continue; + } + if b == 0 { + zeros += 1; + } else { + zeros = 0; + } + out.push(b); + } + out +} + +/// MSB-first bit reader over an unescaped RBSP, with Exp-Golomb decode for SPS parsing. +struct BitReader<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + Self { data, pos: 0 } + } + + fn bit(&mut self) -> Option { + let byte = *self.data.get(self.pos / 8)?; + let bit = (byte >> (7 - (self.pos % 8))) & 1; + self.pos += 1; + Some(bit as u32) + } + + fn bits(&mut self, n: u32) -> Option { + let mut v = 0u32; + for _ in 0..n { + v = (v << 1) | self.bit()?; + } + Some(v) + } + + /// ue(v): count leading zeros, then read that many bits after the marker one. + fn ue(&mut self) -> Option { + let mut zeros = 0u32; + while self.bit()? == 0 { + zeros += 1; + if zeros > 31 { + return None; + } + } + let rest = self.bits(zeros)?; + Some((1u32 << zeros) - 1 + rest) + } + + fn se(&mut self) -> Option { + let k = self.ue()? as i64; + Some(if k % 2 == 0 { -(k / 2) as i32 } else { ((k + 1) / 2) as i32 }) + } +} + +/// Coded frame dimensions parsed out of an H.264 SPS NAL (with its NAL header byte). +/// +/// Walks every field ahead of `pic_width_in_mbs_minus1` — including the high-profile +/// chroma/bit-depth/scaling-list block — and applies the frame-cropping rectangle with the +/// chroma-format-dependent crop units, so 4:2:0, 4:2:2 and 4:4:4 streams from any of the +/// project's encoders all report their true display size. +pub fn parse_sps_dimensions(sps_nal: &[u8]) -> Option<(u32, u32)> { + if sps_nal.len() < 4 || sps_nal[0] & 0x1f != 7 { + return None; + } + let profile_idc = sps_nal[1]; + let rbsp = unescape_rbsp(&sps_nal[4..]); + let mut r = BitReader::new(&rbsp); + r.ue()?; // seq_parameter_set_id + + let mut chroma_format_idc = 1u32; + if matches!( + profile_idc, + 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135 + ) { + chroma_format_idc = r.ue()?; + if chroma_format_idc == 3 { + r.bit()?; // separate_colour_plane_flag + } + r.ue()?; // bit_depth_luma_minus8 + r.ue()?; // bit_depth_chroma_minus8 + r.bit()?; // qpprime_y_zero_transform_bypass_flag + if r.bit()? == 1 { + // seq_scaling_matrix_present_flag + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for i in 0..lists { + if r.bit()? == 1 { + let size = if i < 6 { 16 } else { 64 }; + let mut next_scale = 8i32; + let mut last_scale = 8i32; + for _ in 0..size { + if next_scale != 0 { + let delta = r.se()?; + next_scale = (last_scale + delta + 256) % 256; + } + if next_scale != 0 { + last_scale = next_scale; + } + } + } + } + } + } + + r.ue()?; // log2_max_frame_num_minus4 + let pic_order_cnt_type = r.ue()?; + if pic_order_cnt_type == 0 { + r.ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } else if pic_order_cnt_type == 1 { + r.bit()?; // delta_pic_order_always_zero_flag + r.se()?; // offset_for_non_ref_pic + r.se()?; // offset_for_top_to_bottom_field + let n = r.ue()?; + for _ in 0..n { + r.se()?; + } + } + r.ue()?; // max_num_ref_frames + r.bit()?; // gaps_in_frame_num_value_allowed_flag + let pic_width_in_mbs = r.ue()? + 1; + let pic_height_in_map_units = r.ue()? + 1; + let frame_mbs_only = r.bit()?; + if frame_mbs_only == 0 { + r.bit()?; // mb_adaptive_frame_field_flag + } + r.bit()?; // direct_8x8_inference_flag + + let (mut crop_l, mut crop_r, mut crop_t, mut crop_b) = (0u32, 0u32, 0u32, 0u32); + if r.bit()? == 1 { + crop_l = r.ue()?; + crop_r = r.ue()?; + crop_t = r.ue()?; + crop_b = r.ue()?; + } + + let (sub_w, sub_h) = match chroma_format_idc { + 0 | 3 => (1u32, 1u32), + 2 => (2, 1), + _ => (2, 2), + }; + let crop_unit_x = sub_w; + let crop_unit_y = sub_h * (2 - frame_mbs_only); + let width = pic_width_in_mbs * 16 - crop_unit_x * (crop_l + crop_r); + let height = (2 - frame_mbs_only) * pic_height_in_map_units * 16 - crop_unit_y * (crop_t + crop_b); + Some((width, height)) +} + +fn mk_box(fourcc: &[u8; 4], payload: &[u8]) -> Vec { + let mut b = Vec::with_capacity(8 + payload.len()); + b.extend_from_slice(&((8 + payload.len()) as u32).to_be_bytes()); + b.extend_from_slice(fourcc); + b.extend_from_slice(payload); + b +} + +fn mk_full_box(fourcc: &[u8; 4], version: u8, flags: u32, payload: &[u8]) -> Vec { + let mut p = Vec::with_capacity(4 + payload.len()); + p.push(version); + p.extend_from_slice(&flags.to_be_bytes()[1..]); + p.extend_from_slice(payload); + mk_box(fourcc, &p) +} + +const MATRIX_IDENTITY: [u8; 36] = { + let mut m = [0u8; 36]; + m[1] = 0x01; // 0x00010000 + m[17] = 0x01; + m[32] = 0x40; // 0x40000000 + m +}; + +/// A codec's contribution to the init segment: its `stsd` sample entry (with decoder +/// configuration record) plus the display dimensions for `tkhd`. +pub struct TrackConfig { + pub sample_entry: Vec, + pub width: u32, + pub height: u32, +} + +/// One buffered access unit awaiting its duration (known when the next one arrives). +struct PendingSample { + data: Vec, + sync: bool, + dts: u64, +} + +/// Aggregate counters reported when the writer finishes. +#[derive(Clone, Copy, Debug, Default)] +pub struct Mp4Stats { + pub samples: u64, + pub sync_samples: u64, + pub bytes: u64, + pub duration_us: u64, +} + +/// Codec-agnostic fMP4 fragment writer: `ftyp`+`moov` once, then one `moof`+`mdat` pair +/// per sample, each anchored at its wall-clock decode time via `tfdt`. +pub struct FragmentWriter { + out: W, + seq: u32, + wrote_init: bool, + pending: Option, + last_dts: Option, + /// Every flushed inter-frame duration, kept so the final buffered sample (whose + /// successor never arrives) can close on the MEDIAN — damage-driven capture makes the + /// last observed delta an outlier as often as not (a long static gap, a burst pair). + durations: Vec, + stats: Mp4Stats, +} + +impl FragmentWriter { + pub fn new(out: W) -> Self { + Self { + out, + seq: 0, + wrote_init: false, + pending: None, + last_dts: None, + durations: Vec::new(), + stats: Mp4Stats::default(), + } + } + + pub fn init_written(&self) -> bool { + self.wrote_init + } + + /// Counters for the fragments written so far (the buffered pending sample is not yet + /// included; `finish` folds it in). + pub fn stats(&self) -> Mp4Stats { + self.stats + } + + /// Write `ftyp` + `moov` (track 1, `mvex`/`trex` marking the movie fragmented). Must be + /// called once, before the first sample. + pub fn write_init(&mut self, cfg: &TrackConfig) -> std::io::Result<()> { + let mut ftyp_p = Vec::new(); + ftyp_p.extend_from_slice(b"isom"); + ftyp_p.extend_from_slice(&0x200u32.to_be_bytes()); + for brand in [b"isom", b"iso5", b"iso6", b"avc1", b"mp41"] { + ftyp_p.extend_from_slice(brand); + } + let ftyp = mk_box(b"ftyp", &ftyp_p); + + let mut mvhd_p = Vec::new(); + mvhd_p.extend_from_slice(&[0u8; 8]); // creation/modification time + mvhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mvhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration: unknown (fragmented) + mvhd_p.extend_from_slice(&0x00010000u32.to_be_bytes()); // rate 1.0 + mvhd_p.extend_from_slice(&0x0100u16.to_be_bytes()); // volume 1.0 + mvhd_p.extend_from_slice(&[0u8; 10]); // reserved + mvhd_p.extend_from_slice(&MATRIX_IDENTITY); + mvhd_p.extend_from_slice(&[0u8; 24]); // pre_defined + mvhd_p.extend_from_slice(&2u32.to_be_bytes()); // next_track_ID + let mvhd = mk_full_box(b"mvhd", 0, 0, &mvhd_p); + + let mut tkhd_p = Vec::new(); + tkhd_p.extend_from_slice(&[0u8; 8]); + tkhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + tkhd_p.extend_from_slice(&[0u8; 4]); // reserved + tkhd_p.extend_from_slice(&0u32.to_be_bytes()); // duration + tkhd_p.extend_from_slice(&[0u8; 16]); // reserved, layer, group, volume, reserved + tkhd_p.extend_from_slice(&MATRIX_IDENTITY); + tkhd_p.extend_from_slice(&(cfg.width << 16).to_be_bytes()); + tkhd_p.extend_from_slice(&(cfg.height << 16).to_be_bytes()); + let tkhd = mk_full_box(b"tkhd", 0, 3, &tkhd_p); // enabled | in_movie + + let mut mdhd_p = Vec::new(); + mdhd_p.extend_from_slice(&[0u8; 8]); + mdhd_p.extend_from_slice(&TIMESCALE.to_be_bytes()); + mdhd_p.extend_from_slice(&0u32.to_be_bytes()); + mdhd_p.extend_from_slice(&0x55c4u16.to_be_bytes()); // language: und + mdhd_p.extend_from_slice(&[0u8; 2]); + let mdhd = mk_full_box(b"mdhd", 0, 0, &mdhd_p); + + let mut hdlr_p = Vec::new(); + hdlr_p.extend_from_slice(&[0u8; 4]); // pre_defined + hdlr_p.extend_from_slice(b"vide"); + hdlr_p.extend_from_slice(&[0u8; 12]); + hdlr_p.extend_from_slice(b"pixelflux\0"); + let hdlr = mk_full_box(b"hdlr", 0, 0, &hdlr_p); + + let vmhd = mk_full_box(b"vmhd", 0, 1, &[0u8; 8]); + let url = mk_full_box(b"url ", 0, 1, &[]); // self-contained + let mut dref_p = 1u32.to_be_bytes().to_vec(); + dref_p.extend_from_slice(&url); + let dref = mk_full_box(b"dref", 0, 0, &dref_p); + let dinf = mk_box(b"dinf", &dref); + + let mut stsd_p = 1u32.to_be_bytes().to_vec(); + stsd_p.extend_from_slice(&cfg.sample_entry); + let stsd = mk_full_box(b"stsd", 0, 0, &stsd_p); + let stts = mk_full_box(b"stts", 0, 0, &0u32.to_be_bytes()); + let stsc = mk_full_box(b"stsc", 0, 0, &0u32.to_be_bytes()); + let stsz = mk_full_box(b"stsz", 0, 0, &[0u8; 8]); + let stco = mk_full_box(b"stco", 0, 0, &0u32.to_be_bytes()); + let stbl = mk_box( + b"stbl", + &[stsd, stts, stsc, stsz, stco].concat(), + ); + + let minf = mk_box(b"minf", &[vmhd, dinf, stbl].concat()); + let mdia = mk_box(b"mdia", &[mdhd, hdlr, minf].concat()); + let trak = mk_box(b"trak", &[tkhd, mdia].concat()); + + let mut trex_p = Vec::new(); + trex_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + trex_p.extend_from_slice(&1u32.to_be_bytes()); // default_sample_description_index + trex_p.extend_from_slice(&[0u8; 12]); // default duration/size/flags + let trex = mk_full_box(b"trex", 0, 0, &trex_p); + let mvex = mk_box(b"mvex", &trex); + + let moov = mk_box(b"moov", &[mvhd, trak, mvex].concat()); + + self.out.write_all(&ftyp)?; + self.out.write_all(&moov)?; + self.stats.bytes += (ftyp.len() + moov.len()) as u64; + self.wrote_init = true; + Ok(()) + } + + /// Queue one sample at `pts_us` (wall-clock microseconds since recording start), + /// flushing the previously buffered sample with its now-known duration. Timestamps are + /// clamped strictly monotonic so a repeated or reordered clock can never emit a + /// zero/negative duration. + pub fn push_sample(&mut self, data: Vec, sync: bool, pts_us: u64) -> std::io::Result<()> { + let mut dts = pts_us * (TIMESCALE as u64 / 1000) / 1000; + if let Some(last) = self.last_dts { + if dts <= last { + dts = last + 1; + } + } + self.last_dts = Some(dts); + if let Some(prev) = self.pending.take() { + let duration = (dts - prev.dts).min(u32::MAX as u64) as u32; + self.durations.push(duration); + self.write_fragment(&prev, duration)?; + } + self.pending = Some(PendingSample { data, sync, dts }); + Ok(()) + } + + /// Median of the observed inter-frame durations (default 1/30 s when fewer than two + /// frames were pushed), used to close the final sample. + fn median_duration(&self) -> u32 { + if self.durations.is_empty() { + return DEFAULT_LAST_DURATION; + } + let mut sorted = self.durations.clone(); + sorted.sort_unstable(); + sorted[sorted.len() / 2] + } + + fn write_fragment(&mut self, s: &PendingSample, duration: u32) -> std::io::Result<()> { + self.seq += 1; + + let mut tfhd_p = Vec::new(); + tfhd_p.extend_from_slice(&1u32.to_be_bytes()); // track_ID + let tfhd = mk_full_box(b"tfhd", 0, 0x020000, &tfhd_p); // default-base-is-moof + + let mut tfdt_p = Vec::new(); + tfdt_p.extend_from_slice(&s.dts.to_be_bytes()); + let tfdt = mk_full_box(b"tfdt", 1, 0, &tfdt_p); + + // sample flags: sync = "depends on nothing"; non-sync also sets the non-sync bit. + let sample_flags: u32 = if s.sync { 0x0200_0000 } else { 0x0101_0000 }; + let mut trun_p = Vec::new(); + trun_p.extend_from_slice(&1u32.to_be_bytes()); // sample_count + trun_p.extend_from_slice(&0i32.to_be_bytes()); // data_offset placeholder + trun_p.extend_from_slice(&duration.to_be_bytes()); + trun_p.extend_from_slice(&(s.data.len() as u32).to_be_bytes()); + trun_p.extend_from_slice(&sample_flags.to_be_bytes()); + // data-offset | sample-duration | sample-size | sample-flags + let mut trun = mk_full_box(b"trun", 0, 0x000701, &trun_p); + + let traf_len = 8 + tfhd.len() + tfdt.len() + trun.len(); + let moof_len = 8 + 16 + traf_len; // mfhd is 16 bytes + // First sample byte sits just past the mdat header, relative to moof start. + let data_offset = (moof_len + 8) as i32; + let off_pos = trun.len() - trun_p.len() + 4; // into trun payload: after sample_count + trun[off_pos..off_pos + 4].copy_from_slice(&data_offset.to_be_bytes()); + + let mfhd = mk_full_box(b"mfhd", 0, 0, &self.seq.to_be_bytes()); + let traf = mk_box(b"traf", &[tfhd, tfdt, trun].concat()); + let moof = mk_box(b"moof", &[mfhd, traf].concat()); + debug_assert_eq!(moof.len(), moof_len); + + self.out.write_all(&moof)?; + self.out.write_all(&((8 + s.data.len()) as u32).to_be_bytes())?; + self.out.write_all(b"mdat")?; + self.out.write_all(&s.data)?; + self.out.flush()?; + + self.stats.samples += 1; + if s.sync { + self.stats.sync_samples += 1; + } + self.stats.bytes += (moof.len() + 8 + s.data.len()) as u64; + self.stats.duration_us = (s.dts + duration as u64) * 1000 / (TIMESCALE as u64 / 1000); + Ok(()) + } + + /// Flush the final buffered sample, closed with the MEDIAN observed inter-frame + /// duration (its successor never arrives), and return the aggregate counters. + pub fn finish(mut self) -> std::io::Result { + if let Some(prev) = self.pending.take() { + let d = self.median_duration(); + self.write_fragment(&prev, d)?; + } + self.out.flush()?; + Ok(self.stats) + } +} + +/// H.264-specific front end: captures SPS/PPS from the stream, gates output on the first +/// IDR, converts Annex-B access units to AVCC samples, and builds the `avc1` sample entry. +pub struct H264SampleBuilder { + sps: Option>, + pps: Option>, +} + +/// One converted access unit ready for the fragment writer. +pub struct BuiltSample { + pub data: Vec, + pub sync: bool, +} + +impl Default for H264SampleBuilder { + fn default() -> Self { + Self::new() + } +} + +impl H264SampleBuilder { + pub fn new() -> Self { + Self { sps: None, pps: None } + } + + pub fn have_parameter_sets(&self) -> bool { + self.sps.is_some() && self.pps.is_some() + } + + /// Convert one Annex-B access unit into a length-prefixed AVCC sample, harvesting + /// SPS/PPS on the way. Returns `None` for an AU with no slice data (e.g. bare parameter + /// sets). `sync` is true when the AU contains an IDR slice. + pub fn build_sample(&mut self, annexb: &[u8]) -> Option { + let nals = split_annexb(annexb); + let mut data = Vec::with_capacity(annexb.len() + 8); + let mut sync = false; + let mut has_slice = false; + for nal in nals { + if nal.is_empty() { + continue; + } + match nal[0] & 0x1f { + 7 => { + if self.sps.as_deref() != Some(nal) { + self.sps = Some(nal.to_vec()); + } + } + 8 => { + if self.pps.as_deref() != Some(nal) { + self.pps = Some(nal.to_vec()); + } + } + 5 => { + sync = true; + has_slice = true; + } + 1 => has_slice = true, + _ => {} + } + data.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + data.extend_from_slice(nal); + } + if !has_slice { + return None; + } + Some(BuiltSample { data, sync }) + } + + /// Build the `avc1` sample entry + `avcC` record from the captured parameter sets, with + /// the display dimensions parsed from the SPS. + pub fn track_config(&self) -> Option { + let sps = self.sps.as_deref()?; + let pps = self.pps.as_deref()?; + let (width, height) = parse_sps_dimensions(sps)?; + + let mut avcc_p = Vec::new(); + avcc_p.push(1); // configurationVersion + avcc_p.push(sps[1]); // AVCProfileIndication + avcc_p.push(sps[2]); // profile_compatibility + avcc_p.push(sps[3]); // AVCLevelIndication + avcc_p.push(0xff); // lengthSizeMinusOne = 3 + avcc_p.push(0xe1); // numOfSequenceParameterSets = 1 + avcc_p.extend_from_slice(&(sps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(sps); + avcc_p.push(1); // numOfPictureParameterSets + avcc_p.extend_from_slice(&(pps.len() as u16).to_be_bytes()); + avcc_p.extend_from_slice(pps); + let avcc = mk_box(b"avcC", &avcc_p); + + let mut entry_p = Vec::new(); + entry_p.extend_from_slice(&[0u8; 6]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // data_reference_index + entry_p.extend_from_slice(&[0u8; 16]); // pre_defined / reserved + entry_p.extend_from_slice(&(width as u16).to_be_bytes()); + entry_p.extend_from_slice(&(height as u16).to_be_bytes()); + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // horizresolution 72 dpi + entry_p.extend_from_slice(&0x0048_0000u32.to_be_bytes()); // vertresolution + entry_p.extend_from_slice(&[0u8; 4]); // reserved + entry_p.extend_from_slice(&1u16.to_be_bytes()); // frame_count + entry_p.extend_from_slice(&[0u8; 32]); // compressorname + entry_p.extend_from_slice(&0x0018u16.to_be_bytes()); // depth 24 + entry_p.extend_from_slice(&(-1i16).to_be_bytes()); // pre_defined + entry_p.extend_from_slice(&avcc); + let entry = mk_box(b"avc1", &entry_p); + + Some(TrackConfig { sample_entry: entry, width, height }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // x264 SPS at 1284x722 (High 4:2:0: cropping in both axes on odd-macroblock dims). + const SPS_HIGH_1284X722: &str = "67640020acd9405105de788c0440000003004000000f03c60c6580"; + // x264 SPS at 640x360 (Constrained Baseline). + const SPS_BASE_640X360: &str = "6742c01ed900a02ff970110000030001000003003c0f162e48"; + // x264 SPS at 1920x1080 (High 4:4:4 Predictive: chroma_format_idc == 3 path). + const SPS_444_1920X1080: &str = "67f40028919b280f0044fc4e0220000003002000000781e30632c0"; + + fn hex(s: &str) -> Vec { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()) + .collect() + } + + /// Both start-code lengths delimit NALs; payloads come back exactly, with no framing. + #[test] + fn split_annexb_handles_mixed_start_codes() { + let mut data = Vec::new(); + data.extend_from_slice(&[0, 0, 0, 1, 0x67, 0xAA, 0xBB]); + data.extend_from_slice(&[0, 0, 1, 0x68, 0xCC]); + data.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + let nals = split_annexb(&data); + assert_eq!(nals.len(), 3); + assert_eq!(nals[0], &[0x67, 0xAA, 0xBB]); + assert_eq!(nals[1], &[0x68, 0xCC]); + assert_eq!(nals[2], &[0x65, 0x00, 0x00, 0x03, 0x01, 0xDD]); + } + + /// Dimensions from real x264 SPS across the profiles the project's encoders emit, + /// including the frame-cropping and 4:4:4 chroma paths. + #[test] + fn sps_dimensions_across_profiles() { + assert_eq!(parse_sps_dimensions(&hex(SPS_HIGH_1284X722)), Some((1284, 722))); + assert_eq!(parse_sps_dimensions(&hex(SPS_BASE_640X360)), Some((640, 360))); + assert_eq!(parse_sps_dimensions(&hex(SPS_444_1920X1080)), Some((1920, 1080))); + } + + /// Annex-B -> AVCC: every NAL is length-prefixed, IDR marks sync, parameter sets are + /// harvested, and a parameter-set-only AU yields no sample. + #[test] + fn annexb_to_avcc_sample() { + let sps = hex(SPS_BASE_640X360); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&sps); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + + let mut b = H264SampleBuilder::new(); + assert!(b.build_sample(&[0u8, 0, 0, 1, 0x67, 0x42, 0xc0, 0x1e, 0xd9]).is_none()); + let s = b.build_sample(&au).expect("IDR AU builds a sample"); + assert!(s.sync); + assert!(b.have_parameter_sets()); + // Sample = 3 length-prefixed NALs, sizes preserved. + let mut off = 0usize; + let mut sizes = Vec::new(); + while off < s.data.len() { + let n = u32::from_be_bytes(s.data[off..off + 4].try_into().unwrap()) as usize; + sizes.push(n); + off += 4 + n; + } + assert_eq!(off, s.data.len()); + assert_eq!(sizes, vec![sps.len(), 4, 5]); + + let p = b.build_sample(&[0u8, 0, 0, 1, 0x41, 9, 9]).unwrap(); + assert!(!p.sync); + } + + /// Walk the top-level boxes of a finished two-sample stream: init once, then one + /// moof+mdat pair per sample, with sizes that exactly tile the buffer. + #[test] + fn fragment_stream_box_layout() { + let mut b = H264SampleBuilder::new(); + let mut au = Vec::new(); + au.extend_from_slice(&[0, 0, 0, 1]); + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + let s1 = b.build_sample(&au).unwrap(); + let s2 = b.build_sample(&[0u8, 0, 0, 1, 0x41, 5, 6, 7]).unwrap(); + + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + w.write_init(&b.track_config().unwrap()).unwrap(); + w.push_sample(s1.data, s1.sync, 0).unwrap(); + w.push_sample(s2.data, s2.sync, 33_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 2); + assert_eq!(stats.sync_samples, 1); + assert_eq!(stats.bytes as usize, buf.len()); + + let mut kinds = Vec::new(); + let mut off = 0usize; + while off < buf.len() { + let size = u32::from_be_bytes(buf[off..off + 4].try_into().unwrap()) as usize; + kinds.push(buf[off + 4..off + 8].to_vec()); + assert!(size >= 8 && off + size <= buf.len()); + off += size; + } + assert_eq!(off, buf.len()); + let names: Vec<&str> = kinds.iter().map(|k| std::str::from_utf8(k).unwrap()).collect(); + assert_eq!(names, vec!["ftyp", "moov", "moof", "mdat", "moof", "mdat"]); + } + + /// Every trun sample_duration in stream order (the writer emits one sample per trun). + fn trun_durations(buf: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut off = 0usize; + while off + 24 <= buf.len() { + if &buf[off + 4..off + 8] == b"trun" { + // [size][fourcc][ver+flags][sample_count][data_offset][duration] + out.push(u32::from_be_bytes(buf[off + 20..off + 24].try_into().unwrap())); + } + off += 1; + } + out + } + + /// Each flushed sample's duration is the pts delta to its successor, and the final + /// buffered sample closes with the MEDIAN observed duration — not the last delta, which + /// under damage-driven capture is an outlier as often as not. + #[test] + fn sample_durations_are_pts_deltas_with_median_tail() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + // 33 ms, 33 ms, then a 300 ms static gap before the final frame. + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 33_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 66_000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 366_000).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 4); + // 90 kHz ticks: 33 ms = 2970. The tail closes at the median (2970), not 27000. + assert_eq!(trun_durations(&buf), vec![2970, 2970, 27_000, 2970]); + } + + /// A single-frame recording still closes with a sane nonzero duration. + #[test] + fn single_sample_uses_default_duration() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 0).unwrap(); + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 1); + assert_eq!(trun_durations(&buf), vec![DEFAULT_LAST_DURATION]); + } + + /// Non-monotonic wall-clock input is clamped to strictly increasing decode times, so no + /// fragment can carry a zero or negative duration. + #[test] + fn pts_clamped_strictly_monotonic() { + let mut buf = Vec::new(); + let mut w = FragmentWriter::new(&mut buf); + let cfg = { + let mut b = H264SampleBuilder::new(); + let mut au = vec![0, 0, 0, 1]; + au.extend_from_slice(&hex(SPS_BASE_640X360)); + au.extend_from_slice(&[0, 0, 0, 1, 0x68, 0xCE, 0x38, 0x80]); + au.extend_from_slice(&[0, 0, 0, 1, 0x65, 1]); + b.build_sample(&au); + b.track_config().unwrap() + }; + w.write_init(&cfg).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x65], true, 1000).unwrap(); + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 1000).unwrap(); // repeated clock + w.push_sample(vec![0, 0, 0, 1, 0x41], false, 500).unwrap(); // clock went backwards + let stats = w.finish().unwrap(); + assert_eq!(stats.samples, 3); + + // Extract each tfdt baseMediaDecodeTime and check strict monotonicity. + let mut times = Vec::new(); + let mut off = 0usize; + while off + 8 <= buf.len() { + if &buf[off + 4..off + 8] == b"tfdt" { + let t = u64::from_be_bytes(buf[off + 12..off + 20].try_into().unwrap()); + times.push(t); + } + off += 1; + } + assert_eq!(times.len(), 3); + assert!(times.windows(2).all(|w| w[1] > w[0]), "tfdt times: {times:?}"); + } +} diff --git a/pixelflux/src/recording_sink.rs b/pixelflux/src/recording_sink.rs index b0fa05d..ac96c25 100644 --- a/pixelflux/src/recording_sink.rs +++ b/pixelflux/src/recording_sink.rs @@ -1,73 +1,77 @@ //! Unix-socket H.264 fan-out for external recording. //! -//! A background listener thread binds a Unix domain socket and accepts an arbitrary number of -//! concurrent clients. Each encoded H.264 stripe written through [`RecordingSink`] is broadcast -//! to every connected client as a plain Annex-B elementary stream — the 10-byte wire header -//! (tag, keyframe flag, frame number, y-start, width, height) is stripped so the output is -//! directly muxable. +//! Frames are intercepted at the delivery layer (not inside each encoder), so the tap works +//! uniformly for every full-frame encoder. The 10-byte pixelflux wire header is skipped so +//! consumers receive a plain Annex-B elementary stream that is directly muxable. //! -//! The sink is created once per capture session and shared (via `Arc`) between the encode -//! threads and the delivery layer. When a new client connects, [`RecordingSink::should_force_idr`] -//! returns `true` on the next call, signalling the encoder to emit an IDR so the fresh consumer -//! can start decoding from a clean reference frame. +//! The tap must never perturb the live viewer transport, and it never copies frame bytes: +//! stripe payloads are `Arc`-shared, so the encode thread only clones a handle into a bounded +//! per-client channel drained by a dedicated writer thread. A slow or stalled recorder blocks +//! nothing but itself and is dropped once its queue overflows. A newly connected client arms +//! [`RecordingSink::should_force_idr`] so the next encode emits an IDR it can decode from. use std::fs; use std::io::{ErrorKind, Write}; -use std::os::unix::net::{UnixListener, UnixStream}; +use std::os::unix::net::UnixListener; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; -/// Write timeout applied to each accepted client stream. Keeps a slow consumer from blocking -/// the encode thread; a stalled write simply drops that client. +use crossbeam_channel::{bounded, Sender, TrySendError}; + +use crate::encoders::software::EncodedStripe; + +/// Per-write timeout on a client stream; a stalled write surfaces as a soft error that +/// [`write_all_frame`] retries, keeping the writer thread responsive to teardown. const WRITE_TIMEOUT: Duration = Duration::from_millis(100); /// How often the non-blocking accept loop retries when no client is waiting. const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(50); -/// Environment variable consulted as a fallback when the Python `CaptureSettings.recording_socket` -/// field is empty. -pub const RECORDING_SOCKET_ENV: &str = "PIXELFLUX_RECORDING_SOCKET"; +/// Per-client backlog bound. A recorder that falls this far behind is dropped rather than +/// allowed to grow memory or push back on the shared encode thread. +const CLIENT_QUEUE_CAP: usize = 256; + +/// A queued frame: the `Arc`-shared payload plus the byte offset where the recordable +/// Annex-B stream starts (past the wire header, or `0` when the payload is bare). +type QueuedFrame = (Arc>, usize); + +/// The sink's handle to one connected recorder: the feed end of its bounded queue and a kill +/// switch for its writer thread. The socket itself is owned solely by that writer thread. +struct ClientHandle { + tx: Sender, + stop: Arc, +} /// Unix-socket fan-out that broadcasts every encoded H.264 frame to connected consumers. /// -/// Internally a listener thread accepts connections on a [`UnixListener`] and appends the -/// resulting streams to a shared `Vec`. The encode / delivery threads then call -/// [`write_encoded_frame`] to push data to every client; clients that fall behind or disconnect -/// are removed automatically. -/// -/// [`write_encoded_frame`]: RecordingSink::write_encoded_frame +/// A listener thread accepts connections and gives each its own bounded queue and writer thread +/// (see [`ClientHandle`]) so one slow reader cannot stall the others or the encode thread. pub struct RecordingSink { /// Filesystem path of the Unix socket; removed on drop. path: String, - /// Connected client streams, shared between the accept thread and the write path. - clients: Arc>>, + /// Feed handles for the connected clients, shared with the accept thread. + clients: Arc>>, /// Signals the accept thread to exit; set in [`Drop`]. shutdown: Arc, /// Flipped to `true` each time a new client connects; consumed by [`should_force_idr`]. /// /// [`should_force_idr`]: RecordingSink::should_force_idr client_connected: Arc, + /// One-time notice that the session's H.264 frames are striped and unrecordable. + warned_unrecordable: AtomicBool, } impl RecordingSink { - /// Try to bind a Unix socket at `settings_path` (or the `PIXELFLUX_RECORDING_SOCKET` - /// environment variable when the path is empty). - /// - /// Returns `None` when no path is configured or when the bind fails. On success the - /// listener thread is spawned and the sink is ready to accept clients immediately. + /// Bind a Unix socket at `settings_path`, or return `None` when no path is configured or the + /// bind fails. Recording is an optional tap that must never take the pipeline down, so a bind + /// error is logged and swallowed. pub fn try_bind(settings_path: &str) -> Option> { - let path = if !settings_path.is_empty() { - settings_path.to_string() - } else { - match std::env::var(RECORDING_SOCKET_ENV) { - Ok(p) if !p.is_empty() => p, - _ => return None, - } - }; - - match Self::bind(path) { + if settings_path.is_empty() { + return None; + } + match Self::bind(settings_path.to_string()) { Ok(sink) => Some(Arc::new(sink)), Err(e) => { eprintln!("[recording_sink] bind failed: {:?}", e); @@ -76,13 +80,14 @@ impl RecordingSink { } } - /// Create the socket, spawn the accept thread, and return the sink. + /// Create the socket and spawn the accept thread. Each accepted connection gets a write + /// timeout, a bounded queue, and a writer thread; the sink keeps only the feed handle. fn bind(path: String) -> std::io::Result { let _ = fs::remove_file(&path); let listener = UnixListener::bind(&path)?; listener.set_nonblocking(true)?; - let clients = Arc::new(Mutex::new(Vec::new())); + let clients: Arc>> = Arc::new(Mutex::new(Vec::new())); let shutdown = Arc::new(AtomicBool::new(false)); let client_connected = Arc::new(AtomicBool::new(false)); @@ -100,13 +105,32 @@ impl RecordingSink { eprintln!("[recording_sink] set_write_timeout failed: {:?}", e); continue; } + + let (tx, rx) = bounded::(CLIENT_QUEUE_CAP); + let stop = Arc::new(AtomicBool::new(false)); + let stop_writer = stop.clone(); + thread::spawn(move || { + let mut stream = stream; + for (buf, offset) in rx.iter() { + if stop_writer.load(Ordering::Relaxed) { + break; + } + if let Err(e) = + write_all_frame(&mut stream, &buf[offset..], &stop_writer) + { + eprintln!( + "[recording_sink] writer thread exiting; write failed: {:?}", + e + ); + break; + } + } + }); + let mut guard = clients_acc.lock().unwrap(); - guard.push(stream); + guard.push(ClientHandle { tx, stop }); client_connected_acc.store(true, Ordering::Relaxed); - eprintln!( - "[recording_sink] client connected; total {}", - guard.len() - ); + eprintln!("[recording_sink] client connected; total {}", guard.len()); } Err(e) if e.kind() == ErrorKind::WouldBlock => { thread::sleep(ACCEPT_POLL_INTERVAL); @@ -125,65 +149,196 @@ impl RecordingSink { clients, shutdown, client_connected, + warned_unrecordable: AtomicBool::new(false), }) } - /// Returns `true` exactly once after a new client connects, signalling that the next - /// encode should produce an IDR so the consumer starts from a clean reference frame. + /// Returns `true` exactly once after a new client connects, signalling that the next encode + /// should produce an IDR so the consumer starts from a clean reference frame. pub fn should_force_idr(&self) -> bool { self.client_connected.swap(false, Ordering::Relaxed) } - /// Write `data` to every connected client, silently removing any that are disconnected - /// or too slow. - fn write_all_clients(&self, data: &[u8]) { - if data.is_empty() { + /// Delivery-layer tap for one encoded frame. The socket carries a single H.264 + /// elementary stream, so only a lone full-height stripe (`data_type == 2`) is + /// recordable: striped CPU encodes are N independent per-stripe streams, and + /// interleaving them would produce an undecodable file — those are skipped with a + /// one-time notice (live streaming is unaffected). The 10-byte wire header + /// (`0x04` tag) is skipped via the queued offset so consumers receive plain + /// Annex-B. + /// + /// Never blocks and never copies: the `Arc` payload is cloned into each client's + /// bounded queue with `try_send`, and a client whose queue is full or whose + /// writer died is dropped. + pub fn write_frame(&self, stripes: &[EncodedStripe], full_height: i32) { + let mut h264 = stripes + .iter() + .filter(|s| s.data_type == 2 && !s.data.is_empty()); + let Some(stripe) = h264.next() else { return }; + if h264.next().is_some() || stripe.stripe_y_start != 0 || stripe.stripe_height != full_height + { + if !self.warned_unrecordable.swap(true, Ordering::Relaxed) { + eprintln!( + "[recording_sink] WARNING: striped H.264 frames are not recordable \ + (the socket carries one elementary stream); use a full-frame encoder \ + to record this session" + ); + } return; } + let offset = if stripe.data.len() >= 10 && stripe.data[0] == 0x04 { + 10 + } else { + 0 + }; + if stripe.data.len() == offset { + return; + } + let mut clients = self.clients.lock().unwrap(); if clients.is_empty() { return; } let mut to_remove: Vec = Vec::new(); - for (idx, client) in clients.iter_mut().enumerate() { - if let Err(e) = client.write_all(data) { - if matches!( - e.kind(), - ErrorKind::BrokenPipe | ErrorKind::ConnectionReset | ErrorKind::UnexpectedEof - ) { + for (idx, client) in clients.iter().enumerate() { + match client.tx.try_send((stripe.data.clone(), offset)) { + Ok(()) => {} + Err(TrySendError::Full(_)) => { + eprintln!("[recording_sink] dropping slow client (idx {})", idx); to_remove.push(idx); - } else { - eprintln!("[recording_sink] dropping client (idx {}): {:?}", idx, e); + } + Err(TrySendError::Disconnected(_)) => { to_remove.push(idx); } } } for idx in to_remove.into_iter().rev() { - clients.swap_remove(idx); - } - } - - /// Write a single encoded stripe to all connected clients. - /// - /// Only H.264 frames (`data_type == 2`) are forwarded. When the frame carries the - /// pixelflux wire header (`0x04` tag, 10 bytes) the header is stripped so the output - /// is a plain Annex-B elementary stream suitable for direct muxing. - pub fn write_encoded_frame(&self, stripe: &crate::encoders::software::EncodedStripe) { - if stripe.data.is_empty() || stripe.data_type != 2 { - return; - } - let data = &stripe.data; - if data.len() > 10 && data[0] == 0x04 { - self.write_all_clients(&data[10..]); - } else { - self.write_all_clients(data); + let removed = clients.swap_remove(idx); + removed.stop.store(true, Ordering::Relaxed); } } } impl Drop for RecordingSink { + /// Stop accepting, release every writer thread (set each `stop`, then drop its sender so an + /// idle writer parked on `rx.iter()` wakes), and remove the socket file. fn drop(&mut self) { self.shutdown.store(true, Ordering::Relaxed); + if let Ok(mut clients) = self.clients.lock() { + for client in clients.iter() { + client.stop.store(true, Ordering::Relaxed); + } + clients.clear(); + } let _ = fs::remove_file(&self.path); } } + +#[cfg(test)] +mod cost_tests { + //! The sink's isolation contract, measured: feeding a frame must cost nothing when no + //! recorder is connected (empty-clients early return), microseconds when a healthy + //! recorder drains its queue, and stay bounded (a lock + `try_send`, never a blocking + //! write) when a recorder stalls completely — until the bounded queue overflows and the + //! client is dropped, returning the tap to idle cost. + + use super::*; + use std::io::Read; + use std::os::unix::net::UnixStream; + use std::time::Instant; + + fn frame(len: usize) -> EncodedStripe { + let mut data = vec![0u8; len]; + data[0] = 0x04; // wire-header tag so the 10-byte strip path runs + EncodedStripe { + data: Arc::new(data), + data_type: 2, + stripe_y_start: 0, + stripe_height: 720, + frame_id: 0, + } + } + + fn feed_timed(sink: &RecordingSink, n: usize, len: usize) -> (f64, f64) { + let f = frame(len); + let mut max_us = 0f64; + let mut total_us = 0f64; + for _ in 0..n { + let t = Instant::now(); + sink.write_frame(std::slice::from_ref(&f), 720); + let us = t.elapsed().as_secs_f64() * 1e6; + total_us += us; + max_us = max_us.max(us); + thread::sleep(Duration::from_micros(200)); + } + (total_us / n as f64, max_us) + } + + #[test] + fn stalled_recorder_isolation_cost() { + let path = format!("/tmp/pf-sink-cost-{}.sock", std::process::id()); + let sink = RecordingSink::try_bind(&path).expect("bind"); + + // Idle: no client connected. + let (idle_mean, idle_max) = feed_timed(&sink, 500, 100_000); + + // Healthy: a client draining as fast as it can. + let mut healthy = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let drain = thread::spawn(move || { + let mut buf = vec![0u8; 1 << 20]; + while healthy.read(&mut buf).map(|n| n > 0).unwrap_or(false) {} + }); + let (healthy_mean, healthy_max) = feed_timed(&sink, 500, 100_000); + + // Stalled: a connected client that never reads. The socket buffer fills, then the + // bounded queue fills, then the client is dropped (~256 frames later). + let stalled = UnixStream::connect(&path).expect("connect"); + thread::sleep(Duration::from_millis(200)); + let (stalled_mean, stalled_max) = feed_timed(&sink, 500, 100_000); + drop(stalled); + + println!( + "[sink-cost] idle mean {idle_mean:.3}us max {idle_max:.3}us\n\ + [sink-cost] healthy mean {healthy_mean:.3}us max {healthy_max:.3}us\n\ + [sink-cost] stalled mean {stalled_mean:.3}us max {stalled_max:.3}us" + ); + drop(sink); + let _ = drain.join(); + + assert!(idle_mean < 5.0, "idle feed should be sub-5us, was {idle_mean:.3}us"); + assert!(healthy_mean < 100.0, "healthy feed should be tens of us, was {healthy_mean:.3}us"); + assert!( + stalled_max < 10_000.0, + "a stalled recorder must never block the tap >10ms, was {stalled_max:.3}us" + ); + } +} + +/// Write one whole frame to a recorder's socket, resuming across the soft timeouts a slow reader +/// induces so a partial Annex-B NAL is never left behind. Aborts if `stop` is set (the client was +/// dropped by [`RecordingSink::write_encoded_frame`]) or a hard error occurs. +fn write_all_frame(stream: &mut W, buf: &[u8], stop: &AtomicBool) -> std::io::Result<()> { + let mut written = 0usize; + while written < buf.len() { + if stop.load(Ordering::Relaxed) { + return Err(std::io::Error::other("writer stopped (client dropped)")); + } + match stream.write(&buf[written..]) { + Ok(0) => { + return Err(std::io::Error::new( + ErrorKind::WriteZero, + "failed to write whole frame", + )); + } + Ok(n) => written += n, + Err(ref e) if e.kind() == ErrorKind::TimedOut => {} + Err(ref e) if e.kind() == ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) +} diff --git a/pixelflux/src/wayland/cursor.rs b/pixelflux/src/wayland/cursor.rs index f5e8086..e442603 100644 --- a/pixelflux/src/wayland/cursor.rs +++ b/pixelflux/src/wayland/cursor.rs @@ -9,6 +9,9 @@ //! and render the appropriate frame at the current scale. use image::{ImageBuffer, Rgba}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; +use std::collections::HashMap; use std::io::Cursor as IoCursor; use std::io::Read; use std::time::Duration; @@ -17,6 +20,213 @@ use xcursor::{ CursorTheme, }; +/// One unit of cursor-callback work handed from the calloop thread to the `wl-cursor` +/// worker. The calloop resolves everything renderer/surface-affine (SHM copies, dmabuf +/// readbacks, hotspots); the worker does the PNG encode, the cache, and the GIL-bound Python +/// call so none of that latency lands on the input/render thread. One channel keeps cursor +/// updates ordered with callback (re)registration. +pub enum CursorJob { + SetCallback(Py), + /// Reload the worker's theme handle at a new pixel size; later `Named` jobs render at it. + SetSize(i32), + Named { name: &'static str }, + Hide, + /// wl_shm cursor sprite: raw pool bytes plus the sub-image descriptor. + Shm { + hash: u64, + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, + /// dmabuf cursor sprite already read back to tightly-mapped RGBA on the calloop. + Gles { + hash: u64, + width: i32, + height: i32, + bytes: Vec, + hot_x: i32, + hot_y: i32, + }, +} + +/// Spawn the cursor delivery worker; returns its job channel. The worker owns its own +/// theme handle, the PNG cache, and the Python callback for the life of the process (like the +/// compositor thread itself); `PY_SHUTDOWN` gates every Python call. +pub fn spawn_cursor_worker(cursor_size: i32) -> std::sync::mpsc::Sender { + let (tx, rx) = std::sync::mpsc::channel::(); + let _ = std::thread::Builder::new().name("wl-cursor".into()).spawn(move || { + let mut helper = Cursor::load(cursor_size); + let mut cache: HashMap> = HashMap::new(); + let mut callback: Option> = None; + while let Ok(job) = rx.recv() { + let (msg_type, data, hot_x, hot_y): (&str, Vec, i32, i32) = match job { + CursorJob::SetCallback(cb) => { + callback = Some(cb); + continue; + } + CursorJob::SetSize(size) => { + helper = Cursor::load(size); + continue; + } + CursorJob::Named { name } => match helper.get_png_data(name) { + Some((png, x, y)) => ("png", png, x as i32, y as i32), + None => ("error", Vec::new(), 0, 0), + }, + CursorJob::Hide => ("hide", Vec::new(), 0, 0), + CursorJob::Shm { + hash, + width, + height, + stride, + offset, + opaque, + bytes, + hot_x, + hot_y, + } => { + let png = cached_png(&mut cache, hash, || { + encode_shm_cursor(width, height, stride, offset, opaque, &bytes) + }); + ("png", png, hot_x, hot_y) + } + CursorJob::Gles { hash, width, height, bytes, hot_x, hot_y } => { + let png = cached_png(&mut cache, hash, || { + encode_gles_cursor(width, height, &bytes) + }); + ("png", png, hot_x, hot_y) + } + }; + // A sprite whose pixels could not be read yields empty data; suppressing it + // preserves the consumer's last cursor instead of blanking it (only an + // intentional hide passes with no payload). + if data.is_empty() && msg_type != "hide" { + continue; + } + if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + continue; + } + if let Some(ref cb) = callback { + Python::attach(|py| { + let py_bytes = PyBytes::new(py, &data); + let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); + }); + } + } + }); + tx +} + +/// Content-hash PNG cache lookup with bounded arbitrary eviction; an evicted sprite simply +/// re-encodes on its next appearance. +fn cached_png( + cache: &mut HashMap>, + hash: u64, + encode: impl FnOnce() -> Vec, +) -> Vec { + if let Some(png) = cache.get(&hash) { + return png.clone(); + } + let png = encode(); + if !png.is_empty() { + cache.insert(hash, png.clone()); + if cache.len() > 100 { + if let Some(&evict) = cache.keys().next() { + cache.remove(&evict); + } + } + } + png +} + +/// Convert a wl_shm BGRA/XRGB sprite sub-image to a straight-alpha PNG. Stride/offset are +/// clamped non-negative with checked arithmetic so a garbage descriptor skips pixels instead of +/// panicking; sprites larger than 128x128 are ignored (never a hardware cursor). +fn encode_shm_cursor( + width: i32, + height: i32, + stride: i32, + offset: i32, + opaque: bool, + raw_bytes: &[u8], +) -> Vec { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return Vec::new(); + } + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + let stride_usize = stride.max(0) as usize; + let base_offset = offset.max(0) as usize; + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize) + .checked_mul(stride_usize) + .and_then(|row| base_offset.checked_add(row)) + .and_then(|o| o.checked_add((x as usize) * 4)); + let offset = match offset { + Some(o) => o, + None => continue, + }; + if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { + let alpha = if opaque { 255 } else { raw_bytes[offset + 3] }; + img_buf.put_pixel( + x, + y, + Rgba([raw_bytes[offset + 2], raw_bytes[offset + 1], raw_bytes[offset], alpha]), + ); + } + } + } + crate::unpremultiply_rgba(&mut img_buf); + let mut bytes = Vec::new(); + if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { + bytes + } else { + Vec::new() + } +} + +/// Convert a dmabuf sprite's RGBA readback (stride recovered from the mapping length) to a +/// straight-alpha PNG. +fn encode_gles_cursor(width: i32, height: i32, raw_bytes: &[u8]) -> Vec { + if width <= 0 || height <= 0 || width > 128 || height > 128 || raw_bytes.is_empty() { + return Vec::new(); + } + let stride = super::frontend::rgba_readback_stride( + raw_bytes.len(), + height as usize, + width as usize, + ); + let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); + for y in 0..(height as u32) { + for x in 0..(width as u32) { + let offset = (y as usize * stride) + (x as usize * 4); + if offset + 4 <= raw_bytes.len() { + img_buf.put_pixel( + x, + y, + Rgba([ + raw_bytes[offset], + raw_bytes[offset + 1], + raw_bytes[offset + 2], + raw_bytes[offset + 3], + ]), + ); + } + } + } + crate::unpremultiply_rgba(&mut img_buf); + let mut bytes = Vec::new(); + if img_buf.write_to(&mut IoCursor::new(&mut bytes), image::ImageFormat::Png).is_ok() { + bytes + } else { + Vec::new() + } +} + /// The loaded XCursor theme, held for the whole capture so cursor lookups stay cheap. /// /// The default cursor's frames are parsed once and kept here because they are consulted on nearly diff --git a/pixelflux/src/wayland/dcclient.rs b/pixelflux/src/wayland/dcclient.rs new file mode 100644 index 0000000..0756662 --- /dev/null +++ b/pixelflux/src/wayland/dcclient.rs @@ -0,0 +1,372 @@ +//! `zwlr_data_control_v1` CLIENT: the native clipboard bridge to a NESTED app +//! compositor (labwc/kwin running under pixelflux). +//! +//! Apps in a nested session use the inner compositor's selection, which +//! pixelflux's own clipboard machinery never sees. selkies bridges it over the +//! ScreenCapture ABI backed by this module instead of forking wl-copy/wl-paste +//! per operation: one-shot [`list_types`]/[`read`], a persistent-source [`write`] +//! (a detached thread serves paste requests until another client takes the +//! selection), and [`watch`] (a thread reporting selection changes to a Python +//! callback). Every compositor round-trip is deadline-bounded via +//! [`wlclient::bounded_roundtrip`]. + +use std::collections::HashMap; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use pyo3::{Py, PyAny, Python}; +use wayland_client::backend::ObjectId; +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, Proxy, QueueHandle}; +use wayland_protocols_wlr::data_control::v1::client::{ + zwlr_data_control_device_v1::{self, ZwlrDataControlDeviceV1}, + zwlr_data_control_manager_v1::ZwlrDataControlManagerV1, + zwlr_data_control_offer_v1::{self, ZwlrDataControlOfferV1}, + zwlr_data_control_source_v1::{self, ZwlrDataControlSourceV1}, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, impl_sync_callback, pipe_cloexec, read_fd_to_end, wait_readable, + write_fd_all, SyncState, IO_TIMEOUT, +}; + +/// How often a background thread wakes from its socket poll to check its stop +/// flag, bounding unwatch/shutdown latency. +const STOP_POLL: Duration = Duration::from_millis(500); + +#[derive(Default)] +struct DcState { + seat: Option, + manager: Option, + /// Advertised mimes per live offer. + offer_mimes: HashMap>, + selection: Option, + /// Set on every `selection` event (the watch loop's change edge). + selection_changed: bool, + /// Compositor told this device it is done (seat gone). + finished: bool, + /// The write path's source lost the selection to another client. + cancelled: bool, + /// Mime -> bytes served by the write path's source. + serve: Vec<(String, Vec)>, + sync_done: bool, +} + +impl SyncState for DcState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(DcState); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only an argument, and v1 + // data-control carries the regular selection this bridge needs. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwlr_data_control_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(DcState: ignore wl_seat::WlSeat); +delegate_noop!(DcState: ZwlrDataControlManagerV1); + +impl Dispatch for DcState { + fn event( + state: &mut Self, + offer: &ZwlrDataControlOfferV1, + event: zwlr_data_control_offer_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + if let zwlr_data_control_offer_v1::Event::Offer { mime_type } = event { + state.offer_mimes.entry(offer.id()).or_default().push(mime_type); + } + } +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlDeviceV1, + event: zwlr_data_control_device_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_device_v1::Event::DataOffer { id } => { + state.offer_mimes.entry(id.id()).or_default(); + } + zwlr_data_control_device_v1::Event::Selection { id } => { + // Replaced offers are dead objects; drop their proxy and mimes so + // a long-lived watch connection doesn't accumulate them. + if let Some(old) = state.selection.take() { + if id.as_ref().map(|o| o.id()) != Some(old.id()) { + state.offer_mimes.remove(&old.id()); + old.destroy(); + } + } + state.selection = id; + state.selection_changed = true; + } + zwlr_data_control_device_v1::Event::Finished => { + state.finished = true; + } + _ => {} + } + } + + wayland_client::event_created_child!(DcState, ZwlrDataControlDeviceV1, [ + zwlr_data_control_device_v1::EVT_DATA_OFFER_OPCODE => (ZwlrDataControlOfferV1, ()), + ]); +} + +impl Dispatch for DcState { + fn event( + state: &mut Self, + _: &ZwlrDataControlSourceV1, + event: zwlr_data_control_source_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_data_control_source_v1::Event::Send { mime_type, fd } => { + if let Some((_, data)) = state.serve.iter().find(|(m, _)| *m == mime_type) { + let _ = write_fd_all(&fd, data, IO_TIMEOUT); + } + // fd drops here, closing the pipe so the paster sees EOF. + } + zwlr_data_control_source_v1::Event::Cancelled => { + state.cancelled = true; + } + _ => {} + } + } +} + +/// Connect to `socket_path` and return (connection, queue, state, device) with +/// the current selection already delivered. +fn open_device( + socket_path: &str, +) -> Result< + (Connection, wayland_client::EventQueue, DcState, ZwlrDataControlDeviceV1), + String, +> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + Ok((conn, queue, state, device)) +} + +/// Mimes offered by the current selection (empty when nothing is copied). +pub(crate) fn list_types(socket_path: &str) -> Result, String> { + let (_conn, _queue, state, device) = open_device(socket_path)?; + let out = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + device.destroy(); + Ok(out) +} + +/// The current selection's payload for `mime`, or None when there is no +/// selection or it does not offer that mime. +pub(crate) fn read(socket_path: &str, mime: &str) -> Result>, String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + let Some(offer) = state.selection.clone() else { + device.destroy(); + return Ok(None); + }; + let offered = state.offer_mimes.get(&offer.id()).map_or(false, |m| m.iter().any(|x| x == mime)); + if !offered { + device.destroy(); + return Ok(None); + } + let (rd, wr) = pipe_cloexec()?; + offer.receive(mime.to_string(), wr.as_fd()); + queue.flush().map_err(|e| format!("flush: {e}"))?; + drop(wr); + // The source app writes into the pipe as it pleases; dispatch is not needed + // for the bytes, only the fd read. + let data = read_fd_to_end(&rd, IO_TIMEOUT)?; + let _ = bounded_roundtrip(&conn, &mut queue, &mut state); + device.destroy(); + Ok(Some(data)) +} + +/// Take the selection, serving `entries` (mime, bytes) to every paster from a +/// detached thread until another client takes the selection (or the compositor +/// goes away). Replacing a previous write is implicit: the compositor cancels +/// the old source when the new one takes the selection. +pub(crate) fn write(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-selection".into()) + .spawn(move || { + if let Err(e) = serve_selection(&path, entries) { + eprintln!("[Clipboard] app-compositor selection serve ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +fn serve_selection(socket_path: &str, entries: Vec<(String, Vec)>) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = DcState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.clone().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .clone() + .ok_or("app compositor does not advertise zwlr_data_control_manager_v1")?; + let device = manager.get_data_device(&seat, &qh, ()); + let source = manager.create_data_source(&qh, ()); + for (mime, _) in &entries { + source.offer(mime.clone()); + } + state.serve = entries; + device.set_selection(Some(&source)); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + while !state.cancelled && !state.finished { + // Sends are served inside dispatch; block until the compositor has + // something (with a poll so a dead compositor can't pin the thread). + queue.flush().map_err(|e| format!("flush: {e}"))?; + match conn.prepare_read() { + Some(guard) => { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + None => {} + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + source.destroy(); + device.destroy(); + let _ = queue.flush(); + Ok(()) +} + +/// Drop the selection (the compositor also cancels whatever source held it). +pub(crate) fn clear(socket_path: &str) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + device.set_selection(None); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + device.destroy(); + Ok(()) +} + +struct WatchHandle { + stop: Arc, +} + +static WATCHERS: Mutex>> = Mutex::new(None); + +/// Report every selection change on `socket_path` (including the one current at +/// start) to `callback(mimes: list[str])` from a background thread. A second +/// watch on the same socket replaces the first. +pub(crate) fn watch(socket_path: &str, callback: Py) -> Result<(), String> { + let stop = Arc::new(AtomicBool::new(false)); + { + let mut reg = WATCHERS.lock().unwrap(); + let map = reg.get_or_insert_with(HashMap::new); + if let Some(old) = map.insert(socket_path.to_string(), WatchHandle { stop: stop.clone() }) + { + old.stop.store(true, Ordering::Relaxed); + } + } + let path = socket_path.to_string(); + std::thread::Builder::new() + .name("pf-dc-watch".into()) + .spawn(move || { + if let Err(e) = watch_loop(&path, callback, &stop) { + eprintln!("[Clipboard] app-compositor watch ended: {e}"); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + Ok(()) +} + +/// Stop the watch on `socket_path` (no-op when none is running). +pub(crate) fn unwatch(socket_path: &str) { + let mut reg = WATCHERS.lock().unwrap(); + if let Some(map) = reg.as_mut() { + if let Some(handle) = map.remove(socket_path) { + handle.stop.store(true, Ordering::Relaxed); + } + } +} + +fn watch_loop(socket_path: &str, callback: Py, stop: &AtomicBool) -> Result<(), String> { + let (conn, mut queue, mut state, device) = open_device(socket_path)?; + while !stop.load(Ordering::Relaxed) && !state.finished { + if state.selection_changed { + state.selection_changed = false; + let mimes = state + .selection + .as_ref() + .and_then(|o| state.offer_mimes.get(&o.id()).cloned()) + .unwrap_or_default(); + if !mimes.is_empty() { + Python::attach(|py| { + if let Err(e) = callback.call1(py, (mimes,)) { + e.print(py); + } + }); + } + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + if let Some(guard) = conn.prepare_read() { + use std::os::fd::AsRawFd; + if wait_readable(guard.connection_fd().as_raw_fd(), STOP_POLL)? { + guard.read().map_err(|e| format!("read: {e}"))?; + } + } + queue.dispatch_pending(&mut state).map_err(|e| format!("dispatch: {e}"))?; + } + device.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/frontend.rs b/pixelflux/src/wayland/frontend.rs index fd93224..0e7a447 100644 --- a/pixelflux/src/wayland/frontend.rs +++ b/pixelflux/src/wayland/frontend.rs @@ -20,11 +20,9 @@ use std::borrow::Cow; use std::fs::File; -use std::io::Cursor as IoCursor; use std::time::Instant; use gbm::{BufferObject, Device as RawGbmDevice}; -use image::{ImageBuffer, ImageFormat, Rgba}; use pyo3::prelude::*; use pyo3::types::PyBytes; use std::sync::Mutex; @@ -33,12 +31,19 @@ use std::hash::{Hash, Hasher}; use smithay::backend::renderer::utils::RendererSurfaceState; use smithay::backend::allocator::dmabuf::Dmabuf; use smithay::backend::allocator::{ Buffer, Fourcc}; +use smithay::backend::renderer::damage::OutputDamageTracker; use smithay::backend::renderer::{ Bind, ExportMem, gles::GlesRenderer, pixman::PixmanRenderer, ImportDma, }; use smithay::input::dnd::{DndFocus, Source}; use std::sync::Arc; -use crate::wayland::cursor::Cursor; +use crate::wayland::cursor::{Cursor, CursorJob}; +use crate::wayland::keymap::KeymapPolicy; +use smithay::reexports::wayland_protocols_misc::zwp_virtual_keyboard_v1::server::{ + zwp_virtual_keyboard_manager_v1::{self, ZwpVirtualKeyboardManagerV1}, + zwp_virtual_keyboard_v1::{self, ZwpVirtualKeyboardV1}, +}; +use smithay::reexports::wayland_server::{DataInit, Dispatch, GlobalDispatch, New}; use smithay::wayland::viewporter::ViewporterState; use smithay::delegate_viewporter; use smithay::wayland::pointer_warp::{PointerWarpHandler, PointerWarpManager}; @@ -78,8 +83,8 @@ use smithay::delegate_primary_selection; use smithay::{ delegate_compositor, delegate_data_device, delegate_dmabuf, delegate_fractional_scale, - delegate_output, delegate_seat, delegate_shm, delegate_virtual_keyboard_manager, - delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, + delegate_output, delegate_seat, delegate_shm, + delegate_xdg_shell, delegate_relative_pointer, delegate_pointer_warp, delegate_pointer_constraints, desktop::{Space, Window}, input::{ @@ -97,7 +102,7 @@ use smithay::{ reexports::{ wayland_protocols::xdg::shell::server::xdg_toplevel::State as XdgState, wayland_server::{ - backend::{ClientData, ClientId, DisconnectReason, ObjectId}, + backend::{ClientData, ClientId, DisconnectReason, GlobalId, ObjectId}, protocol::{wl_buffer::WlBuffer, wl_surface::WlSurface}, Client, DisplayHandle, Resource, }, @@ -115,8 +120,8 @@ use smithay::{ seat::WaylandFocus, selection::{ data_device::{ - request_data_device_client_selection, DataDeviceHandler, DataDeviceState, - WaylandDndGrabHandler, + request_data_device_client_selection, set_data_device_focus, DataDeviceHandler, + DataDeviceState, WaylandDndGrabHandler, }, SelectionHandler, SelectionSource, SelectionTarget, }, @@ -125,7 +130,6 @@ use smithay::{ XdgToplevelSurfaceData, }, shm::{with_buffer_contents, ShmHandler, ShmState, BufferAccessError}, - virtual_keyboard::VirtualKeyboardManagerState, }, }; @@ -193,14 +197,126 @@ pub enum GpuEncoder { Nvenc(NvencEncoder), } +/// One capture pipeline bound to one output (display id): its settings, encoder set, +/// frame pools, delivery thread, and per-stream bookkeeping. Exactly one capture may run per +/// output; all fields mirror the pipeline strategy documented on [`AppState`], instantiated +/// per display. +pub struct WlCapture { + pub settings: RustCaptureSettings, + /// Zero-copy GPU session (GLES render + same-GPU dmabuf encode), calloop-affine; + /// `None` whenever a readback path is active for this display. + pub video_encoder: Option, + pub vaapi_state: StripeState, + pub recording_sink: Option>, + pub deliver_tx: Option>>, + pub deliver_join: Option>, + pub pending_hw_delivery: Option>, + pub pending_hw_damage: bool, + pub encode_pool: Option>, + pub encode_join: Option>>, + pub encode_controls: Arc, + pub encode_stats: Arc, + pub pool_last_render: Vec, + pub render_seq: u64, + pub pool_content_gen: Vec, + pub content_gen: u64, + pub frame_counter: u16, + pub pending_force_idr: bool, + pub needs_full_render: bool, + /// Last tick this capture actually rendered; paces per-display fps under the one + /// shared render timer (which fires at the fastest active capture's rate). + pub last_tick: Option, +} + +impl WlCapture { + /// Arm a keyframe on whichever path is live, plus a full render so a static + /// screen still produces the frame. Exactly ONE flag is set: the pool encode + /// loop consumes the atomic, every other path consumes `pending_force_idr` — + /// setting both would leave one armed forever (the host-mode idle gate polls + /// them every tick, so a stuck flag disables idle skipping for the session). + pub fn request_idr(&mut self) { + if self.encode_pool.is_some() { + self.encode_controls.force_idr.store(true, Ordering::Relaxed); + } else { + self.pending_force_idr = true; + } + self.needs_full_render = true; + } +} + +/// One virtual output and everything sized to it: the Smithay `Output` + its advertised +/// global, its layout position, the damage tracker, render targets, and the (at most one) +/// capture bound to it. `id` is the Python-facing display key; id 0 is the primary +/// HEADLESS-1 output, which is never destroyed. +pub struct OutputNode { + pub id: u32, + pub output: Output, + pub global: GlobalId, + /// Layout offset: the output's logical position in the Space AND its physical offset + /// for absolute input injection. The two coincide at scale 1; with mixed scales the + /// caller must place outputs so neither the logical nor the physical rectangles + /// overlap. + pub pos: (i32, i32), + pub damage_tracker: OutputDamageTracker, + /// Host-side scratch target: pixman throttle path renders here, GLES screenshots read + /// back here. + pub frame_buffer: Vec, + /// GPU render target for this output (GLES mode): the GBM BO and its dmabuf export. + pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, + /// Watermark overlay for THIS output: loaded at the output's scale, positioned (and, + /// for the bouncing anchor, animated) against the output's own frame dimensions. + pub overlay_state: OverlayState, + pub capture: Option, +} + +impl OutputNode { + /// The output's logical geometry in layout coordinates: origin plus mode/scale-derived + /// logical size. + pub fn logical_geometry(&self) -> Option> { + let mode = self.output.current_mode()?; + let scale = self.output.current_scale().fractional_scale(); + Some(Rectangle::new( + Point::from(self.pos), + ( + (mode.size.w as f64 / scale).round() as i32, + (mode.size.h as f64 / scale).round() as i32, + ) + .into(), + )) + } +} + +static NEXT_WINDOW_ID: AtomicU32 = AtomicU32::new(1); + +/// Per-window bookkeeping carried in the window's user-data map: a stable numeric id the +/// Python side addresses the window by, and the display id of the output it is placed on. +pub struct WindowMeta { + pub id: u32, + pub output: AtomicU32, +} + +/// The window's meta, inserted at `new_toplevel`; windows created before that (none in +/// practice) read as id 0 / primary output. +pub fn window_meta(window: &Window) -> Option<&WindowMeta> { + window.user_data().get::() +} + +/// The display id of the output this window is placed on (primary when untagged). +pub fn window_output_id(window: &Window) -> u32 { + window_meta(window).map(|m| m.output.load(Ordering::Relaxed)).unwrap_or(0) +} + /// Central context threaded through every Smithay handler; owns the Wayland globals, the /// GBM/EGL (or pixman) renderer state, and the capture/encode pipeline state. /// /// A single `AppState` lives for the whole compositor thread and is mutated in place by the -/// calloop event sources (client dispatch, input injection, the capture timer). The non-obvious -/// fields encode the pipeline's threading and buffer strategy: +/// calloop event sources (client dispatch, input injection, the capture timer). The frame +/// pipeline is instantiated PER DISPLAY: each `OutputNode` in `output_nodes` owns its output, +/// damage tracker, and render targets, and its optional `WlCapture` owns that display's +/// encoder set and delivery. The non-obvious `WlCapture`/`OutputNode` fields encode the +/// pipeline's threading and buffer strategy: /// -/// 1. **Render / encode targets**: +/// 1. **Render / encode targets** (per display): /// - **`video_encoder`**: the zero-copy GPU session only (GLES render + same-GPU dmabuf /// encode). Its EGL/dmabuf handles are calloop-affine, so this encoder runs inline on the /// calloop thread; it is `None` whenever a readback path is active. @@ -255,7 +371,9 @@ pub struct AppState { pub dh: DisplayHandle, #[allow(dead_code)] pub seat: Seat, - pub outputs: Vec, + /// Every live output with its per-display render/capture state; index 0 is the + /// primary (display id 0), which is never destroyed. + pub output_nodes: Vec, pub pending_windows: Vec, pub foreign_toplevel_list: ForeignToplevelListState, @@ -263,69 +381,57 @@ pub struct AppState { pub xdg_activation_state: XdgActivationState, pub primary_selection_state: PrimarySelectionState, pub popups: PopupManager, - pub frame_buffer: Vec, pub gles_renderer: Option, pub pixman_renderer: Option, pub gbm_device: Option>, - pub offscreen_buffer: Option<(BufferObject<()>, Dmabuf)>, - pub is_capturing: bool, + /// Mirror of the PRIMARY display's capture settings (geometry fallbacks, computer-use + /// info); per-display settings live on each capture. pub settings: RustCaptureSettings, - pub callback: Option>, - pub cursor_callback: Option>, + /// A cursor callback is registered on the `wl-cursor` worker (which owns the actual + /// `Py` object); tracked here so sprite resolution is skipped while nobody listens. + pub cursor_callback_set: bool, + /// Cursor delivery jobs to the `wl-cursor` worker (PNG encode + Python call off-thread). + pub cursor_tx: std::sync::mpsc::Sender, pub clipboard_callback: Option>, pub pending_clipboard_read: Option, + /// Preferred mime of the current CLIENT-owned clipboard selection, recorded even while + /// no callback is registered so `SetClipboardCallback` can re-stage a read of a copy made + /// in the gap; `None` when the selection is cleared or compositor-owned. + pub current_selection_mime: Option, pub last_log_time: Instant, pub start_time: Instant, pub clock: Clock, - pub frame_counter: u16, - pub pending_force_idr: bool, - pub needs_full_render: bool, pub use_gpu: bool, - pub video_encoder: Option, - pub vaapi_state: StripeState, pub cursor_helper: Cursor, - pub overlay_state: OverlayState, - - pub virtual_keyboard_state: VirtualKeyboardManagerState, + /// Seat keymap owner: base layout plus batched overlay binds. Every seat keymap swap + /// flows through this policy, so keymap identity has exactly one writer. + pub keymap_policy: KeymapPolicy, + /// Host-capture session when pixelflux captures an EXTERNAL compositor: + /// frames arrive by screencopy and input routes to its virtual devices. + pub host: Option, pub current_cursor_icon: Option, pub cursor_buffer: Option, - pub cursor_cache: std::collections::HashMap>, pub render_cursor_on_framebuffer: bool, pub pointer_warp_state: PointerWarpManager, pub relative_pointer_state: RelativePointerManagerState, pub pointer_constraints_state: PointerConstraintsState, pub render_node_path: String, pub auto_gpu_selected: bool, - pub recording_sink: Option>, - pub deliver_tx: Option>>, - pub deliver_join: Option>, - /// Zero-copy frame parked after a full delivery channel. The hardware encode and - /// its delivery both run on the calloop thread, and a blocking send there would - /// freeze input/command/Wayland dispatch for as long as the Python consumer - /// stalls. An encoded frame is part of the H.264 reference chain and can never - /// be dropped, so it parks here instead, and new encodes pause until it leaves. - pub pending_hw_delivery: Option>, - /// Damage seen on ticks skipped while `pending_hw_delivery` was parked, folded - /// into the next encode's change detection so a change that happened during the - /// pause is never lost (each tick's damage list is otherwise discarded). - pub pending_hw_damage: bool, - pub encode_pool: Option>, - pub encode_join: Option>>, - pub encode_controls: Arc, - pub encode_stats: Arc, - pub pool_last_render: Vec, - pub render_seq: u64, - pub pool_content_gen: Vec, - pub content_gen: u64, - pub pending_screenshot: Option>>, + /// Computer-use screenshot request `(display id, reply)`; served from that output's + /// next render (the id was validated live when the request was queued). + pub pending_screenshot: Option<(u32, std::sync::mpsc::Sender, String>>)>, + /// The command channel, drained in place (wakeups arrive on a separate ping channel) so + /// the render tick can apply every queued command BEFORE starting a long render/encode — + /// queued input is never starved behind the tick it arrived during. + pub command_rx: Option>, } /// Pointer-constraints protocol wiring. The headless capture path never enforces a lock or @@ -440,9 +546,9 @@ impl WlrLayerShellHandler for AppState { namespace: String, ) { let smithay_output = if let Some(wlo) = output.as_ref() { - self.outputs.iter().find(|o| o.owns(wlo)) + self.output_nodes.iter().map(|n| &n.output).find(|o| o.owns(wlo)) } else { - self.outputs.first() + self.primary_output() }; if let Some(output) = smithay_output { @@ -505,17 +611,12 @@ impl CompositorHandler for AppState { fn commit(&mut self, surface: &WlSurface) { smithay::backend::renderer::utils::on_commit_buffer_handler::(surface); - if let Some(output) = self.outputs.first() { - let mut layer_map = layer_map_for_output(output); - let mut found = false; - for layer in layer_map.layers() { - if layer.wl_surface() == surface { - found = true; - break; - } - } + for node in &self.output_nodes { + let mut layer_map = layer_map_for_output(&node.output); + let found = layer_map.layers().any(|layer| layer.wl_surface() == surface); if found { layer_map.arrange(); + break; } } @@ -541,12 +642,26 @@ impl CompositorHandler for AppState { } } - if let Some(window) = self + let mapped = self .space .elements() .find(|w| w.toplevel().map(|tl| tl.wl_surface() == surface).unwrap_or(false)) - { + .cloned(); + if let Some(window) = mapped { window.on_commit(); + // A null-buffer commit unmaps the toplevel (xdg-shell): purge it from the + // space so it no longer lists or renders, and re-queue it so a client that + // maps again goes back through the configure handshake. + let has_buffer = smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_some(), + ) + .unwrap_or(false); + if !has_buffer { + self.space.unmap_elem(&window); + self.pending_windows.push(window); + return; + } } if let Some(idx) = self.pending_windows.iter().position(|w| { @@ -566,20 +681,51 @@ impl CompositorHandler for AppState { }); if !initial_configure_sent { - let (logical_width, logical_height) = if let Some(output) = self.outputs.first() { - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ) - } else { - let scale = self.settings.scale.max(0.1); - ( - (self.settings.width as f64 / scale).round() as i32, - (self.settings.height as f64 / scale).round() as i32, - ) - }; + // A new toplevel opens fullscreened on the output the pointer is on + // (primary when indeterminate); the choice is pinned on the window's + // meta so the acked commit maps to the same output. + let mut target_id = self.pointer_display(); + // A second fullscreen surface from a client that already owns one on + // the target output is screen-like (a nested compositor opens one + // host toplevel per screen): place it on an empty output when one + // exists instead of stacking it. + if let Some(client) = toplevel.wl_surface().client() { + let same_client = |w: &Window| { + w.wl_surface() + .and_then(|s| s.client()) + .map_or(false, |c| c.id() == client.id()) + }; + let crowded = self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == target_id && same_client(w)); + if crowded { + let empty = self.output_nodes.iter().map(|n| n.id).find(|oid| { + !self + .space + .elements() + .chain(self.pending_windows.iter()) + .any(|w| window_output_id(w) == *oid) + }); + if let Some(oid) = empty { + target_id = oid; + } + } + } + if let Some(meta) = window_meta(&window) { + meta.output.store(target_id, Ordering::Relaxed); + } + let (logical_width, logical_height) = + if let Some(size) = self.logical_size_of(target_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; toplevel.with_pending_state(|state| { state.states.set(XdgState::Activated); @@ -589,28 +735,52 @@ impl CompositorHandler for AppState { toplevel.send_configure(); self.pending_windows.push(window); + } else if smithay::backend::renderer::utils::with_renderer_surface_state( + surface, + |s| s.buffer().is_none(), + ) + .unwrap_or(true) + { + // Configured but still buffer-less (a decoration-triggered configure, an + // ack-only commit, or a remap after a null-buffer unmap — xdg + // initial_configure_sent stays true there): keep waiting; mapping now would + // list and hit-test a phantom window. Answer with the forced-fullscreen + // configure so the client draws its first buffer at the right geometry. + let tl = toplevel.clone(); + self.pending_windows.push(window); + self.send_forced_fullscreen_configure(&tl); } else { - self.space.map_element(window.clone(), (0, 0), true); + let target_id = window_output_id(&window); + let node_idx = self.node_idx_for_id(target_id).unwrap_or(0); + let (target_output, pos) = { + let node = &self.output_nodes[node_idx]; + (node.output.clone(), node.pos) + }; + self.space.map_element(window.clone(), pos, true); window.on_commit(); - if let Some(output) = self.outputs.first() { - output.enter(surface); - - let mode = output.current_mode().unwrap(); - let scale = output.current_scale().fractional_scale(); - let (expected_w, expected_h) = ( - (mode.size.w as f64 / scale).round() as i32, - (mode.size.h as f64 / scale).round() as i32, - ); - - let geo = window.geometry(); - if (geo.size.w - expected_w).abs() > 1 || (geo.size.h - expected_h).abs() > 1 { - toplevel.with_pending_state(|state| { - state.states.set(XdgState::Activated); - state.states.set(XdgState::Fullscreen); - state.size = Some((expected_w, expected_h).into()); + { + target_output.enter(surface); + let scale = target_output.current_scale().fractional_scale(); + with_states(surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); }); - toplevel.send_configure(); + }); + + if let Some(geo_out) = self.output_nodes[node_idx].logical_geometry() { + let (expected_w, expected_h) = (geo_out.size.w, geo_out.size.h); + let geo = window.geometry(); + if (geo.size.w - expected_w).abs() > 1 + || (geo.size.h - expected_h).abs() > 1 + { + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Activated); + state.states.set(XdgState::Fullscreen); + state.size = Some((expected_w, expected_h).into()); + }); + toplevel.send_configure(); + } } } @@ -626,18 +796,145 @@ impl CompositorHandler for AppState { impl AppState { + /// The primary (display id 0) output. + pub(crate) fn primary_output(&self) -> Option<&Output> { + self.output_nodes.first().map(|n| &n.output) + } + + pub(crate) fn node_idx_for_id(&self, id: u32) -> Option { + self.output_nodes.iter().position(|n| n.id == id) + } + + /// Index of the node whose LOGICAL rect contains `p`. + pub(crate) fn node_idx_under(&self, p: Point) -> Option { + self.output_nodes.iter().position(|n| { + n.logical_geometry() + .map(|g| g.to_f64().contains(p)) + .unwrap_or(false) + }) + } + + /// Map absolute PHYSICAL union-layout coordinates to a logical layout point: each + /// output occupies the physical rectangle at its layout offset sized by its mode; the + /// point is clamped into the nearest output when it falls outside all of them, so the + /// pointer can never leave the layout. + pub(crate) fn layout_physical_to_logical(&self, x: f64, y: f64) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(mode) = node.output.current_mode() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let (px, py) = (node.pos.0 as f64, node.pos.1 as f64); + let cx = x.max(px).min(px + mode.size.w as f64 - 1.0); + let cy = y.max(py).min(py + mode.size.h as f64 - 1.0); + let d2 = (x - cx).powi(2) + (y - cy).powi(2); + let logical = Point::from(( + node.pos.0 as f64 + (cx - px) / scale, + node.pos.1 as f64 + (cy - py) / scale, + )); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, logical)); + } + } + best.map(|(_, p)| p).unwrap_or_else(|| (0.0, 0.0).into()) + } + + /// Clamp a logical layout point into the nearest output's logical rectangle. + pub(crate) fn clamp_logical(&self, p: Point) -> Point { + let mut best: Option<(f64, Point)> = None; + for node in &self.output_nodes { + let Some(geo) = node.logical_geometry() else { continue }; + let scale = node.output.current_scale().fractional_scale(); + let g = geo.to_f64(); + let margin = 1.0 / scale.max(0.1); + let cx = p.x.max(g.loc.x).min(g.loc.x + g.size.w - margin); + let cy = p.y.max(g.loc.y).min(g.loc.y + g.size.h - margin); + let d2 = (p.x - cx).powi(2) + (p.y - cy).powi(2); + if best.as_ref().map(|(bd, _)| d2 < *bd).unwrap_or(true) { + best = Some((d2, (cx, cy).into())); + } + } + best.map(|(_, p)| p).unwrap_or(p) + } + + /// Logical layout point -> physical union-layout coordinates (inverse of + /// `layout_physical_to_logical` for in-bounds points; primary-relative otherwise). + pub(crate) fn layout_logical_to_physical(&self, p: Point) -> (f64, f64) { + let idx = self.node_idx_under(p).unwrap_or(0); + let Some(node) = self.output_nodes.get(idx) else { return (p.x, p.y) }; + let scale = node.output.current_scale().fractional_scale(); + ( + node.pos.0 as f64 + (p.x - node.pos.0 as f64) * scale, + node.pos.1 as f64 + (p.y - node.pos.1 as f64) * scale, + ) + } + + /// Logical size of the given display's output. + pub(crate) fn logical_size_of(&self, id: u32) -> Option<(i32, i32)> { + let idx = self.node_idx_for_id(id)?; + let geo = self.output_nodes[idx].logical_geometry()?; + Some((geo.size.w, geo.size.h)) + } + + /// The display id under the pointer (primary when indeterminate). + pub(crate) fn pointer_display(&self) -> u32 { + self.seat + .get_pointer() + .map(|p| p.current_location()) + .and_then(|pos| self.node_idx_under(pos)) + .map(|idx| self.output_nodes[idx].id) + .unwrap_or(0) + } + + /// Place `window` on output `id`: retag its meta, remap it at the output's layout + /// origin, move output enter/leave, push the output's fractional scale, and send the + /// forced-fullscreen configure at that output's logical size. + pub(crate) fn place_window_on_output(&mut self, window: &Window, id: u32) -> bool { + let Some(idx) = self.node_idx_for_id(id) else { return false }; + let old_id = window_output_id(window); + let (new_output, pos) = { + let node = &self.output_nodes[idx]; + (node.output.clone(), node.pos) + }; + let old_output = self + .node_idx_for_id(old_id) + .map(|i| self.output_nodes[i].output.clone()); + if let Some(meta) = window_meta(window) { + meta.output.store(id, Ordering::Relaxed); + } + self.space.map_element(window.clone(), pos, true); + if let Some(surface) = window.wl_surface() { + if let Some(old) = old_output { + if old_id != id { + old.leave(&surface); + } + } + new_output.enter(&surface); + let scale = new_output.current_scale().fractional_scale(); + with_states(&surface, |states| { + smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { + fs.set_preferred_scale(scale); + }); + }); + } + if let Some(toplevel) = window.toplevel() { + let toplevel = toplevel.clone(); + self.send_forced_fullscreen_configure(&toplevel); + } + true + } + /// Drain a clipboard read staged by `new_selection` and hand `(mime, bytes)` to the /// Python callback off-thread. /// /// Runs from the loop *after* the dispatch that stored the new client source, so the request /// targets the current selection rather than the previous one. It clones the callback, opens a /// pipe, and asks the owning client source to write the chosen mime into the pipe's writer. A - /// spawned reader thread then reads the response — capped at 64 MiB so a hostile client cannot - /// balloon memory, and bounded by a 10 s poll deadline so a client that takes the selection but - /// never writes nor closes its fd cannot pin the thread forever (each clipboard change would - /// otherwise leak one zombie thread + pipe) — and, unless the interpreter is finalizing, - /// delivers the bytes to Python. The `PY_SHUTDOWN` checks keep this off a shutting-down - /// interpreter. + /// spawned reader thread then reads the response. The overall bound is by SIZE (64 MiB, then + /// delivered truncated) so a hostile client cannot balloon memory; time only bounds + /// INACTIVITY — a producer that keeps bytes flowing may take as long as it needs (a large + /// transfer from a slow source still delivers), while one that goes silent for 10 s without + /// closing its fd is dropped so each clipboard change cannot leak a pinned thread + pipe. + /// The `PY_SHUTDOWN` checks keep this off a shutting-down interpreter. pub(crate) fn process_pending_clipboard_read(&mut self) { let Some(mime) = self.pending_clipboard_read.take() else { return }; if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { @@ -660,12 +957,14 @@ impl AppState { use std::io::Read; use std::os::fd::AsRawFd; const CAP: usize = 64 * 1024 * 1024; - const DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); - let start = Instant::now(); + const IDLE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(10); + let mut last_data = Instant::now(); let mut buf = Vec::new(); let mut chunk = [0u8; 65536]; loop { - let Some(remaining) = DEADLINE.checked_sub(start.elapsed()) else { return }; + let Some(remaining) = IDLE_DEADLINE.checked_sub(last_data.elapsed()) else { + return; + }; let mut pfd = libc::pollfd { fd: reader.as_raw_fd(), events: libc::POLLIN, @@ -680,11 +979,12 @@ impl AppState { return; } if ready == 0 { - return; + continue; } match (&reader).read(&mut chunk) { Ok(0) => break, Ok(n) => { + last_data = Instant::now(); let room = CAP - buf.len(); let take_n = n.min(room); buf.extend_from_slice(&chunk[..take_n]); @@ -711,253 +1011,236 @@ impl AppState { } - /// Resolve a `CursorImageStatus` to PNG bytes plus hotspot and invoke the Python cursor - /// callback. Also re-invoked from the calloop command handlers to replay the retained cursor - /// when a callback re-registers or a capture restarts. + /// Resolve a `CursorImageStatus` into a job for the `wl-cursor` worker, which does the + /// PNG encode, caching, and the GIL-bound Python call off the calloop thread. Also re-invoked + /// from the calloop command handlers to replay the retained cursor when a callback + /// (re)registers or a capture restarts. /// - /// Dispatches on the cursor status: + /// Only the renderer/surface-affine work happens here: /// - /// 1. **Named**: look the themed cursor up by CSS name and emit its cached PNG (`"png"`), or - /// `"error"` if the theme lacks it. - /// 2. **Hidden**: emit `"hide"` — the only message allowed through with empty data, since an - /// intentional pointer hide must blank the consumer's cursor. - /// 3. **Surface** (a client-supplied cursor sprite): ignore any surface without the + /// 1. **Named** / **Hidden**: forwarded as-is (the worker owns its own theme handle). + /// 2. **Surface** (a client-supplied cursor sprite): ignore any surface without the /// `cursor_image` role, read the hotspot, then read the backing buffer by one of two paths: /// - **SHM**: hash only the sprite's sub-region — width/height/stride/offset/format plus the /// pixel span — because many sprites share one pool and differ only by `offset`, so hashing - /// the whole pool would collide. On a cache miss (and only when ≤128×128) convert the - /// BGRA/XRGB pixels to RGBA, un-premultiply (wl_shm cursor content is premultiplied by - /// convention; PNG carries straight alpha), and PNG-encode; `Xrgb8888` has no alpha so - /// byte 3 is forced opaque, and stride/offset are clamped non-negative with checked - /// arithmetic so a garbage descriptor skips pixels instead of panicking. + /// the whole pool would collide; ship the raw pool bytes plus descriptor to the worker. /// - **dmabuf** (`NotManaged`): bind it to the GLES renderer, copy the framebuffer to - /// `Abgr8888`, map it back, un-premultiply, and PNG-encode using the derived readback - /// stride. - /// The PNG cache is bounded at 100 entries by arbitrary eviction; content-hashing means an - /// evicted sprite simply re-inserts on the next render. - /// - /// A surface whose buffer could not be read yields `("surface", empty)`, which the final gate - /// suppresses — Python is called only for non-empty data or `"hide"` — so a transient read miss - /// (common during a stop/start replay) preserves the consumer's last cursor instead of blanking - /// it. `PY_SHUTDOWN` gates the whole call off a finalizing interpreter. + /// `Abgr8888`, map it back (calloop-affine), and ship the raw RGBA readback. + /// A sprite whose buffer could not be read is dropped by the worker, preserving the + /// consumer's last cursor instead of blanking it (only "hide" carries empty data). pub(crate) fn send_cursor_image(&mut self, image: &CursorImageStatus) { - if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { + if !self.cursor_callback_set { return; } - if let Some(ref cb) = self.cursor_callback { - let (msg_type, data, hot_x, hot_y) = match image { - CursorImageStatus::Named(icon) => { - self.cursor_buffer = None; - let name = cursor_icon_to_str(icon); - if let Some((png_bytes, x, y)) = self.cursor_helper.get_png_data(name) { - ("png", png_bytes, x as i32, y as i32) - } else { - ("error", Vec::new(), 0, 0) + let job = match image { + CursorImageStatus::Named(icon) => { + self.cursor_buffer = None; + CursorJob::Named { name: cursor_icon_to_str(icon) } + } + CursorImageStatus::Hidden => { + self.cursor_buffer = None; + CursorJob::Hide + } + CursorImageStatus::Surface(ref surface) => { + let mut hot_x = 0; + let mut hot_y = 0; + let mut is_cursor_role = false; + + with_states(surface, |states| { + if states.role == Some("cursor_image") { + is_cursor_role = true; } + if let Some(attributes) = states.data_map.get::>() { + if let Ok(guard) = attributes.lock() { + hot_x = guard.hotspot.x; + hot_y = guard.hotspot.y; + } + } + }); + + if !is_cursor_role { + return; } - CursorImageStatus::Hidden => { - self.cursor_buffer = None; - ("hide", Vec::new(), 0, 0) - }, - CursorImageStatus::Surface(ref surface) => { - let mut final_png = Vec::new(); - let mut hot_x = 0; - let mut hot_y = 0; - let mut is_cursor_role = false; - with_states(surface, |states| { - if states.role == Some("cursor_image") { - is_cursor_role = true; - } - if let Some(attributes) = states.data_map.get::>() { - if let Ok(guard) = attributes.lock() { - hot_x = guard.hotspot.x; - hot_y = guard.hotspot.y; - } - } - }); + let buffer_found = with_states(surface, |states| { + let mut attrs = states.cached_state.get::(); - if !is_cursor_role { - return; + if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { + return Some(b.clone()); } - let buffer_found = with_states(surface, |states| { - let mut attrs = states.cached_state.get::(); - - if let Some(BufferAssignment::NewBuffer(b)) = &attrs.current().buffer { - return Some(b.clone()); - } - - if let Some(mutex) = states.data_map.get::>() { - if let Ok(renderer_state) = mutex.try_lock() { - if let Some(b) = renderer_state.buffer() { - let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; - return Some(wl_buffer.clone()); - } + if let Some(mutex) = states.data_map.get::>() { + if let Ok(renderer_state) = mutex.try_lock() { + if let Some(b) = renderer_state.buffer() { + let wl_buffer: &wayland_server::protocol::wl_buffer::WlBuffer = b; + return Some(wl_buffer.clone()); } } - None - }); - - if let Some(buffer) = buffer_found { - let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { - let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; - let mut hasher = DefaultHasher::new(); - spec.width.hash(&mut hasher); - spec.height.hash(&mut hasher); - spec.stride.hash(&mut hasher); - spec.offset.hash(&mut hasher); - spec.format.hash(&mut hasher); - let start = (spec.offset.max(0) as usize).min(len); - let span = (spec.stride.max(0) as usize) - .saturating_mul(spec.height.max(0) as usize); - let end = start.saturating_add(span).min(len); - slice[start..end].hash(&mut hasher); - let hash = hasher.finish(); - (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) - }); + } + None + }); - match shm_result { - Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = stride.max(0) as usize; - let base_offset = buf_offset.max(0) as usize; - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize) - .checked_mul(stride_usize) - .and_then(|row| base_offset.checked_add(row)) - .and_then(|o| o.checked_add((x as usize) * 4)); - let offset = match offset { - Some(o) => o, - None => continue, - }; - if offset.checked_add(4).is_some_and(|end| end <= raw_bytes.len()) { - let alpha = if format == wl_shm::Format::Xrgb8888 { - 255 - } else { - raw_bytes[offset + 3] - }; - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset + 2], - raw_bytes[offset + 1], - raw_bytes[offset], - alpha - ])); - } - } - } + let Some(buffer) = buffer_found else { return }; + + let shm_result = with_buffer_contents(&buffer, |ptr, len, spec| { + let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; + let mut hasher = DefaultHasher::new(); + spec.width.hash(&mut hasher); + spec.height.hash(&mut hasher); + spec.stride.hash(&mut hasher); + spec.offset.hash(&mut hasher); + spec.format.hash(&mut hasher); + let start = (spec.offset.max(0) as usize).min(len); + let span = (spec.stride.max(0) as usize) + .saturating_mul(spec.height.max(0) as usize); + let end = start.saturating_add(span).min(len); + slice[start..end].hash(&mut hasher); + let hash = hasher.finish(); + (hash, spec.width, spec.height, spec.stride, spec.format, spec.offset, slice.to_vec()) + }); - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - }, - Err(BufferAccessError::NotManaged) => { - let mut gles_data: Option<(u64, i32, i32, Vec)> = None; - - let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); - - if let Some(mut dmabuf) = dmabuf_opt { - if let Some(renderer) = self.gles_renderer.as_mut() { - let width = dmabuf.width() as i32; - let height = dmabuf.height() as i32; - - match renderer.bind(&mut dmabuf) { - Ok(frame) => { - let rect = Rectangle::new((0, 0).into(), (width, height).into()); - - match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { - Ok(mapping) => { - match renderer.map_texture(&mapping) { - Ok(data) => { - let mut hasher = DefaultHasher::new(); - data.hash(&mut hasher); - let hash = hasher.finish(); - gles_data = Some((hash, width, height, data.to_vec())); - }, - Err(e) => eprintln!("Failed to map texture: {:?}", e) - } - }, - Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e) + let job = match shm_result { + Ok((hash, width, height, stride, format, buf_offset, raw_bytes)) => { + Some(CursorJob::Shm { + hash, + width, + height, + stride, + offset: buf_offset, + opaque: format == wl_shm::Format::Xrgb8888, + bytes: raw_bytes, + hot_x, + hot_y, + }) + } + Err(BufferAccessError::NotManaged) => { + let mut gles_job = None; + let dmabuf_opt = get_dmabuf(&buffer).ok().cloned(); + if let Some(mut dmabuf) = dmabuf_opt { + if let Some(renderer) = self.gles_renderer.as_mut() { + let width = dmabuf.width() as i32; + let height = dmabuf.height() as i32; + + match renderer.bind(&mut dmabuf) { + Ok(frame) => { + let rect = Rectangle::new((0, 0).into(), (width, height).into()); + match renderer.copy_framebuffer(&frame, rect, Fourcc::Abgr8888) { + Ok(mapping) => match renderer.map_texture(&mapping) { + Ok(data) => { + let mut hasher = DefaultHasher::new(); + data.hash(&mut hasher); + gles_job = Some(CursorJob::Gles { + hash: hasher.finish(), + width, + height, + bytes: data.to_vec(), + hot_x, + hot_y, + }); } + Err(e) => eprintln!("Failed to map texture: {:?}", e), }, - Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e) + Err(e) => eprintln!("Failed to copy framebuffer: {:?}", e), } } + Err(e) => eprintln!("Failed to bind dmabuf to renderer: {:?}", e), } - - if let Some((hash, width, height, raw_bytes)) = gles_data { - if let Some(cached_png) = self.cursor_cache.get(&hash) { - final_png = cached_png.clone(); - } else { - if width <= 128 && height <= 128 && !raw_bytes.is_empty() { - let mut img_buf = ImageBuffer::, Vec>::new(width as u32, height as u32); - let stride_usize = rgba_readback_stride(raw_bytes.len(), height as usize, width as usize); - - for y in 0..(height as u32) { - for x in 0..(width as u32) { - let offset = (y as usize * stride_usize) + (x as usize * 4); - if offset + 4 <= raw_bytes.len() { - img_buf.put_pixel(x, y, Rgba([ - raw_bytes[offset], - raw_bytes[offset + 1], - raw_bytes[offset + 2], - raw_bytes[offset + 3] - ])); - } - } - } - - crate::unpremultiply_rgba(&mut img_buf); - let mut bytes = Vec::new(); - if img_buf.write_to(&mut IoCursor::new(&mut bytes), ImageFormat::Png).is_ok() { - self.cursor_cache.insert(hash, bytes.clone()); - final_png = bytes; - if self.cursor_cache.len() > 100 { - let evict = *self.cursor_cache.keys().next().unwrap(); - self.cursor_cache.remove(&evict); - } - } - } - } - } - }, - Err(_) => {} + } } - - self.cursor_buffer = Some(buffer); + gles_job } + Err(_) => None, + }; - if !final_png.is_empty() { - ("png", final_png, hot_x, hot_y) - } else { - ("surface", Vec::new(), 0, 0) - } - } - }; + self.cursor_buffer = Some(buffer); + let Some(job) = job else { return }; + job + } + }; + let _ = self.cursor_tx.send(job); + } - if !data.is_empty() || msg_type == "hide" { - Python::attach(|py| { - let py_bytes = PyBytes::new(py, &data); - let _ = cb.call1(py, (msg_type, py_bytes, hot_x, hot_y)); - }); + /// Re-apply the policy's keymap (base + overlays) to the seat keyboard, broadcasting to + /// clients only when the content actually changed (smithay dedupes by content hash). + pub(crate) fn apply_keymap_policy(&mut self) { + let text = self.keymap_policy.keymap_text(); + if text.is_empty() { + return; + } + // Host-capture mode: the same managed keymap rides on the virtual + // keyboard, so the host compositor translates injected keycodes with + // selkies' keymap (overlay binds included) instead of its own. + if let Some(host) = &self.host { + host.set_keymap(&text); + } + if let Some(keyboard) = self.seat.get_keyboard() { + if let Err(e) = keyboard.set_keymap_from_string(self, text) { + eprintln!("[Wayland] keymap swap failed: {e:?}"); } } } + + /// Resolve `keysyms` to `(keycode, level)` pairs, overlay-binding whatever the base + /// cannot produce — at most ONE keymap swap for the whole batch, and never rebinding a + /// keycode that is currently held down. + pub(crate) fn bind_keysyms(&mut self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// `bind_keysyms` restricted to level-0 resolutions (see + /// [`KeymapPolicy::bind_many_plain`]); used by the virtual-keyboard translation path, which + /// cannot synthesize modifiers. + pub(crate) fn bind_keysyms_plain(&mut self, keysyms: &[u32]) -> Vec { + let pressed: std::collections::HashSet = self + .seat + .get_keyboard() + .map(|k| k.pressed_keys().iter().map(|c| c.raw()).collect()) + .unwrap_or_default(); + let (out, changed) = self.keymap_policy.bind_many_plain(keysyms, &pressed); + if changed { + self.apply_keymap_policy(); + } + out + } + + /// Answer any client fullscreen/maximize (un)set request with the compositor's forced + /// policy: every toplevel is Fullscreen+Activated at the CURRENT logical size of the + /// output the window is placed on. The configure is always sent, so an app toggling + /// fullscreen gets an explicit, current-geometry answer instead of silence or stale + /// pending state. + pub(crate) fn send_forced_fullscreen_configure(&mut self, toplevel: &ToplevelSurface) { + let output_id = self + .space + .elements() + .chain(self.pending_windows.iter()) + .find(|w| w.toplevel().map(|t| t == toplevel).unwrap_or(false)) + .map(window_output_id) + .unwrap_or(0); + let (logical_width, logical_height) = if let Some(size) = self.logical_size_of(output_id) { + size + } else { + let scale = self.settings.scale.max(0.1); + ( + (self.settings.width as f64 / scale).round() as i32, + (self.settings.height as f64 / scale).round() as i32, + ) + }; + toplevel.with_pending_state(|state| { + state.states.set(XdgState::Fullscreen); + state.states.set(XdgState::Activated); + state.size = Some((logical_width, logical_height).into()); + }); + toplevel.send_configure(); + } } /// Clipboard mime types the bridge can hand to Python, most specific first; `new_selection` @@ -997,21 +1280,25 @@ impl SelectionHandler for AppState { if ty != SelectionTarget::Clipboard { return; } + let Some(source) = source else { + self.current_selection_mime = None; + return; + }; + let mimes = source.mime_types(); + let mime = CLIPBOARD_MIME_PREFERENCE + .iter() + .find(|want| mimes.iter().any(|m| m == *want)) + .map(|s| s.to_string()); + // Recorded even with no callback armed, so SetClipboardCallback can re-stage a + // read of a copy made while nobody was listening. + self.current_selection_mime = mime.clone(); if crate::PY_SHUTDOWN.load(std::sync::atomic::Ordering::Relaxed) { return; } if self.clipboard_callback.is_none() { return; } - let Some(source) = source else { return }; - let mimes = source.mime_types(); - let Some(mime) = CLIPBOARD_MIME_PREFERENCE - .iter() - .find(|want| mimes.iter().any(|m| m == *want)) - .map(|s| s.to_string()) - else { - return; - }; + let Some(mime) = mime else { return }; self.pending_clipboard_read = Some(mime); let _ = seat; } @@ -1102,7 +1389,7 @@ impl FractionalScaleHandler for AppState { &mut self, surface: smithay::reexports::wayland_server::protocol::wl_surface::WlSurface, ) { - if let Some(output) = self.outputs.first() { + if let Some(output) = self.primary_output() { let scale = output.current_scale().fractional_scale(); with_states(&surface, |states| { smithay::wayland::fractional_scale::with_fractional_scale(states, |fs| { @@ -1613,17 +1900,17 @@ impl SeatHandler for AppState { self.send_cursor_image(&image); } - /// Keep the primary selection's focus following keyboard focus, so middle-click paste - /// targets the currently focused client (or clears it when nothing is focused). + /// Keep BOTH selections' focus following keyboard focus: without the data-device half, + /// the focused client never receives wl_data_offer events and Ctrl+V paste is a silent + /// no-op even while the compositor-side selection is correct (primary covers only + /// middle-click paste). fn focus_changed(&mut self, seat: &Seat, focus: Option<&Self::KeyboardFocus>) { - if let Some(focus_target) = focus { - let dh = &self.dh; - let client = focus_target.wl_surface().and_then(|s| dh.get_client(s.id()).ok()); - set_primary_focus(dh, seat, client); - } else { - let dh = &self.dh; - set_primary_focus(dh, seat, None); - } + let dh = &self.dh; + let client = focus + .and_then(|t| t.wl_surface()) + .and_then(|s| dh.get_client(s.id()).ok()); + set_data_device_focus(dh, seat, client.clone()); + set_primary_focus(dh, seat, client); } } @@ -1679,8 +1966,16 @@ impl XdgShellHandler for AppState { } /// A new toplevel appears: wrap it in a `Window`, queue it for mapping, and register a /// foreign-toplevel handle (seeded with title / app-id) stored on the surface for later updates. + /// The window is pinned to the pointer's output HERE — the first commit can't be relied on + /// for that, because a decoration-negotiating client (foot) has its initial configure sent + /// by `new_decoration` before it ever commits, skipping the pre-configure commit branch. fn new_toplevel(&mut self, surface: ToplevelSurface) { + let target_id = self.pointer_display(); let window = Window::new_wayland_window(surface.clone()); + window.user_data().insert_if_missing_threadsafe(|| WindowMeta { + id: NEXT_WINDOW_ID.fetch_add(1, Ordering::Relaxed), + output: AtomicU32::new(target_id), + }); self.pending_windows.push(window); let (title, app_id) = with_states(surface.wl_surface(), |states| { let attributes = states.data_map.get::().unwrap().lock().unwrap(); @@ -1729,19 +2024,233 @@ impl XdgShellHandler for AppState { } let _ = surface.send_repositioned(token); } + /// Client fullscreen request: always granted at the compositor's forced-fullscreen + /// geometry (the CURRENT logical size), so the toggle gets a definite answer. + fn fullscreen_request( + &mut self, + surface: ToplevelSurface, + _output: Option, + ) { + self.send_forced_fullscreen_configure(&surface); + } + /// Client unfullscreen request: the forced-fullscreen policy stands, but the client + /// still receives an explicit configure at the current geometry (the Smithay default sends + /// NOTHING here, leaving the app waiting on a toggle that never answers). + fn unfullscreen_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Maximize request: answered with the forced-fullscreen configure (same geometry). + fn maximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } + /// Unmaximize request: explicit current-geometry configure, policy unchanged. + fn unmaximize_request(&mut self, surface: ToplevelSurface) { + self.send_forced_fullscreen_configure(&surface); + } /// A toplevel closed: drop it from the pending-window queue so a window that never - /// finished mapping can't linger there, and remove its foreign-toplevel handle so taskbar-style - /// clients stop listing a window that is gone. + /// finished mapping can't linger there, unmap it from the space at once so `list_windows` + /// and hit-testing never see a husk (the periodic `space.refresh` would only reap it + /// later), and remove its foreign-toplevel handle so taskbar-style clients stop listing a + /// window that is gone. fn toplevel_destroyed(&mut self, surface: ToplevelSurface) { if let Some(idx) = self.pending_windows.iter().position(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) { self.pending_windows.remove(idx); } + let mapped = self + .space + .elements() + .find(|w| w.toplevel().map(|t| *t == surface).unwrap_or(false)) + .cloned(); + if let Some(window) = mapped { + self.space.unmap_elem(&window); + } if let Some(handle) = with_states(surface.wl_surface(), |states| states.data_map.get::().cloned()) { self.foreign_toplevel_list.remove_toplevel(&handle); } } } +/// In-house `zwp_virtual_keyboard_v1` implementation. Smithay's manager swaps the +/// client-visible seat keymap to the virtual keyboard's keymap on every VK event and never +/// restores it, leaving every client holding a foreign keymap (and killing the compositor's +/// overlay keycodes) after any VK use. Here VK key events are TRANSLATED instead: each keycode +/// resolves to its level-0 keysym under the VK client's own uploaded keymap, maps onto the seat +/// keymap (overlay-binding on demand, batched at keymap upload), and injects through the seat's +/// regular input path — the seat keymap identity never changes and modifier/pressed-key state +/// stays coherent with server-side injection. VK `modifiers` requests are ignored: applying a +/// foreign modifier mask would corrupt the seat's own tracked state, and the supported VK +/// client (selkies' wayland_typer) binds every keysym at level 0 and never sends them. +pub struct PfVirtualKeyboard { + inner: Mutex, +} + +#[derive(Default)] +struct PfVkState { + /// Level-0 keysym per xkb keycode of the client's uploaded keymap. + syms: Option>, + /// VK xkb keycode -> injected seat keycode, so a release always matches its press even + /// across policy rebinds. + pressed: std::collections::HashMap, +} + +impl GlobalDispatch for AppState { + fn bind( + _state: &mut Self, + _dh: &DisplayHandle, + _client: &Client, + resource: New, + _global_data: &(), + data_init: &mut DataInit<'_, Self>, + ) { + data_init.init(resource, ()); + } +} + +impl Dispatch for AppState { + fn request( + _state: &mut Self, + _client: &Client, + _resource: &ZwpVirtualKeyboardManagerV1, + request: zwp_virtual_keyboard_manager_v1::Request, + _data: &(), + _dh: &DisplayHandle, + data_init: &mut DataInit<'_, Self>, + ) { + if let zwp_virtual_keyboard_manager_v1::Request::CreateVirtualKeyboard { seat: _, id } = + request + { + data_init.init(id, PfVirtualKeyboard { inner: Mutex::new(PfVkState::default()) }); + } + } +} + +impl Dispatch for AppState { + fn request( + state: &mut Self, + _client: &Client, + resource: &ZwpVirtualKeyboardV1, + request: zwp_virtual_keyboard_v1::Request, + data: &PfVirtualKeyboard, + _dh: &DisplayHandle, + _data_init: &mut DataInit<'_, Self>, + ) { + use smithay::input::keyboard::xkb; + match request { + zwp_virtual_keyboard_v1::Request::Keymap { format, fd, size } => { + if format != 1 { + return; + } + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = unsafe { + xkb::Keymap::new_from_fd( + &ctx, + fd, + size as usize, + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) + }; + let Ok(Some(keymap)) = keymap else { + eprintln!("[Wayland] virtual-keyboard keymap failed to compile; ignoring."); + return; + }; + let syms = crate::wayland::keymap::level0_syms(&keymap); + // Pre-bind everything this keymap can type that the seat cannot, in ONE + // seat keymap swap, so the following key events bind nothing. + let missing: Vec = syms + .values() + .copied() + .filter(|&s| !state.keymap_policy.resolves_plain(s)) + .collect(); + if !missing.is_empty() { + let _ = state.bind_keysyms_plain(&missing); + } + data.inner.lock().unwrap().syms = Some(syms); + } + zwp_virtual_keyboard_v1::Request::Key { time: _, key, state: key_state } => { + let mut vk = data.inner.lock().unwrap(); + if vk.syms.is_none() { + drop(vk); + resource.post_error( + zwp_virtual_keyboard_v1::Error::NoKeymap, + "`key` sent before keymap.", + ); + return; + } + let vk_kc = key.wrapping_add(8); + let seat_kc = if key_state == 1 { + let sym = vk.syms.as_ref().and_then(|m| m.get(&vk_kc)).copied(); + // Translate through the seat keymap; an untranslatable keycode + // passes through raw (base sections of both keymaps agree for + // ordinary pc keycodes). + let kc = match sym { + Some(sym) => { + let bound = state.bind_keysyms_plain(&[sym])[0]; + if bound != 0 { bound } else { vk_kc } + } + None => vk_kc, + }; + vk.pressed.insert(vk_kc, kc); + kc + } else { + vk.pressed.remove(&vk_kc).unwrap_or(vk_kc) + }; + drop(vk); + let pressed = key_state == 1; + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(seat_kc), + if pressed { + smithay::backend::input::KeyState::Pressed + } else { + smithay::backend::input::KeyState::Released + }, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + zwp_virtual_keyboard_v1::Request::Modifiers { .. } => {} + zwp_virtual_keyboard_v1::Request::Destroy => {} + _ => {} + } + } + + /// Release every seat key this virtual keyboard still holds, so a VK client that + /// disconnects mid-press cannot leave keys logically stuck. + fn destroyed( + state: &mut Self, + _client: ClientId, + _resource: &ZwpVirtualKeyboardV1, + data: &PfVirtualKeyboard, + ) { + let held: Vec = data.inner.lock().unwrap().pressed.drain().map(|(_, kc)| kc).collect(); + if held.is_empty() { + return; + } + if let Some(keyboard) = state.seat.get_keyboard() { + let keyboard = keyboard.clone(); + for kc in held { + let serial = next_serial(); + let time = wayland_time(); + keyboard.input( + state, + smithay::backend::input::Keycode::new(kc), + smithay::backend::input::KeyState::Released, + serial, + time, + |_, _, _| smithay::input::keyboard::FilterResult::<()>::Forward, + ); + } + } + } +} + /// Per-client data attached to every Wayland client connection; holds the compositor's /// per-client surface state. #[derive(Default)] @@ -1761,7 +2270,6 @@ delegate_seat!(AppState); delegate_xdg_shell!(AppState); delegate_dmabuf!(AppState); delegate_fractional_scale!(AppState); -delegate_virtual_keyboard_manager!(AppState); delegate_data_device!(AppState); delegate_data_control!(AppState); delegate_pointer_warp!(AppState); @@ -1782,7 +2290,7 @@ delegate_primary_selection!(AppState); /// Dividing the buffer length by the height recovers a padded stride, so a padded readback cannot /// skew the cursor image; the result never drops below one full `width*4` row, and a zero height /// short-circuits to one row to avoid dividing by zero. -fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { +pub(crate) fn rgba_readback_stride(buf_len: usize, height: usize, width: usize) -> usize { let row = width.saturating_mul(4); if height == 0 { return row; diff --git a/pixelflux/src/wayland/host.rs b/pixelflux/src/wayland/host.rs new file mode 100644 index 0000000..1b3e11f --- /dev/null +++ b/pixelflux/src/wayland/host.rs @@ -0,0 +1,1431 @@ +//! Host-capture mode: pixelflux as a CLIENT of an external Wayland compositor +//! (e.g. labwc running `WLR_BACKENDS=headless`), inverting the nested topology. +//! +//! The host compositor owns the session — one seat, one selection, one screen +//! model — and pixelflux captures and injects as a privileged-protocol client: +//! frames via `zwlr_screencopy_v1` (v3 `copy_with_damage`, so idle screens cost +//! nothing), keyboard via a persistent `zwp_virtual_keyboard_v1` device carrying +//! selkies' own keymap text, pointer via `zwlr_virtual_pointer_v1`. Zero-copy is +//! preserved by allocating the capture buffers from pixelflux's OWN GBM device +//! (render node — no privileges): the compositor blits straight into the dmabufs +//! the encoder imports, so no CPU ever touches a frame. Without a GPU the same +//! loop degrades to wl_shm buffers feeding the CPU encode pool. +//! +//! Every host `wl_output` maps to one selkies display id, in registry order: a +//! control thread on the primary connection owns wlr-output-management and +//! applies the whole layout (per-output custom modes plus row positions) as one +//! atomic configuration, while each output gets its own capture connection and +//! thread. Input proxies are called from the calloop thread directly +//! (wayland-client proxies are thread-safe); pointer coordinates arrive in the +//! same union-layout space selkies uses, so the virtual-pointer extent is the +//! layout's bounding box. All capture-side waits poll a wake pipe next to the +//! Wayland socket, so a resize or teardown lands immediately even while the +//! thread is parked in a `copy_with_damage` wait on a static screen. + +use std::fs::File; +use std::os::fd::{AsFd, AsRawFd, OwnedFd, RawFd}; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, TryRecvError}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use gbm::{BufferObjectFlags, Device as GbmDevice, Format as GbmFormat}; +use smithay::backend::allocator::dmabuf::Dmabuf; +use smithay::backend::allocator::Buffer as _; +use smithay::utils::{Physical, Rectangle}; +use wayland_client::protocol::{wl_output, wl_pointer, wl_registry, wl_seat, wl_shm, wl_shm_pool}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, Proxy, QueueHandle, WEnum}; +use wayland_protocols::wp::linux_dmabuf::zv1::client::{ + zwp_linux_buffer_params_v1::{self, ZwpLinuxBufferParamsV1}, + zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1, +}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; +use wayland_protocols_wlr::screencopy::v1::client::{ + zwlr_screencopy_frame_v1::{self, ZwlrScreencopyFrameV1}, + zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1, +}; +use wayland_protocols_wlr::output_management::v1::client::{ + zwlr_output_configuration_head_v1::ZwlrOutputConfigurationHeadV1, + zwlr_output_configuration_v1::{self, ZwlrOutputConfigurationV1}, + zwlr_output_head_v1::{self, ZwlrOutputHeadV1}, + zwlr_output_manager_v1::{self, ZwlrOutputManagerV1}, + zwlr_output_mode_v1::ZwlrOutputModeV1, +}; +use wayland_protocols_wlr::virtual_pointer::v1::client::{ + zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1, + zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1, +}; + +use crate::wayland::wlclient::{ + bounded_roundtrip, drain_pipe, impl_sync_callback, memfd_with, socket_path, wait_readable2, + wake_pipe, wake_write, SyncState, +}; + +const KEYMAP_FORMAT_XKB_V1: u32 = 1; +/// Frames in flight between a capture thread and the encode tick. Two slots: +/// one being blitted by the compositor while the previous one is encoded. +const SLOTS: usize = 2; + +/// A frame the compositor finished blitting, ready for the encoder. +pub struct HostFrame { + gen: u64, + slot: usize, + /// GPU path: the filled dmabuf (Arc-backed; the buffer itself stays owned by + /// the capture thread and is reused once the slot is released). + pub dmabuf: Option, + /// Software path: the shm mapping itself — the consumer converts straight + /// out of it (one copy total); the slot is not reused until release. + pub cpu: Option, + pub width: i32, + pub height: i32, + pub damage: Vec>, +} + +/// Borrow-by-Arc view of a software frame still sitting in its shm mapping. +pub struct HostCpuFrame { + map: Arc, + stride: usize, + format: u32, +} + +impl HostCpuFrame { + /// Convert the frame into tight BGRA rows in `dst` (sized `w*4*h`); the + /// announced shm format decides the per-pixel conversion (compositors pick + /// their renderer's preferred read format, e.g. 24-bit BGR on NVIDIA GLES). + pub fn write_bgra(&self, w: i32, h: i32, dst: &mut [u8]) { + let row = (w * 4) as usize; + let (src_bpp, swap_rb) = match self.format { + f if f == wl_shm::Format::Xbgr8888 as u32 + || f == wl_shm::Format::Abgr8888 as u32 => (4usize, true), + f if f == wl_shm::Format::Bgr888 as u32 => (3, false), + _ => (4, false), // xrgb/argb: already BGRA byte order + }; + for y in 0..h as usize { + let src_start = y * self.stride; + let src_end = (src_start + src_bpp * w as usize).min(self.map.len()); + if src_start >= src_end || (y + 1) * row > dst.len() { + break; + } + let src_row = &self.map[src_start..src_end]; + let dst_row = &mut dst[y * row..(y + 1) * row]; + match (src_bpp, swap_rb) { + (4, false) => { + let n = row.min(src_row.len()); + dst_row[..n].copy_from_slice(&src_row[..n]); + } + (4, true) => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(4)) { + d[0] = s[2]; + d[1] = s[1]; + d[2] = s[0]; + d[3] = s[3]; + } + } + _ => { + for (d, s) in dst_row.chunks_exact_mut(4).zip(src_row.chunks_exact(3)) { + d[0] = s[0]; + d[1] = s[1]; + d[2] = s[2]; + d[3] = 0xff; + } + } + } + } + } +} + +/// One display's slice of the union layout, as selkies configured it. Slots are +/// keyed by selkies display id ('display2' is id 2 — ids are sparse); active ids +/// map onto host outputs by ascending rank. +#[derive(Clone, Copy, Default)] +struct LayoutSlot { + want: (i32, i32), + pos: (i32, i32), + active: bool, +} + +enum ToHost { + Start { width: i32, height: i32 }, + /// Slot return; `gen` guards against slots recycled by a renegotiation + /// while the consumer still held the frame. + Release { gen: u64, slot: usize }, + /// Park the capture (display stopped) without ending the thread. + Idle, +} + +enum CtrlMsg { + Apply(Vec), +} + +// --------------------------------------------------------------------------- +// Control connection: seat + virtual input devices + wlr-output-management. +// --------------------------------------------------------------------------- + +struct CtrlState { + seat: Option, + vk_mgr: Option, + vptr_mgr: Option, + has_screencopy: bool, + outputs: Vec<(wl_output::WlOutput, Option)>, + /// Registry indices in display order (natural name sort — registry + /// announcement order is not guaranteed to match output creation order). + order: Vec, + output_mgr: Option, + heads: Vec<(ZwlrOutputHeadV1, Option)>, + om_serial: Option, + cfg_result: Option, + cfg_cancelled: bool, + sync_done: bool, +} + +impl Default for CtrlState { + fn default() -> Self { + Self { + seat: None, + vk_mgr: None, + vptr_mgr: None, + has_screencopy: false, + outputs: Vec::new(), + order: Vec::new(), + output_mgr: None, + heads: Vec::new(), + om_serial: None, + cfg_result: None, + cfg_cancelled: false, + sync_done: false, + } + } +} + +/// Natural sort key for an output name: text prefix plus trailing number, so +/// HEADLESS-2 orders before HEADLESS-10. Unnamed outputs keep registry order +/// after every named one. +fn output_order_key(name: Option<&String>, registry_idx: usize) -> (bool, String, u64, usize) { + match name { + Some(n) => { + let digits_at = n.rfind(|c: char| !c.is_ascii_digit()).map(|i| i + 1).unwrap_or(0); + let num = n[digits_at..].parse::().unwrap_or(0); + (false, n[..digits_at].to_string(), num, registry_idx) + } + None => (true, String::new(), 0, registry_idx), + } +} + +impl SyncState for CtrlState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(CtrlState); + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())) + } + "zwp_virtual_keyboard_manager_v1" => { + state.vk_mgr = Some(registry.bind(name, 1, qh, ())) + } + "zwlr_virtual_pointer_manager_v1" => { + state.vptr_mgr = Some(registry.bind(name, version.min(2), qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => state.has_screencopy = true, + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } + "zwlr_output_manager_v1" => { + state.output_mgr = Some(registry.bind(name, 1, qh, ())) + } + _ => {} + } + } + } +} + +macro_rules! impl_output_name { + ($t:ty) => { + impl Dispatch for $t { + fn event( + state: &mut Self, + _: &wl_output::WlOutput, + event: wl_output::Event, + idx: &usize, + _: &Connection, + _: &QueueHandle, + ) { + if let wl_output::Event::Name { name } = event { + if let Some(o) = state.outputs.get_mut(*idx) { + o.1 = Some(name); + } + } + } + } + }; +} +impl_output_name!(CtrlState); + +delegate_noop!(CtrlState: ignore wl_seat::WlSeat); +delegate_noop!(CtrlState: ZwpVirtualKeyboardManagerV1); +delegate_noop!(CtrlState: ZwpVirtualKeyboardV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerManagerV1); +delegate_noop!(CtrlState: ZwlrVirtualPointerV1); +delegate_noop!(CtrlState: ignore ZwlrOutputModeV1); +delegate_noop!(CtrlState: ZwlrOutputConfigurationHeadV1); + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + _: &ZwlrOutputManagerV1, + event: zwlr_output_manager_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_manager_v1::Event::Head { head } => state.heads.push((head, None)), + zwlr_output_manager_v1::Event::Done { serial } => state.om_serial = Some(serial), + _ => {} + } + } + + wayland_client::event_created_child!(CtrlState, ZwlrOutputManagerV1, [ + zwlr_output_manager_v1::EVT_HEAD_OPCODE => (ZwlrOutputHeadV1, ()), + ]); +} + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + head: &ZwlrOutputHeadV1, + event: zwlr_output_head_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_head_v1::Event::Name { name } => { + if let Some(h) = state.heads.iter_mut().find(|(h, _)| h.id() == head.id()) { + h.1 = Some(name); + } + } + zwlr_output_head_v1::Event::Finished => { + state.heads.retain(|(h, _)| h.id() != head.id()); + } + _ => {} + } + } + + wayland_client::event_created_child!(CtrlState, ZwlrOutputHeadV1, [ + zwlr_output_head_v1::EVT_MODE_OPCODE => (ZwlrOutputModeV1, ()), + ]); +} + +impl Dispatch for CtrlState { + fn event( + state: &mut Self, + _: &ZwlrOutputConfigurationV1, + event: zwlr_output_configuration_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_output_configuration_v1::Event::Succeeded => state.cfg_result = Some(true), + zwlr_output_configuration_v1::Event::Failed => state.cfg_result = Some(false), + zwlr_output_configuration_v1::Event::Cancelled => state.cfg_cancelled = true, + _ => {} + } + } +} + +// --------------------------------------------------------------------------- +// Capture connections: one per host output (screencopy + buffer allocation). +// --------------------------------------------------------------------------- + +struct CaptureState { + shm: Option, + dmabuf: Option, + screencopy: Option, + outputs: Vec<(wl_output::WlOutput, Option)>, + // Per-frame capture negotiation/results. + announce_dmabuf: Option<(u32, i32, i32)>, // fourcc, w, h + announce_shm: Option<(u32, i32, i32, i32)>, // format, w, h, stride + buffer_done: bool, + damage: Vec>, + ready: bool, + failed: bool, + sync_done: bool, +} + +impl Default for CaptureState { + fn default() -> Self { + Self { + shm: None, + dmabuf: None, + screencopy: None, + outputs: Vec::new(), + announce_dmabuf: None, + announce_shm: None, + buffer_done: false, + damage: Vec::new(), + ready: false, + failed: false, + sync_done: false, + } + } +} + +impl CaptureState { + fn reset_frame(&mut self) { + self.announce_dmabuf = None; + self.announce_shm = None; + self.buffer_done = false; + self.damage.clear(); + self.ready = false; + self.failed = false; + } +} + +impl SyncState for CaptureState { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(CaptureState); +impl_output_name!(CaptureState); + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, version } = event { + match interface.as_str() { + "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), + "zwp_linux_dmabuf_v1" if version >= 3 => { + state.dmabuf = Some(registry.bind(name, 3, qh, ())) + } + "zwlr_screencopy_manager_v1" if version >= 3 => { + state.screencopy = Some(registry.bind(name, 3, qh, ())) + } + "wl_output" => { + let idx = state.outputs.len(); + let out = registry.bind(name, version.min(4), qh, idx); + state.outputs.push((out, None)); + } + _ => {} + } + } + } +} + +delegate_noop!(CaptureState: ignore wl_shm::WlShm); +delegate_noop!(CaptureState: ignore wl_shm_pool::WlShmPool); +delegate_noop!(CaptureState: ignore wayland_client::protocol::wl_buffer::WlBuffer); +delegate_noop!(CaptureState: ZwlrScreencopyManagerV1); + +impl Dispatch for CaptureState { + fn event( + _: &mut Self, + _: &ZwpLinuxDmabufV1, + _: ::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // format/modifier advertisements: the allocation follows the screencopy + // frame's announcement instead. + } +} + +impl Dispatch for CaptureState { + fn event( + _: &mut Self, + _: &ZwpLinuxBufferParamsV1, + _: zwp_linux_buffer_params_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + // created/failed are only sent for non-immed creation. + } +} + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + _: &ZwlrScreencopyFrameV1, + event: zwlr_screencopy_frame_v1::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + zwlr_screencopy_frame_v1::Event::Buffer { format, width, height, stride } => { + if let WEnum::Value(f) = format { + state.announce_shm = Some((f as u32, width as i32, height as i32, stride as i32)); + } + } + zwlr_screencopy_frame_v1::Event::LinuxDmabuf { format, width, height } => { + state.announce_dmabuf = Some((format, width as i32, height as i32)); + } + zwlr_screencopy_frame_v1::Event::BufferDone => state.buffer_done = true, + zwlr_screencopy_frame_v1::Event::Damage { x, y, width, height } => { + state.damage.push(Rectangle::new( + (x as i32, y as i32).into(), + (width as i32, height as i32).into(), + )); + } + zwlr_screencopy_frame_v1::Event::Ready { .. } => state.ready = true, + zwlr_screencopy_frame_v1::Event::Failed => state.failed = true, + _ => {} + } + } +} + +enum SlotBuffer { + Gpu { + _bo: gbm::BufferObject<()>, + dmabuf: Dmabuf, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, + Cpu { + _pool: wl_shm_pool::WlShmPool, + map: Arc, + stride: i32, + format: u32, + wl: wayland_client::protocol::wl_buffer::WlBuffer, + }, +} + +/// One host output's calloop-side handle: control channel, wake pipe, frames. +struct OutputHandle { + to_thread: Sender, + wake: OwnedFd, + frames: Receiver, + /// Newest frame, kept (slot and all) until replaced so an IDR request on a + /// static screen can re-encode current content like compositor mode does. + retained: Mutex>, + name: Option, +} + +impl OutputHandle { + fn send(&self, msg: ToHost) { + let _ = self.to_thread.send(msg); + wake_write(self.wake.as_raw_fd()); + } +} + +/// The calloop-side session: input proxies plus one capture handle per output. +pub struct HostSession { + conn: Connection, + vk: Option, + vptr: Option, + ctrl_tx: Sender, + ctrl_wake: OwnedFd, + outputs: Vec, + layout: Mutex>, + alive: Arc, +} + +impl HostSession { + /// Connect to `display`, bring up input devices, enumerate the host's + /// outputs and spawn one capture thread per output (idle until + /// [`start_capture`]) plus the layout-control thread. `gbm_path` (this + /// process's render node) enables the zero-copy path. + pub fn connect(display: &str, gbm_path: Option) -> Result { + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = CtrlState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let seat = state.seat.clone().ok_or("host compositor advertises no wl_seat")?; + if !state.has_screencopy { + return Err("host compositor lacks zwlr_screencopy_manager_v1 (v3)".into()); + } + if state.outputs.is_empty() { + return Err("host compositor has no wl_output".into()); + } + + let vk = match &state.vk_mgr { + Some(mgr) => { + let vk = mgr.create_virtual_keyboard(&seat, &qh, ()); + // A keymap must precede any key event; selkies replaces this with + // its managed keymap through the ABI as soon as it starts. + if let Some(text) = crate::wayland::vkclient::us_base_text() { + let mut data = text.as_bytes().to_vec(); + data.push(0); + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + } + Some(vk) + } + None => { + eprintln!("[HostCapture] no zwp_virtual_keyboard_manager_v1: keyboard injection disabled."); + None + } + }; + let vptr = match &state.vptr_mgr { + Some(mgr) => Some(mgr.create_virtual_pointer(Some(&seat), &qh, ())), + None => { + eprintln!("[HostCapture] no zwlr_virtual_pointer_manager_v1: pointer injection disabled."); + None + } + }; + // Second round-trip: wl_output v4 name events and output-management heads. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let alive = Arc::new(AtomicBool::new(true)); + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let names: Vec> = + order.iter().map(|&i| state.outputs[i].1.clone()).collect(); + state.order = order; + println!( + "[HostCapture] host outputs: {}.", + names + .iter() + .enumerate() + .map(|(i, n)| format!("{i}={}", n.as_deref().unwrap_or("?"))) + .collect::>() + .join(", ") + ); + + let mut outputs = Vec::new(); + for (i, name) in names.iter().cloned().enumerate() { + let (wake_rd, wake_wr) = wake_pipe()?; + let (to_thread, from_main) = std::sync::mpsc::channel::(); + let (frame_tx, frames) = std::sync::mpsc::channel::(); + let display = display.to_string(); + let expect = name.clone(); + let gbm_path = gbm_path.clone(); + let alive = alive.clone(); + std::thread::Builder::new() + .name(format!("pf-host-cap{i}")) + .spawn(move || { + if let Err(e) = + capture_loop(&display, i, expect, gbm_path, from_main, wake_rd, frame_tx) + { + eprintln!("[HostCapture] output {i} capture ended: {e}"); + } + if i == 0 { + alive.store(false, Ordering::Relaxed); + } + }) + .map_err(|e| format!("spawn: {e}"))?; + outputs.push(OutputHandle { + to_thread, + wake: wake_wr, + frames, + retained: Mutex::new(None), + name, + }); + } + + let (ctrl_tx, ctrl_rx) = std::sync::mpsc::channel::(); + let (ctrl_wake_rd, ctrl_wake) = wake_pipe()?; + { + let conn = conn.clone(); + std::thread::Builder::new() + .name("pf-host-ctrl".into()) + .spawn(move || control_loop(conn, queue, state, ctrl_rx, ctrl_wake_rd)) + .map_err(|e| format!("spawn: {e}"))?; + } + + let layout = Mutex::new(std::collections::BTreeMap::new()); + Ok(Self { conn, vk, vptr, ctrl_tx, ctrl_wake, outputs, layout, alive }) + } + + pub fn output_count(&self) -> usize { + self.outputs.len() + } + + /// The host output backing `display_id`: active display ids map onto host + /// outputs by ascending rank (selkies ids are sparse — 'display2' is id 2). + fn output_index_for(&self, display_id: u32) -> Option { + let layout = self.layout.lock().unwrap(); + let rank = layout + .iter() + .filter(|(_, s)| s.active) + .position(|(id, _)| *id == display_id)?; + (rank < self.outputs.len()).then_some(rank) + } + + /// True when an active capture on `display_id` has a host output behind it. + pub fn has_output_for(&self, display_id: u32) -> bool { + self.output_index_for(display_id).is_some() + } + + /// Record where selkies laid out `display_id` (union coordinates); pushed + /// to the host with the next capture (re)start. + pub fn set_layout(&self, display_id: u32, x: i32, y: i32) { + let mut layout = self.layout.lock().unwrap(); + layout.entry(display_id).or_default().pos = (x, y); + } + + /// Aim `display_id` at `width`x`height`: assigns every active display its + /// host output (by rank), asks the host for all modes and positions in one + /// atomic configuration, and points the capture threads at their sizes; + /// frames gate until the host applies them. + pub fn start_capture(&self, display_id: u32, width: i32, height: i32) { + let assignments = { + let mut layout = self.layout.lock().unwrap(); + let slot = layout.entry(display_id).or_default(); + slot.want = (width, height); + slot.active = true; + let active: Vec<(u32, LayoutSlot)> = layout + .iter() + .filter(|(_, s)| s.active) + .map(|(id, s)| (*id, *s)) + .collect(); + if active.iter().position(|(id, _)| *id == display_id).unwrap_or(usize::MAX) + >= self.outputs.len() + { + eprintln!( + "[HostCapture] display {display_id} has no host output (host has {}); not captured.", + self.outputs.len() + ); + } + active + }; + // Ranks may have shifted (a lower id joined): repoint every assigned + // capture thread; a same-size Start is a no-op for an unaffected one. + let mut by_output: Vec = Vec::new(); + for (rank, (_, slot)) in assignments.iter().enumerate() { + if rank >= self.outputs.len() { + break; + } + self.outputs[rank].send(ToHost::Start { width: slot.want.0, height: slot.want.1 }); + by_output.push(*slot); + } + let _ = self.ctrl_tx.send(CtrlMsg::Apply(by_output)); + wake_write(self.ctrl_wake.as_raw_fd()); + } + + /// Park `display_id`'s capture (its slot leaves the pointer extent). The + /// retained frame goes back to the capture pool — dropping it silently + /// would wedge one of the two slots until the next renegotiation. + pub fn idle_output(&self, display_id: u32) { + let Some(idx) = self.output_index_for(display_id) else { + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + return; + }; + self.layout.lock().unwrap().entry(display_id).or_default().active = false; + self.outputs[idx].send(ToHost::Idle); + if let Some(old) = self.outputs[idx].retained.lock().unwrap().take() { + self.outputs[idx].send(ToHost::Release { gen: old.gen, slot: old.slot }); + } + } + + pub fn alive(&self) -> bool { + self.alive.load(Ordering::Relaxed) + } + + /// Newest ready frame for `display_id`, releasing any staler ones straight + /// back to the pool. + pub fn try_take_frame(&self, display_id: u32) -> Option { + let handle = self.outputs.get(self.output_index_for(display_id)?)?; + let mut newest: Option = None; + loop { + match handle.frames.try_recv() { + Ok(frame) => { + if let Some(stale) = newest.replace(frame) { + handle.send(ToHost::Release { gen: stale.gen, slot: stale.slot }); + } + } + Err(TryRecvError::Empty) | Err(TryRecvError::Disconnected) => break, + } + } + newest + } + + /// Keep `frame` as `display_id`'s current content (releasing the one it + /// replaces). + pub fn retain_frame(&self, display_id: u32, frame: HostFrame) { + let Some(handle) = self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) + else { + return; + }; + let old = handle.retained.lock().unwrap().replace(frame); + if let Some(old) = old { + handle.send(ToHost::Release { gen: old.gen, slot: old.slot }); + } + } + + /// Run `f` with `display_id`'s retained (current-content) frame, if any. + pub fn with_retained( + &self, + display_id: u32, + f: impl FnOnce(Option<&HostFrame>) -> R, + ) -> R { + match self.output_index_for(display_id).and_then(|i| self.outputs.get(i)) { + Some(handle) => f(handle.retained.lock().unwrap().as_ref()), + None => f(None), + } + } + + /// Upload selkies' managed keymap to the virtual keyboard verbatim. + pub fn set_keymap(&self, text: &str) { + let Some(vk) = &self.vk else { return }; + let mut data = text.as_bytes().to_vec(); + data.push(0); + match memfd_with(&data) { + Ok(fd) => { + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + let _ = self.conn.flush(); + } + Err(e) => eprintln!("[HostCapture] keymap upload failed: {e}"), + } + } + + /// Key event in xkb numbering (evdev + 8), matching the seat injectors. + pub fn key(&self, xkb_keycode: u32, pressed: bool) { + let Some(vk) = &self.vk else { return }; + if xkb_keycode < 8 { + return; + } + vk.key(0, xkb_keycode - 8, if pressed { 1 } else { 0 }); + let _ = self.conn.flush(); + } + + /// Union-layout bounding box of every active, output-backed display (the + /// virtual-pointer extent, matching the space selkies injects in). + fn extent(&self) -> (i32, i32) { + let layout = self.layout.lock().unwrap(); + let mut w = 0; + let mut h = 0; + for (rank, (_, slot)) in layout.iter().filter(|(_, s)| s.active).enumerate() { + if rank >= self.outputs.len() { + break; + } + w = w.max(slot.pos.0 + slot.want.0); + h = h.max(slot.pos.1 + slot.want.1); + } + (w, h) + } + + pub fn pointer_motion_abs(&self, x: f64, y: f64) { + let Some(vp) = &self.vptr else { return }; + let (w, h) = self.extent(); + if w <= 0 || h <= 0 { + return; + } + let cx = x.clamp(0.0, (w - 1) as f64) as u32; + let cy = y.clamp(0.0, (h - 1) as f64) as u32; + vp.motion_absolute(0, cx, cy, w as u32, h as u32); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_button(&self, btn: u32, pressed: bool) { + let Some(vp) = &self.vptr else { return }; + vp.button( + 0, + btn, + if pressed { wl_pointer::ButtonState::Pressed } else { wl_pointer::ButtonState::Released }, + ); + vp.frame(); + let _ = self.conn.flush(); + } + + pub fn pointer_axis(&self, dx: f64, dy: f64) { + let Some(vp) = &self.vptr else { return }; + if dy != 0.0 { + vp.axis(0, wl_pointer::Axis::VerticalScroll, dy); + } + if dx != 0.0 { + vp.axis(0, wl_pointer::Axis::HorizontalScroll, dx); + } + vp.frame(); + let _ = self.conn.flush(); + } +} + +// Dropping the session drops every channel sender and wake-pipe write end; the +// capture and control threads observe the disconnect (poll sees POLLHUP) and +// exit on their own. + +// --------------------------------------------------------------------------- +// Control thread: pumps the primary connection and applies layout requests. +// --------------------------------------------------------------------------- + +fn control_loop( + conn: Connection, + mut queue: EventQueue, + mut state: CtrlState, + ctrl_rx: Receiver, + wake_rd: OwnedFd, +) { + let qh = queue.handle(); + loop { + let mut pending: Option> = None; + loop { + match ctrl_rx.try_recv() { + Ok(CtrlMsg::Apply(slots)) => pending = Some(slots), + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + if let Some(slots) = pending { + apply_layout(&conn, &mut queue, &mut state, &qh, &slots); + } + if queue.dispatch_pending(&mut state).is_err() { + return; + } + let _ = queue.flush(); + if let Some(guard) = conn.prepare_read() { + match wait_readable2( + guard.connection_fd().as_raw_fd(), + wake_rd.as_raw_fd(), + Some(Duration::from_secs(1)), + ) { + Ok((wl, wake)) => { + if wake { + drain_pipe(wake_rd.as_raw_fd()); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + } + } + Err(_) => return, + } + } + } +} + +/// Ask the host (wlr-output-management) to give every active output its wanted +/// mode and layout position in one atomic configuration. Retries across a +/// `cancelled` (stale serial); a refusal only warns — capture threads keep +/// gating on the size they were promised. +fn apply_layout( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CtrlState, + qh: &QueueHandle, + slots: &[LayoutSlot], +) { + let Some(mgr) = state.output_mgr.clone() else { + eprintln!("[HostCapture] host lacks zwlr_output_manager_v1; capture follows the host's own size."); + return; + }; + let deadline = Instant::now() + Duration::from_secs(5); + for _ in 0..3 { + while state.om_serial.is_none() && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + return; + } + } + let Some(serial) = state.om_serial else { + eprintln!("[HostCapture] output-management serial never arrived; resize skipped."); + return; + }; + state.cfg_result = None; + state.cfg_cancelled = false; + let cfg = mgr.create_configuration(serial, qh, ()); + let mut any = false; + for (i, slot) in slots.iter().enumerate() { + if !slot.active { + continue; + } + let want_name = state + .order + .get(i) + .and_then(|&oi| state.outputs.get(oi)) + .and_then(|(_, n)| n.clone()); + let head = match want_name + .as_ref() + .and_then(|n| state.heads.iter().find(|(_, hn)| hn.as_ref() == Some(n))) + .or_else(|| state.heads.get(i)) + { + Some((h, _)) => h.clone(), + None => { + eprintln!("[HostCapture] no output-management head for output {i}; not resized."); + continue; + } + }; + let cfg_head = cfg.enable_head(&head, qh, ()); + cfg_head.set_custom_mode(slot.want.0, slot.want.1, 0); + cfg_head.set_position(slot.pos.0, slot.pos.1); + any = true; + } + if !any { + cfg.destroy(); + return; + } + cfg.apply(); + let _ = queue.flush(); + while state.cfg_result.is_none() && !state.cfg_cancelled && Instant::now() < deadline { + if !pump_ctrl(conn, queue, state) { + cfg.destroy(); + return; + } + } + cfg.destroy(); + if state.cfg_cancelled { + // Stale serial: the compositor re-announces its state with a fresh one. + state.om_serial = None; + continue; + } + if state.cfg_result != Some(true) { + eprintln!("[HostCapture] host refused the layout; capture follows the host's own size."); + } + return; + } +} + +/// One bounded dispatch step for the control connection (false = connection died). +fn pump_ctrl(conn: &Connection, queue: &mut EventQueue, state: &mut CtrlState) -> bool { + if queue.dispatch_pending(state).is_err() { + return false; + } + let _ = queue.flush(); + let Some(guard) = conn.prepare_read() else { return true }; + match crate::wayland::wlclient::wait_readable( + guard.connection_fd().as_raw_fd(), + Duration::from_millis(200), + ) { + Ok(readable) => { + if readable { + let _ = guard.read(); + } else { + drop(guard); + } + true + } + Err(_) => false, + } +} + +// --------------------------------------------------------------------------- +// Capture threads: one connection + screencopy loop per host output. +// --------------------------------------------------------------------------- + +fn fourcc_to_gbm(fourcc: u32) -> GbmFormat { + // XR24 / AR24; anything else falls back to ARGB (the encoder reads BGRA bytes + // and ignores alpha). + match fourcc { + 0x34325258 => GbmFormat::Xrgb8888, + _ => GbmFormat::Argb8888, + } +} + +/// What a control-channel drain decided while a capture wait was in progress. +enum Ctl { + None, + Renegotiate, + Idle, + Dead, +} + +enum Pump { + Done, + Control, + Timeout, +} + +/// Dispatch events until `done(state)` holds, waiting on the Wayland socket and +/// the wake pipe together so control messages interrupt any capture wait. The +/// predicate is re-checked after every dispatch, so a wait whose condition was +/// satisfied by already-queued events returns without ever blocking. +fn pump_until( + conn: &Connection, + queue: &mut EventQueue, + state: &mut CaptureState, + wake_rd: RawFd, + timeout: Option, + done: impl Fn(&CaptureState) -> bool, +) -> Result { + let deadline = timeout.map(|t| Instant::now() + t); + loop { + queue.dispatch_pending(state).map_err(|e| format!("dispatch: {e}"))?; + if done(state) { + return Ok(Pump::Done); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = match deadline { + Some(d) => match d.checked_duration_since(Instant::now()) { + Some(r) => Some(r), + None => return Ok(Pump::Timeout), + }, + None => None, + }; + let Some(guard) = conn.prepare_read() else { continue }; + let (wl, wake) = wait_readable2(guard.connection_fd().as_raw_fd(), wake_rd, remaining)?; + if wake { + drop(guard); + drain_pipe(wake_rd); + return Ok(Pump::Control); + } + if wl { + let _ = guard.read(); + } else { + drop(guard); + if deadline.is_some_and(|d| Instant::now() >= d) { + return Ok(Pump::Timeout); + } + } + } +} + +fn capture_loop( + display: &str, + index: usize, + expect_name: Option, + gbm_path: Option, + from_main: Receiver, + wake_rd: OwnedFd, + frame_tx: Sender, +) -> Result<(), String> { + let path = socket_path(display).ok_or("XDG_RUNTIME_DIR is unset")?; + let stream = UnixStream::connect(&path).map_err(|e| format!("connect {path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = CaptureState::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + // Names (wl_output v4) arrive after the binds from the first round-trip. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let screencopy = state.screencopy.clone().ok_or("no zwlr_screencopy_manager_v1")?; + // Select this thread's output by the name the control connection assigned + // it; ordering fallback only when the host names nothing. + let by_name = expect_name.as_ref().and_then(|expect| { + state.outputs.iter().find(|(_, n)| n.as_ref() == Some(expect)).cloned() + }); + let (output, _name) = match by_name { + Some(o) => o, + None => { + let mut order: Vec = (0..state.outputs.len()).collect(); + order.sort_by_key(|&i| output_order_key(state.outputs[i].1.as_ref(), i)); + let oi = *order + .get(index) + .ok_or_else(|| format!("host has no wl_output {index}"))?; + state.outputs[oi].clone() + } + }; + let gbm = gbm_path + .as_ref() + .and_then(|p| File::options().read(true).write(true).open(p).ok()) + .and_then(|f| GbmDevice::new(f).ok()); + let wake = wake_rd.as_raw_fd(); + + let mut slots: Vec> = (0..SLOTS).map(|_| None).collect(); + let mut free: Vec = (0..SLOTS).collect(); + let mut gen: u64 = 0; + let mut announced: Option<(i32, i32)> = None; + let mut warned_mismatch = false; + let mut consecutive_failures = 0u32; + let mut want: Option<(i32, i32)> = None; + + 'main: loop { + // Drain control; block while idle or out of slots. + loop { + let blocking = want.is_none() || free.is_empty(); + let msg = if blocking { + match from_main.recv() { + Ok(m) => m, + Err(_) => return Ok(()), + } + } else { + match from_main.try_recv() { + Ok(m) => m, + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return Ok(()), + } + }; + match msg { + ToHost::Release { gen: g, slot } => { + if g == gen { + free.push(slot); + } + } + ToHost::Idle => want = None, + ToHost::Start { width, height } => { + if want != Some((width, height)) { + warned_mismatch = false; + } + want = Some((width, height)); + } + } + } + let (want_w, want_h) = match want { + Some(w) => w, + None => continue, + }; + + // One frame: negotiate, attach our buffer, wait for damage. Control + // messages (resize, idle, teardown) interrupt any of the waits. + state.reset_frame(); + let frame = screencopy.capture_output(1, &output, &qh, ()); + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| { + s.buffer_done || s.failed + })? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + frame.destroy(); + continue 'main; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } + } + if state.failed { + frame.destroy(); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(100)), + |_| false, + ); + continue; + } + + let (fw, fh) = state + .announce_dmabuf + .map(|(_, w, h)| (w, h)) + .or(state.announce_shm.map(|(_, w, h, _)| (w, h))) + .ok_or("screencopy announced no buffer type")?; + if announced != Some((fw, fh)) { + announced = Some((fw, fh)); + eprintln!( + "[HostCapture] output {index} negotiated: {fw}x{fh} gbm={} dmabuf_global={} dmabuf_announce={:?} shm={:?}", + gbm.is_some(), + state.dmabuf.is_some(), + state.announce_dmabuf, + state.announce_shm, + ); + // Size change: drop stale buffers. The generation bump makes any + // release still in flight for the old buffers a no-op. + for s in slots.iter_mut() { + *s = None; + } + free = (0..SLOTS).collect(); + gen += 1; + } + if (fw, fh) != (want_w, want_h) { + // The encoder is configured for (want_w, want_h): hold frames until + // the host applies the resize rather than encode mismatched buffers. + frame.destroy(); + if !warned_mismatch { + warned_mismatch = true; + eprintln!( + "[HostCapture] output {index}: waiting for host {want_w}x{want_h} (currently {fw}x{fh})." + ); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(150)), + |_| false, + ); + continue; + } + warned_mismatch = false; + + let slot_idx = *free.last().unwrap(); + if slots[slot_idx].is_none() { + slots[slot_idx] = Some(match (&gbm, &state.dmabuf, state.announce_dmabuf) { + (Some(dev), Some(dmabuf_global), Some((fourcc, w, h))) => { + let bo = dev + .create_buffer_object::<()>( + w as u32, + h as u32, + fourcc_to_gbm(fourcc), + BufferObjectFlags::RENDERING, + ) + .map_err(|e| format!("GBM allocation {w}x{h}: {e:?}"))?; + let dmabuf = crate::create_dmabuf_from_bo(&bo); + let params = dmabuf_global.create_params(&qh, ()); + let modifier: u64 = dmabuf.format().modifier.into(); + for (i, handle) in dmabuf.handles().enumerate() { + params.add( + handle, + i as u32, + dmabuf.offsets().nth(i).unwrap_or(0), + dmabuf.strides().nth(i).unwrap_or(0), + (modifier >> 32) as u32, + (modifier & 0xffff_ffff) as u32, + ); + } + let wl = params.create_immed(w, h, fourcc, zwp_linux_buffer_params_v1::Flags::empty(), &qh, ()); + params.destroy(); + SlotBuffer::Gpu { _bo: bo, dmabuf, wl } + } + _ => { + let (format, w, h, stride) = + state.announce_shm.ok_or("no shm fallback announced")?; + let shm = state.shm.clone().ok_or("host compositor lacks wl_shm")?; + let size = (stride * h) as usize; + let fd = memfd_with(&vec![0u8; size])?; + let pool = shm.create_pool(fd.as_fd(), size as i32, &qh, ()); + let wl = pool.create_buffer( + 0, + w, + h, + stride, + WEnum::::from(format).into_result().unwrap_or(wl_shm::Format::Xrgb8888), + &qh, + (), + ); + let file = File::from(fd); + let map = unsafe { memmap2::MmapMut::map_mut(&file) } + .map_err(|e| format!("shm map: {e}"))?; + SlotBuffer::Cpu { _pool: pool, map: Arc::new(map), stride, format, wl } + } + }); + } + free.pop(); + + { + let slot = slots[slot_idx].as_ref().unwrap(); + let wl = match slot { + SlotBuffer::Gpu { wl, .. } => wl, + SlotBuffer::Cpu { wl, .. } => wl, + }; + frame.copy_with_damage(wl); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let mut aborted = false; + loop { + match pump_until(&conn, &mut queue, &mut state, wake, None, |s| s.ready || s.failed)? { + Pump::Done => break, + Pump::Control => match drain_ctl(&from_main, gen, &mut free, &mut want) { + Ctl::None => {} + Ctl::Renegotiate | Ctl::Idle => { + aborted = true; + break; + } + Ctl::Dead => { + frame.destroy(); + return Ok(()); + } + }, + Pump::Timeout => {} + } + } + frame.destroy(); + if aborted { + free.push(slot_idx); + continue 'main; + } + if state.failed { + free.push(slot_idx); + consecutive_failures += 1; + if consecutive_failures == 3 { + eprintln!("[HostCapture] output {index}: repeated screencopy failures; is the output alive?"); + } + let _ = pump_until( + &conn, + &mut queue, + &mut state, + wake, + Some(Duration::from_millis(50)), + |_| false, + ); + continue; + } + consecutive_failures = 0; + + let damage = std::mem::take(&mut state.damage); + let out = match slots[slot_idx].as_ref().unwrap() { + SlotBuffer::Gpu { dmabuf, .. } => HostFrame { + gen, + slot: slot_idx, + dmabuf: Some(dmabuf.clone()), + cpu: None, + width: fw, + height: fh, + damage, + }, + SlotBuffer::Cpu { map, stride, format, .. } => HostFrame { + gen, + slot: slot_idx, + dmabuf: None, + cpu: Some(HostCpuFrame { + map: map.clone(), + stride: *stride as usize, + format: *format, + }), + width: fw, + height: fh, + damage, + }, + }; + if frame_tx.send(out).is_err() { + return Ok(()); + } + } +} + +/// Drain the control channel from inside a capture wait: releases are applied +/// (generation-checked), and the newest Start/Idle decides the wait's fate. +fn drain_ctl( + rx: &Receiver, + gen: u64, + free: &mut Vec, + want: &mut Option<(i32, i32)>, +) -> Ctl { + let mut out = Ctl::None; + loop { + match rx.try_recv() { + Ok(ToHost::Release { gen: g, slot }) => { + if g == gen { + free.push(slot); + } + } + Ok(ToHost::Start { width, height }) => { + if *want != Some((width, height)) { + *want = Some((width, height)); + out = Ctl::Renegotiate; + } + } + Ok(ToHost::Idle) => { + *want = None; + out = Ctl::Idle; + } + Err(TryRecvError::Empty) => return out, + Err(TryRecvError::Disconnected) => return Ctl::Dead, + } + } +} diff --git a/pixelflux/src/wayland/keymap.rs b/pixelflux/src/wayland/keymap.rs new file mode 100644 index 0000000..fe3185e --- /dev/null +++ b/pixelflux/src/wayland/keymap.rs @@ -0,0 +1,427 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! Compositor-side keymap policy: one owner for the seat keymap. +//! +//! The seat keymap is BASE text (US by default, replaceable at runtime with a full +//! XKB_KEYMAP_FORMAT_TEXT_V1 string or RMLVO names) plus an OVERLAY of spare keycodes bound to +//! keysyms the base cannot produce (Unicode / IME output). All rebinds are batched: resolving N +//! new keysyms produces ONE keymap swap, and a keycode that is currently held down is never +//! recycled, so its release event always means the symbol its press meant. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use smithay::input::keyboard::xkb; + +/// First overlay keycode. Sits above both the evdev/pc105 range and the legacy selkies +/// overlay range (257-272) so a base keymap carrying those legacy binds cannot collide. +pub const OVERLAY_FIRST_KEYCODE: u32 = 0x120; +/// Last overlay keycode (inclusive). Keycodes past the X11 255 ceiling are fine for +/// pure-Wayland clients: they look keycodes up in the delivered keymap via xkbcommon. +pub const OVERLAY_LAST_KEYCODE: u32 = 0x2ff; +/// Overlay slot count (keycodes `OVERLAY_FIRST..=OVERLAY_LAST`). +pub const OVERLAY_CAPACITY: usize = (OVERLAY_LAST_KEYCODE - OVERLAY_FIRST_KEYCODE + 1) as usize; + +/// Highest shift level consulted when reverse-mapping the base keymap (plain, Shift, +/// AltGr, Shift+AltGr). +const MAX_LEVELS: u32 = 4; + +/// Seat keymap state: the base text, its reverse keysym map, and the overlay slots. +pub struct KeymapPolicy { + base_text: String, + /// keysym -> (xkb keycode, level) in the base keymap; lowest level wins. + base_map: HashMap, + /// slot index -> bound keysym. + slots: Vec>, + /// keysym -> slot index. + by_sym: HashMap, + /// Slot recycle order, oldest bind first. + lru: VecDeque, + /// First overlay keycode (xkb numbering); slot i lives at `overlay_first + i`. + overlay_first: u32, + /// Overlay slot count. + overlay_capacity: usize, +} + +/// Keysym one literal character types as: Latin-1 printables map 1:1, `\n` types +/// Return (the raw utf32 table maps it to Linefeed, which no keymap binds), other +/// control characters their `0xffXX` function keysyms, everything else the +/// `0x01000000 | codepoint` Unicode form (0 = unmappable). The keysym then resolves +/// against an ACTIVE keymap, never a hardcoded layout table. +pub fn keysym_for_char(c: char) -> u32 { + let c = if c == '\n' { '\r' } else { c }; + xkb::utf32_to_keysym(c as u32).raw() +} + +/// Compile an XKB_KEYMAP_FORMAT_TEXT_V1 string, or `None` when it does not compile. +pub fn compile_keymap(text: &str) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + xkb::Keymap::new_from_string( + &ctx, + text.to_string(), + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) +} + +/// Compile RMLVO names to keymap text, or `None` when compilation fails. Empty strings +/// select the xkbcommon defaults for that component. +pub fn compile_rmlvo( + rules: &str, + model: &str, + layout: &str, + variant: &str, + options: &str, +) -> Option { + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let options = (!options.is_empty()).then(|| options.to_string()); + let keymap = xkb::Keymap::new_from_names( + &ctx, + rules, + model, + layout, + variant, + options, + xkb::KEYMAP_COMPILE_NO_FLAGS, + )?; + Some(keymap.get_as_string(xkb::KEYMAP_FORMAT_TEXT_V1)) +} + +/// Level-0 keysym for every key of a compiled keymap (used to pre-bind a virtual-keyboard +/// client's keymap in one batch). Keys with zero or multiple level-0 syms are skipped. +pub fn level0_syms(keymap: &xkb::Keymap) -> HashMap { + let mut out = HashMap::new(); + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + for kc in lo..=hi { + let syms = keymap.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + if syms.len() == 1 { + let sym = syms[0].raw(); + if sym != 0 { + out.insert(kc, sym); + } + } + } + out +} + +impl KeymapPolicy { + /// Placeholder policy before the seat keymap is known; `rebuild_base` fills it in. + pub fn empty() -> Self { + Self::with_overlay_range(OVERLAY_FIRST_KEYCODE, OVERLAY_LAST_KEYCODE) + } + + /// Policy with a custom overlay keycode range (inclusive, xkb numbering). The seat + /// uses `empty()`'s above-255 range (pure-Wayland clients resolve it fine); the + /// virtual-keyboard client typing into a nested compositor uses a sub-256 range so + /// XWayland apps under that compositor stay reachable. + pub fn with_overlay_range(first: u32, last: u32) -> Self { + Self { + base_text: String::new(), + base_map: HashMap::new(), + slots: Vec::new(), + by_sym: HashMap::new(), + lru: VecDeque::new(), + overlay_first: first, + overlay_capacity: (last - first + 1) as usize, + } + } + + /// Replace the base keymap text and rebuild the reverse map. Overlay assignments are + /// retained (same keycodes), so keycodes already handed out stay valid across the swap. + pub fn rebuild_base(&mut self, base_text: String) { + self.base_map.clear(); + if let Some(keymap) = compile_keymap(&base_text) { + let lo = keymap.min_keycode().raw(); + let hi = keymap.max_keycode().raw(); + // Lower levels win across ALL keys, so a keysym reachable unshifted never + // resolves to a shifted position. + for level in 0..MAX_LEVELS { + for kc in lo..=hi { + let code = xkb::Keycode::new(kc); + if keymap.num_levels_for_key(code, 0) <= level { + continue; + } + for sym in keymap.key_get_syms_by_level(code, 0, level) { + let raw = sym.raw(); + if raw != 0 { + self.base_map.entry(raw).or_insert((kc, level)); + } + } + } + } + } + self.base_text = base_text; + } + + /// True once a base keymap has been installed. + pub fn has_base(&self) -> bool { + !self.base_text.is_empty() + } + + /// Resolve `keysym` without binding: base first, then an existing overlay slot. + pub fn resolve(&self, keysym: u32) -> Option<(u32, u32)> { + if let Some(&hit) = self.base_map.get(&keysym) { + return Some(hit); + } + self.by_sym + .get(&keysym) + .map(|&slot| (self.overlay_first + slot as u32, 0)) + } + + /// True when `keysym` resolves at level 0 (base or overlay) — i.e. typable without + /// synthetic modifiers. + pub fn resolves_plain(&self, keysym: u32) -> bool { + matches!(self.resolve(keysym), Some((_, 0))) + } + + /// Resolve every keysym, overlay-binding the unresolvable ones. Returns one + /// `(keycode, level)` per input keysym (`(0, 0)` when it cannot be bound) plus whether the + /// keymap changed and must be re-applied — at most ONE swap per call, however many new + /// keysyms were bound. Slots whose keycode is in `pressed` are never recycled. + pub fn bind_many( + &mut self, + keysyms: &[u32], + pressed: &HashSet, + ) -> (Vec<(u32, u32)>, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, false, &mut changed)); + } + (out, changed) + } + + /// Like `bind_many` but only accepts level-0 resolutions: a keysym reachable in the + /// base solely behind a modifier (e.g. `A` behind Shift) is overlay-bound instead, so the + /// caller can inject it without synthesizing modifiers. Returns keycodes (0 = unbindable). + pub fn bind_many_plain(&mut self, keysyms: &[u32], pressed: &HashSet) -> (Vec, bool) { + let mut out = Vec::with_capacity(keysyms.len()); + let mut changed = false; + for &sym in keysyms { + out.push(self.bind_one(sym, pressed, true, &mut changed).0); + } + (out, changed) + } + + fn bind_one( + &mut self, + sym: u32, + pressed: &HashSet, + plain_only: bool, + changed: &mut bool, + ) -> (u32, u32) { + if sym == 0 { + return (0, 0); + } + if let Some(&(kc, level)) = self.base_map.get(&sym) { + if !plain_only || level == 0 { + return (kc, level); + } + } + if let Some(&slot) = self.by_sym.get(&sym) { + if let Some(at) = self.lru.iter().position(|&s| s == slot) { + self.lru.remove(at); + } + self.lru.push_back(slot); + return (self.overlay_first + slot as u32, 0); + } + let slot = if self.slots.len() < self.overlay_capacity { + self.slots.push(None); + self.slots.len() - 1 + } else { + match self.recycle_slot(pressed) { + Some(s) => s, + None => return (0, 0), + } + }; + if let Some(old) = self.slots[slot].replace(sym) { + self.by_sym.remove(&old); + } + self.by_sym.insert(sym, slot); + self.lru.push_back(slot); + *changed = true; + (self.overlay_first + slot as u32, 0) + } + + /// Oldest slot whose keycode is not currently held down; a held keycode must keep its + /// meaning until its release has been delivered. + fn recycle_slot(&mut self, pressed: &HashSet) -> Option { + let at = self + .lru + .iter() + .position(|&slot| !pressed.contains(&(self.overlay_first + slot as u32)))?; + self.lru.remove(at) + } + + /// The full seat keymap: the base text with every occupied overlay slot spliced into the + /// `xkb_keycodes` and `xkb_symbols` sections (and `maximum` raised to cover them). With no + /// overlays, the base text verbatim. + pub fn keymap_text(&self) -> String { + let occupied: Vec<(usize, u32)> = self + .slots + .iter() + .enumerate() + .filter_map(|(i, s)| s.map(|sym| (i, sym))) + .collect(); + if occupied.is_empty() { + return self.base_text.clone(); + } + let base = &self.base_text; + let Some(max_at) = base.find("maximum = ") else { + return self.base_text.clone(); + }; + let num_at = max_at + "maximum = ".len(); + let Some(num_len) = base[num_at..].find(';') else { + return self.base_text.clone(); + }; + let old_max: u32 = base[num_at..num_at + num_len].trim().parse().unwrap_or(255); + let need_max = self.overlay_first + occupied.last().map(|&(i, _)| i as u32).unwrap_or(0); + let mut text = String::with_capacity(base.len() + occupied.len() * 48); + text.push_str(&base[..num_at]); + text.push_str(&old_max.max(need_max).to_string()); + let rest = &base[num_at + num_len..]; + // First "};" after the maximum line closes xkb_keycodes. + let Some(kc_end) = rest.find("};") else { + return self.base_text.clone(); + }; + text.push_str(&rest[..kc_end]); + for &(i, _) in &occupied { + text.push_str(&format!("\t = {};\n", i, self.overlay_first + i as u32)); + } + let rest = &rest[kc_end..]; + let Some(close_at) = rest + .find("xkb_symbols") + .and_then(|sym_at| Self::section_close(rest, sym_at)) + else { + return self.base_text.clone(); + }; + text.push_str(&rest[..close_at]); + for &(i, sym) in &occupied { + text.push_str(&format!("\tkey {{ [ {:#x} ] }};\n", i, sym)); + } + text.push_str(&rest[close_at..]); + text + } + + /// Byte offset of the `}` closing the brace-block that starts at/after `from`. + fn section_close(text: &str, from: usize) -> Option { + let open = from + text[from..].find('{')?; + let mut depth = 0usize; + for (i, ch) in text[open..].char_indices() { + match ch { + '{' => depth += 1, + '}' => { + depth -= 1; + if depth == 0 { + return Some(open + i); + } + } + _ => {} + } + } + None + } +} + +#[cfg(test)] +mod tests { + //! Invariants: one `bind_many` call binds any number of new keysyms with a single + //! keymap change; base keysyms resolve without consuming overlay slots; a pressed + //! overlay keycode survives LRU pressure; the spliced keymap text compiles and + //! resolves the overlay keysyms at their assigned keycodes. + use super::*; + + fn us_base() -> String { + compile_rmlvo("", "", "us", "", "").expect("us keymap") + } + + fn policy() -> KeymapPolicy { + let mut p = KeymapPolicy::empty(); + p.rebuild_base(us_base()); + p + } + + #[test] + fn base_keysyms_resolve_without_overlay() { + let mut p = policy(); + // 'a' plain, 'A' shifted. + let (out, changed) = p.bind_many(&[0x61, 0x41], &HashSet::new()); + assert!(!changed); + assert_eq!(out[0].1, 0); + assert_eq!(out[1].0, out[0].0); + assert_eq!(out[1].1, 1); + } + + #[test] + fn batch_bind_is_one_swap_and_compiles() { + let mut p = policy(); + let syms: Vec = (0..30).map(|i| 0x1004E00 + i).collect(); + let (out, changed) = p.bind_many(&syms, &HashSet::new()); + assert!(changed); + let (_, changed_again) = p.bind_many(&syms, &HashSet::new()); + assert!(!changed_again, "re-binding bound keysyms must not swap"); + let text = p.keymap_text(); + let km = compile_keymap(&text).expect("overlay keymap compiles"); + for (i, &(kc, level)) in out.iter().enumerate() { + assert_eq!(level, 0); + let got = km.key_get_syms_by_level(xkb::Keycode::new(kc), 0, 0); + assert_eq!(got.len(), 1, "keycode {kc} has one sym"); + assert_eq!(got[0].raw(), syms[i]); + } + } + + #[test] + fn pressed_keycode_is_never_recycled() { + let mut p = policy(); + let syms: Vec = (0..OVERLAY_CAPACITY as u32).map(|i| 0x1005000 + i).collect(); + let (out, _) = p.bind_many(&syms, &HashSet::new()); + let held_kc = out[0].0; + let held_sym = syms[0]; + let pressed: HashSet = [held_kc].into_iter().collect(); + // Force full recycling pressure past capacity. + let extra: Vec = (0..8).map(|i| 0x1006000 + i).collect(); + let (extra_out, changed) = p.bind_many(&extra, &pressed); + assert!(changed); + for &(kc, _) in &extra_out { + assert_ne!(kc, held_kc, "held keycode must not be rebound"); + } + assert_eq!(p.resolve(held_sym), Some((held_kc, 0))); + } + + #[test] + fn sub256_overlay_range_overrides_base_keycode_names() { + // The virtual-keyboard client's range collides with keycodes the base + // already names (…); the spliced definitions must win so overlay + // keysyms resolve at their assigned keycodes. + let mut p = KeymapPolicy::with_overlay_range(150, 255); + p.rebuild_base(us_base()); + let (out, changed) = p.bind_many_plain(&[0x1004E2D, 0x61], &HashSet::new()); + assert!(changed); + assert_eq!(out[0], 150); + let km = compile_keymap(&p.keymap_text()).expect("sub-256 overlay keymap compiles"); + let got = km.key_get_syms_by_level(xkb::Keycode::new(out[0]), 0, 0); + assert_eq!(got.len(), 1); + assert_eq!(got[0].raw(), 0x1004E2D); + // 'a' resolves plain in the base without consuming a slot. + assert!(out[1] < 150); + assert_eq!(out[1], p.resolve(0x61).unwrap().0); + } + + #[test] + fn rebuild_base_keeps_overlay_assignments() { + let mut p = policy(); + let (out, _) = p.bind_many(&[0x1004E2D], &HashSet::new()); + let de = compile_rmlvo("", "", "de", "", "").expect("de keymap"); + p.rebuild_base(de); + assert_eq!(p.resolve(0x1004E2D), Some((out[0].0, 0))); + // udiaeresis resolves in the German base without an overlay. + let (u_out, changed) = p.bind_many(&[0xFC], &HashSet::new()); + assert!(!changed); + assert_eq!(u_out[0].1, 0); + assert!(compile_keymap(&p.keymap_text()).is_some()); + } +} diff --git a/pixelflux/src/wayland/mod.rs b/pixelflux/src/wayland/mod.rs index f2d0679..b9e94a3 100644 --- a/pixelflux/src/wayland/mod.rs +++ b/pixelflux/src/wayland/mod.rs @@ -11,5 +11,15 @@ /// Headless Smithay compositor, protocol handlers, and input routing. pub mod frontend; -/// Wayland cursor shape to PNG resolution. +/// Wayland cursor shape to PNG resolution and the cursor delivery worker. pub mod cursor; +/// Seat keymap ownership: base layout plus batched overlay keysym binding. +pub mod keymap; +/// Virtual-keyboard client for typing into a nested app compositor's socket. +pub mod vkclient; +/// Shared plumbing for outbound Wayland client connections. +pub mod wlclient; +/// Data-control clipboard client bridging a nested app compositor's selection. +pub mod dcclient; +/// Host-capture mode: capture/inject as a client of an external compositor. +pub mod host; diff --git a/pixelflux/src/wayland/vkclient.rs b/pixelflux/src/wayland/vkclient.rs new file mode 100644 index 0000000..9b2a5fa --- /dev/null +++ b/pixelflux/src/wayland/vkclient.rs @@ -0,0 +1,157 @@ +//! `zwp_virtual_keyboard_v1` client for typing Unicode text into ANOTHER Wayland +//! compositor's socket. +//! +//! pixelflux is normally the compositor, but in a nested deployment (a labwc/kwin +//! session running as a client of pixelflux) the apps live on that inner +//! compositor's socket, and keys injected into pixelflux's own seat resolve +//! against pixelflux's keymap — an overlay the inner compositor never sees. So +//! text is typed here as a client of whichever compositor the apps live under — +//! by Computer-Use actions and by selkies over the `type_text_wayland` ABI — +//! reusing the seat's [`KeymapPolicy`] over a US base: base-reachable characters +//! press their ordinary keycodes, everything else is overlay-bound, one upload +//! per batch. One-shot (connect, type, disconnect): a call carries a whole commit +//! of text, and a fresh connection leaves no stale-socket state to manage. +//! Blocking, off the compositor thread, with every round-trip deadline-bounded so +//! a wedged compositor cannot hang the caller forever. + +use std::collections::HashSet; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::sync::OnceLock; +use std::time::Duration; + +use wayland_client::protocol::{wl_registry, wl_seat}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, QueueHandle}; +use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{ + zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1, + zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1, +}; + +use crate::wayland::keymap::{compile_rmlvo, keysym_for_char, KeymapPolicy}; +use crate::wayland::wlclient::{bounded_roundtrip, impl_sync_callback, memfd_with, SyncState}; + +/// Overlay keycodes stay under the X11 255 ceiling so XWayland apps under the app +/// compositor can still receive them (the seat's own overlay sits above 255). +const OVERLAY_FIRST_XKB: u32 = 150; +const OVERLAY_LAST_XKB: u32 = 255; +const OVERLAY_SLOTS: usize = (OVERLAY_LAST_XKB - OVERLAY_FIRST_XKB + 1) as usize; +/// wl_keyboard / zwp_virtual_keyboard key events carry evdev codes (xkb - 8). +const EVDEV_OFFSET: u32 = 8; +const KEYMAP_FORMAT_XKB_V1: u32 = 1; + +#[derive(Default)] +struct Globals { + seat: Option, + manager: Option, + sync_done: bool, +} + +impl SyncState for Globals { + fn sync_done_mut(&mut self) -> &mut bool { + &mut self.sync_done + } +} +impl_sync_callback!(Globals); + +impl Dispatch for Globals { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { name, interface, .. } = event { + // Version 1 of each suffices: the seat is only the manager argument. + match interface.as_str() { + "wl_seat" if state.seat.is_none() => { + state.seat = Some(registry.bind(name, 1, qh, ())); + } + "zwp_virtual_keyboard_manager_v1" if state.manager.is_none() => { + state.manager = Some(registry.bind(name, 1, qh, ())); + } + _ => {} + } + } + } +} + +delegate_noop!(Globals: ignore wl_seat::WlSeat); +delegate_noop!(Globals: ZwpVirtualKeyboardManagerV1); +delegate_noop!(Globals: ZwpVirtualKeyboardV1); + +/// The US base keymap text, compiled once per process: selkies types per text +/// commit, and xkbcommon compilation is the expensive part of a call. +pub(crate) fn us_base_text() -> Option<&'static str> { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| compile_rmlvo("", "", "us", "", "")).as_deref() +} + +fn upload_keymap( + vk: &ZwpVirtualKeyboardV1, + queue: &mut EventQueue, + text: &str, +) -> Result<(), String> { + let mut data = text.as_bytes().to_vec(); + data.push(0); // compositors parse the mapping as a NUL-terminated string + let fd = memfd_with(&data)?; + vk.keymap(KEYMAP_FORMAT_XKB_V1, fd.as_fd(), data.len() as u32); + queue.flush().map_err(|e| format!("flush keymap: {e}")) +} + +/// One-shot: connect to `socket_path`, type `text` in order, disconnect. Codepoints +/// with no keysym are skipped. Blocking; call off the compositor's calloop thread. +pub fn type_text_to(socket_path: &str, text: &str) -> Result<(), String> { + let stream = + UnixStream::connect(socket_path).map_err(|e| format!("connect {socket_path}: {e}"))?; + let conn = Connection::from_socket(stream).map_err(|e| format!("wayland setup: {e}"))?; + let mut queue = conn.new_event_queue(); + let qh = queue.handle(); + let _registry = conn.display().get_registry(&qh, ()); + let mut state = Globals::default(); + bounded_roundtrip(&conn, &mut queue, &mut state)?; + let seat = state.seat.take().ok_or("app compositor advertises no wl_seat")?; + let manager = state + .manager + .take() + .ok_or("app compositor does not advertise zwp_virtual_keyboard_manager_v1")?; + let vk = manager.create_virtual_keyboard(&seat, &qh, ()); + // Surfaces an "unauthorized" bind error before the first keymap upload. + bounded_roundtrip(&conn, &mut queue, &mut state)?; + + let mut policy = KeymapPolicy::with_overlay_range(OVERLAY_FIRST_XKB, OVERLAY_LAST_XKB); + policy.rebuild_base( + us_base_text().ok_or("us base keymap failed to compile")?.to_string(), + ); + let syms: Vec = text.chars().map(keysym_for_char).filter(|&s| s != 0).collect(); + let none = HashSet::new(); + let mut uploaded = false; + // Chunk bound: at most OVERLAY_SLOTS keysyms per bind call, so a batch can + // never recycle a slot it assigned earlier in the same batch. + for chunk in syms.chunks(OVERLAY_SLOTS) { + let (keycodes, changed) = policy.bind_many_plain(chunk, &none); + // The protocol requires a keymap before the first key event even when the + // whole text resolves in the base. + if changed || !uploaded { + upload_keymap(&vk, &mut queue, &policy.keymap_text())?; + bounded_roundtrip(&conn, &mut queue, &mut state)?; + std::thread::sleep(Duration::from_millis(10)); + uploaded = true; + } + for &kc in &keycodes { + if kc < EVDEV_OFFSET { + continue; // unbindable keysym + } + for pressed in [1u32, 0] { + vk.key(0, kc - EVDEV_OFFSET, pressed); + queue.flush().map_err(|e| format!("flush key: {e}"))?; + std::thread::sleep(Duration::from_millis(2)); + } + } + } + bounded_roundtrip(&conn, &mut queue, &mut state)?; + vk.destroy(); + let _ = queue.flush(); + Ok(()) +} diff --git a/pixelflux/src/wayland/wlclient.rs b/pixelflux/src/wayland/wlclient.rs new file mode 100644 index 0000000..7979c96 --- /dev/null +++ b/pixelflux/src/wayland/wlclient.rs @@ -0,0 +1,254 @@ +//! Shared plumbing for pixelflux's outbound Wayland CLIENT connections (the +//! virtual-keyboard typer and the data-control clipboard bridge, both talking to +//! a nested app compositor): socket resolution, deadline-bounded round-trips so a +//! wedged compositor turns into an error instead of a hang, and fd read/write +//! helpers for clipboard pipes. + +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::time::{Duration, Instant}; + +use wayland_client::protocol::wl_callback; +use wayland_client::{Connection, Dispatch, EventQueue}; + +pub(crate) const IO_TIMEOUT: Duration = Duration::from_secs(5); + +/// Absolute socket path for a Wayland display name (absolute paths pass through, +/// names join XDG_RUNTIME_DIR). +pub(crate) fn socket_path(name: &str) -> Option { + if name.starts_with('/') { + return Some(name.to_string()); + } + let rt = std::env::var("XDG_RUNTIME_DIR").ok()?; + Some(format!("{}/{}", rt.trim_end_matches('/'), name)) +} + +/// Implemented by every client state so [`bounded_roundtrip`] can flag the sync +/// callback's completion; pair with [`impl_sync_callback`]. +pub(crate) trait SyncState { + fn sync_done_mut(&mut self) -> &mut bool; +} + +/// `Dispatch` for a [`SyncState`] type (a blanket impl would violate +/// the orphan rule, so each state stamps its own). +macro_rules! impl_sync_callback { + ($t:ty) => { + impl wayland_client::Dispatch + for $t + { + fn event( + state: &mut Self, + _: &wayland_client::protocol::wl_callback::WlCallback, + event: wayland_client::protocol::wl_callback::Event, + _: &(), + _: &wayland_client::Connection, + _: &wayland_client::QueueHandle, + ) { + if let wayland_client::protocol::wl_callback::Event::Done { .. } = event { + *crate::wayland::wlclient::SyncState::sync_done_mut(state) = true; + } + } + } + }; +} +pub(crate) use impl_sync_callback; + +/// Block until `fd` is readable or `timeout` passes (false = timed out). +pub(crate) fn wait_readable(fd: RawFd, timeout: Duration) -> Result { + loop { + let mut pfd = libc::pollfd { fd, events: libc::POLLIN, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, timeout.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + return Ok(n > 0); + } +} + +/// Poll two fds for readability at once (wayland socket + a wake pipe); returns +/// `(a_readable, b_readable)`, both false on timeout. Hangup counts as readable +/// so a closed wake pipe unblocks its waiter. +pub(crate) fn wait_readable2( + a: RawFd, + b: RawFd, + timeout: Option, +) -> Result<(bool, bool), String> { + loop { + let mut pfds = [ + libc::pollfd { fd: a, events: libc::POLLIN, revents: 0 }, + libc::pollfd { fd: b, events: libc::POLLIN, revents: 0 }, + ]; + let ms = timeout.map(|t| t.as_millis().max(1) as libc::c_int).unwrap_or(-1); + let n = unsafe { libc::poll(pfds.as_mut_ptr(), 2, ms) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + let hit = |r: libc::c_short| r & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0; + return Ok((hit(pfds[0].revents), hit(pfds[1].revents))); + } +} + +/// Drain every pending byte from a wake pipe without blocking. +pub(crate) fn drain_pipe(fd: RawFd) { + let mut buf = [0u8; 64]; + loop { + let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; + if n <= 0 { + return; + } + } +} + +/// One wake byte, non-blocking; a full pipe already guarantees a pending wake. +pub(crate) fn wake_write(fd: RawFd) { + let b = [1u8]; + unsafe { libc::write(fd, b.as_ptr() as *const libc::c_void, 1) }; +} + +/// `EventQueue::roundtrip` with a deadline: a wedged compositor becomes an error +/// instead of hanging the calling thread (and everything queued behind it). +pub(crate) fn bounded_roundtrip( + conn: &Connection, + queue: &mut EventQueue, + state: &mut S, +) -> Result<(), String> +where + S: SyncState + Dispatch + 'static, +{ + *state.sync_done_mut() = false; + let _cb = conn.display().sync(&queue.handle(), ()); + let deadline = Instant::now() + IO_TIMEOUT; + loop { + queue + .dispatch_pending(state) + .map_err(|e| format!("dispatch: {e}"))?; + if *state.sync_done_mut() { + return Ok(()); + } + queue.flush().map_err(|e| format!("flush: {e}"))?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or("compositor round-trip timed out")?; + let Some(guard) = conn.prepare_read() else { + continue; + }; + if !wait_readable(guard.connection_fd().as_raw_fd(), remaining)? { + return Err("compositor round-trip timed out".into()); + } + guard.read().map_err(|e| format!("read: {e}"))?; + } +} + +/// Anonymous CLOEXEC memfd holding `data` (keymap uploads, shm-style payloads). +pub(crate) fn memfd_with(data: &[u8]) -> Result { + let name = b"pixelflux-wl\0"; + let fd = + unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) }; + if fd < 0 { + return Err(format!("memfd_create: {}", std::io::Error::last_os_error())); + } + let owned = unsafe { OwnedFd::from_raw_fd(fd) }; + let mut written = 0; + while written < data.len() { + let n = unsafe { + libc::write( + owned.as_raw_fd(), + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if n < 0 { + return Err(format!("write memfd: {}", std::io::Error::last_os_error())); + } + written += n as usize; + } + Ok(owned) +} + +/// CLOEXEC pipe as (read end, write end). +pub(crate) fn pipe_cloexec() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + +/// Non-blocking CLOEXEC pipe for wake signalling (read end, write end). +pub(crate) fn wake_pipe() -> Result<(OwnedFd, OwnedFd), String> { + let mut fds = [0i32; 2]; + if unsafe { libc::pipe2(fds.as_mut_ptr(), libc::O_CLOEXEC | libc::O_NONBLOCK) } < 0 { + return Err(format!("pipe2: {}", std::io::Error::last_os_error())); + } + Ok(unsafe { (OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1])) }) +} + +/// Read `fd` to EOF. The deadline is per-chunk (idle), so a large transfer that +/// keeps flowing is never cut off while a stalled writer still errors out. +pub(crate) fn read_fd_to_end(fd: &OwnedFd, idle: Duration) -> Result, String> { + let mut out = Vec::new(); + let mut chunk = [0u8; 65536]; + loop { + if !wait_readable(fd.as_raw_fd(), idle)? { + return Err("clipboard source stalled".into()); + } + let n = unsafe { + libc::read(fd.as_raw_fd(), chunk.as_mut_ptr() as *mut libc::c_void, chunk.len()) + }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("read: {err}")); + } + if n == 0 { + return Ok(out); + } + out.extend_from_slice(&chunk[..n as usize]); + } +} + +/// Write all of `data` to `fd`, tolerating a slow reader up to `idle` per chunk. +/// EPIPE is success-shaped: the paster stopped reading, which is its right. +pub(crate) fn write_fd_all(fd: &OwnedFd, data: &[u8], idle: Duration) -> Result<(), String> { + let mut written = 0; + while written < data.len() { + let mut pfd = libc::pollfd { fd: fd.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; + let n = unsafe { libc::poll(&mut pfd, 1, idle.as_millis().max(1) as libc::c_int) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EINTR) { + continue; + } + return Err(format!("poll: {err}")); + } + if n == 0 { + return Err("clipboard reader stalled".into()); + } + let w = unsafe { + libc::write( + fd.as_raw_fd(), + data[written..].as_ptr() as *const libc::c_void, + data.len() - written, + ) + }; + if w < 0 { + let err = std::io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EINTR) => continue, + Some(libc::EPIPE) => return Ok(()), + _ => return Err(format!("write: {err}")), + } + } + written += w as usize; + } + Ok(()) +} diff --git a/pixelflux/src/x11/computer_use.rs b/pixelflux/src/x11/computer_use.rs new file mode 100644 index 0000000..52f4d11 --- /dev/null +++ b/pixelflux/src/x11/computer_use.rs @@ -0,0 +1,503 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +//! X11 backend for the Computer Use HTTP API: XTEST injection and one-shot root screenshots +//! over a private x11rb connection, so the agent can drive an X session with no active capture +//! and no shared state with any streaming capture thread (the X server itself serializes). +//! +//! Keysyms the active layout cannot produce (after base+AltGr resolution, which stays the +//! preferred path) are typed through a transient remap: under `XGrabServer` the keymap is +//! re-fetched, the needed keysyms are bound onto all-NoSymbol spare keycodes with ONE +//! `XChangeKeyboardMapping`, and the key sequence is injected; after a settle window (which +//! lets the focused client re-fetch the bound map — see `type_with_transient_binds`) one +//! conditional `XChangeKeyboardMapping` under a second grab puts the spares back — two +//! MappingNotify broadcasts per action. This coexists with a selkies session's own +//! spare-keycode overlay allocator on the same server: spares here are chosen DESCENDING +//! from the top of the keycode range while selkies allocates ASCENDING, each grab re-fetches +//! the keymap so every binding selkies already made is respected, the restore clears only +//! keycodes still carrying our content, and selkies' foreign-change invalidation (fired by +//! each MappingNotify) is self-healing because its held keys live on BOUND (non-NoSymbol) +//! keycodes an all-NoSymbol spare can never steal. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::thread; +use std::time::Duration; + +use x11rb::connection::Connection; +use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; +use x11rb::protocol::xproto::{ + ConnectionExt as XprotoExt, ImageFormat, BUTTON_PRESS_EVENT, BUTTON_RELEASE_EVENT, + KEY_PRESS_EVENT, KEY_RELEASE_EVENT, MOTION_NOTIFY_EVENT, +}; +use x11rb::protocol::xtest::ConnectionExt as XtestExt; +use x11rb::rust_connection::RustConnection; + +use crate::computer_use::{encode_png_rgba, CuBackend, CuButton}; + +/// One wheel "click" of scroll per unit of CU `scroll_amount`, capped so a hostile amount +/// cannot flood the server with press/release pairs. +const MAX_SCROLL_CLICKS: i32 = 100; + +const KEYSYM_ISO_LEVEL3_SHIFT: u32 = 0xfe03; +const KEYSYM_MODE_SWITCH: u32 = 0xff7e; + +/// Reverse view of the server's current keymap. +struct ServerKeymap { + /// keysym -> (keycode, shift level); level bit 0 = Shift, bit 1 = AltGr, lowest + /// level wins across ALL keys so a keysym reachable unshifted never resolves to a + /// modified position. + by_sym: HashMap, + /// Keycode carrying `ISO_Level3_Shift` (or, failing that, `Mode_switch`) in the core + /// map — the key synthesized around AltGr-level hits. 0 when the layout has neither, + /// in which case no AltGr levels are offered at all. + altgr_keycode: u32, +} + +pub struct CuX11Backend { + conn: RustConnection, + root: u32, + has_xfixes: bool, + /// Reverse map of the SERVER keymap, built lazily from `GetKeyboardMapping` and kept + /// for this backend's lifetime — one HTTP request (the backend is per-request), so a + /// `setxkbmap` switch is seen by the next request. + reverse_keymap: RefCell>, +} + +impl CuX11Backend { + /// Connect to the X server named by `DISPLAY` and negotiate XTEST (required for any + /// injection). XFixes is optional and only gates cursor compositing in screenshots. + pub fn connect() -> Result { + let (conn, screen_num) = + x11rb::connect(None).map_err(|e| format!("X11 connect failed: {e}"))?; + let root = conn.setup().roots[screen_num].root; + conn.xtest_get_version(2, 2) + .map_err(|e| format!("xtest_get_version: {e}"))? + .reply() + .map_err(|e| format!("XTEST unavailable: {e}"))?; + let has_xfixes = conn + .xfixes_query_version(5, 0) + .ok() + .and_then(|c| c.reply().ok()) + .is_some(); + Ok(Self { conn, root, has_xfixes, reverse_keymap: RefCell::new(None) }) + } + + /// Build the reverse view of the server's current keymap from `GetKeyboardMapping`. + /// + /// Column -> (group, level) convention, as observed on XKB-compat servers (Xvfb/Xorg + /// under `setxkbmap de`: keycode 24 = `q Q q Q at Greek_OMEGA at`): columns 0/1 are + /// group-1 plain/Shift, columns 2/3 mirror them as core group 2, and columns 4/5 + /// carry group-1 levels 3/4 (AltGr, Shift+AltGr). Columns 4/5 are consulted only + /// when the map carries a level-3 modifier key to synthesize; columns 2/3 (group 2) + /// never are. + fn build_reverse_keymap(&self) -> ServerKeymap { + let mut km = ServerKeymap { by_sym: HashMap::new(), altgr_keycode: 0 }; + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let Some(reply) = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .ok() + .and_then(|c| c.reply().ok()) + else { + return km; + }; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return km; + } + let keycode_of = |wanted: u32| { + reply + .keysyms + .chunks_exact(per) + .position(|syms| syms.contains(&wanted)) + .map(|i| lo as u32 + i as u32) + }; + km.altgr_keycode = keycode_of(KEYSYM_ISO_LEVEL3_SHIFT) + .or_else(|| keycode_of(KEYSYM_MODE_SWITCH)) + .unwrap_or(0); + // Ascending level order so lower levels win; level bit 0 = Shift, bit 1 = AltGr. + let columns: [(usize, u32); 4] = [(0, 0), (1, 1), (4, 2), (5, 3)]; + for (col, level) in columns { + if col >= per || (level & 2 != 0 && km.altgr_keycode == 0) { + continue; + } + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate() { + let sym = syms[col]; + if sym != 0 { + km.by_sym.entry(sym).or_insert((lo as u32 + i as u32, level)); + } + } + } + km + } + + /// Fire one XTEST fake event and flush so it reaches the server before the action + /// layer's inter-event pacing sleep, matching the immediacy of real input. + fn fake_input(&self, kind: u8, detail: u8, root: u32, x: i16, y: i16) { + let _ = self + .conn + .xtest_fake_input(kind, detail, x11rb::CURRENT_TIME, root, x, y, 0) + .map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } + + fn root_geometry(&self) -> Result<(u16, u16), String> { + let geo = self + .conn + .get_geometry(self.root) + .map_err(|e| format!("get_geometry: {e}"))? + .reply() + .map_err(|e| format!("get_geometry reply: {e}"))?; + Ok((geo.width, geo.height)) + } + + /// One server round trip, forcing everything already sent on this connection to be + /// processed before the next request is issued (the XSync idiom). + fn sync(&self) -> Result<(), String> { + self.conn + .get_input_focus() + .map_err(|e| format!("sync: {e}"))? + .reply() + .map_err(|e| format!("sync reply: {e}"))?; + Ok(()) + } + + /// Bind `keysyms` onto spare keycodes and run `seq` with the keysym -> keycode map, + /// choose/bind/inject atomically under a server grab; then, after a settle window, + /// restore the spares under a second grab. The settle exists because clients + /// translate a keycode by RE-FETCHING the keymap when they process the bind's + /// MappingNotify — a fetch the grab itself blocks — so a restore issued inside the + /// first grab would be what they read back and every transient key would translate + /// to nothing (observed with xterm). Once `seq` has run, every failure is logged and + /// swallowed so the caller never re-runs the sequence. + fn type_with_transient_binds( + &self, + keysyms: &[u32], + seq: &mut dyn FnMut(&HashMap), + ) -> Result<(), String> { + let setup = self.conn.setup(); + let (lo, hi) = (setup.min_keycode, setup.max_keycode); + let mut chosen: Vec = Vec::with_capacity(keysyms.len()); + let (span_lo, count, per, bound_syms) = { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + // Re-fetched UNDER the grab: a spare keycode any other client bound before + // the grab is visible as bound here and never chosen. + let reply = self + .conn + .get_keyboard_mapping(lo, hi - lo + 1) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let per = reply.keysyms_per_keycode as usize; + if per == 0 { + return Err("empty keymap".to_string()); + } + // Spares are all-NoSymbol keycodes taken DESCENDING from the top of the + // range. selkies' overlay allocator scans ASCENDING, so the two only meet + // when nearly every spare on the server is taken; and any held key + // (selkies' overlay binds included) lives on a BOUND, non-NoSymbol keycode, + // so an all-NoSymbol spare can never steal a key that is currently down. + for (i, syms) in reply.keysyms.chunks_exact(per).enumerate().rev() { + if syms.iter().all(|&s| s == 0) { + chosen.push(lo as u32 + i as u32); + if chosen.len() == keysyms.len() { + break; + } + } + } + if chosen.len() < keysyms.len() { + return Err(format!( + "only {} spare keycodes for {} unresolved keysyms", + chosen.len(), + keysyms.len() + )); + } + // ONE ChangeKeyboardMapping over the span from the lowest chosen spare + // upward. Every all-NoSymbol keycode above the lowest chosen one was itself + // chosen, so the span's other keycodes are bound ones, rewritten with their + // existing content (a content no-op). Each transient key carries its keysym + // at the plain and Shift levels so a stray held Shift cannot change what it + // types. + let span_lo = *chosen.last().unwrap(); + let span_hi = chosen[0]; + let base = ((span_lo - lo as u32) as usize) * per; + let end = ((span_hi - lo as u32) as usize + 1) * per; + let mut bound_syms = reply.keysyms[base..end].to_vec(); + let mut map = HashMap::new(); + for (&sym, &kc) in keysyms.iter().zip(chosen.iter()) { + let off = ((kc - span_lo) as usize) * per; + bound_syms[off] = sym; + if per > 1 { + bound_syms[off + 1] = sym; + } + map.insert(sym, kc); + } + let count = (span_hi - span_lo + 1) as u8; + self.conn + .change_keyboard_mapping(count, span_lo as u8, per as u8, &bound_syms) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + // The binding must be live server-side before the first fake press resolves + // against it. + self.sync()?; + seq(&map); + (span_lo, count, per, bound_syms) + // Guard drops: ungrab + flush, releasing clients to process the injected + // events against the still-live bindings. + }; + // Settle: clients consume the queued MappingNotify + key events and re-fetch the + // BOUND map before the spares disappear again. + thread::sleep(TRANSIENT_BIND_SETTLE); + if let Err(e) = self.restore_transient_binds(span_lo, count, per, &bound_syms, &chosen) { + eprintln!("[ComputerUse] transient keysym restore failed: {e}"); + } + Ok(()) + } + + /// Return the transiently bound spares to all-NoSymbol with ONE conditional + /// `ChangeKeyboardMapping` under its own grab. The span is re-fetched under that + /// grab and only spares still carrying OUR content are cleared; a keycode another + /// client (selkies' allocator) claimed during the settle window keeps that client's + /// content — this restore can never clobber a foreign binding. + fn restore_transient_binds( + &self, + span_lo: u32, + count: u8, + per: usize, + bound_syms: &[u32], + chosen: &[u32], + ) -> Result<(), String> { + self.conn + .grab_server() + .map_err(|e| format!("grab_server: {e}"))?; + let _guard = ServerGrabGuard { conn: &self.conn }; + let reply = self + .conn + .get_keyboard_mapping(span_lo as u8, count) + .map_err(|e| format!("get_keyboard_mapping: {e}"))? + .reply() + .map_err(|e| format!("get_keyboard_mapping reply: {e}"))?; + let cur_per = reply.keysyms_per_keycode as usize; + if cur_per == 0 || per == 0 { + return Err("empty keymap".to_string()); + } + let mut restore = reply.keysyms.clone(); + let mut changed = false; + for &kc in chosen { + let sym = bound_syms[((kc - span_lo) as usize) * per]; + let cur = &mut restore[((kc - span_lo) as usize) * cur_per..][..cur_per]; + // Still ours when every populated level carries OUR keysym: the server's XKB + // integration mirrors a core single-group binding into the group-2 columns, + // so the refetch shows `sym` at more levels than the bind wrote. + let still_ours = + cur.iter().any(|&s| s == sym) && cur.iter().all(|&s| s == 0 || s == sym); + if still_ours { + cur.fill(0); + changed = true; + } + } + if changed { + self.conn + .change_keyboard_mapping(count, span_lo as u8, cur_per as u8, &restore) + .map_err(|e| format!("change_keyboard_mapping: {e}"))? + .check() + .map_err(|e| format!("change_keyboard_mapping check: {e}"))?; + } + Ok(()) + } +} + +/// How long transient binds outlive the injected key events before being restored: the +/// focused client has to wake up, see the bind's MappingNotify, and re-fetch the keymap +/// while the bindings are still live, or the presses translate against the restored map +/// and type nothing. +const TRANSIENT_BIND_SETTLE: Duration = Duration::from_millis(50); + +/// RAII server grab release: the grab is dropped (and the request flushed) on every exit +/// path, early error returns and panics included — a leaked server grab freezes every +/// client on the display. +struct ServerGrabGuard<'a> { + conn: &'a RustConnection, +} + +impl Drop for ServerGrabGuard<'_> { + fn drop(&mut self) { + let _ = self.conn.ungrab_server().map(|c| c.ignore_error()); + let _ = self.conn.flush(); + } +} + +impl Drop for CuX11Backend { + /// Closing the connection can race the server's client teardown against still-buffered + /// fake-input requests (observed as lost button releases); one round trip forces the + /// server to consume everything sent on this connection before it goes away. + fn drop(&mut self) { + if let Ok(cookie) = self.conn.get_input_focus() { + let _ = cookie.reply(); + } + } +} + +impl CuBackend for CuX11Backend { + fn name(&self) -> &'static str { + "x11" + } + + fn fb_size(&self) -> Result<(i32, i32), String> { + let (w, h) = self.root_geometry()?; + Ok((w as i32, h as i32)) + } + + fn key(&self, scancode: u32, pressed: bool) { + if scancode > u8::MAX as u32 { + return; + } + let kind = if pressed { KEY_PRESS_EVENT } else { KEY_RELEASE_EVENT }; + self.fake_input(kind, scancode as u8, x11rb::NONE, 0, 0); + } + + fn mouse_move(&self, x: f64, y: f64) { + // detail = 0 makes the motion absolute in root coordinates. + self.fake_input( + MOTION_NOTIFY_EVENT, + 0, + self.root, + x.round() as i16, + y.round() as i16, + ); + } + + fn button(&self, btn: CuButton, pressed: bool) { + let detail = match btn { + CuButton::Left => 1, + CuButton::Middle => 2, + CuButton::Right => 3, + }; + let kind = if pressed { BUTTON_PRESS_EVENT } else { BUTTON_RELEASE_EVENT }; + self.fake_input(kind, detail, x11rb::NONE, 0, 0); + } + + fn scroll(&self, dx: f64, dy: f64) { + // X has no smooth axis over XTEST: a scroll is N discrete clicks of the wheel + // buttons (4 = up, 5 = down, 6 = left, 7 = right), one press/release pair each. + let emit = |button: u8, clicks: i32| { + for _ in 0..clicks.min(MAX_SCROLL_CLICKS) { + self.fake_input(BUTTON_PRESS_EVENT, button, x11rb::NONE, 0, 0); + self.fake_input(BUTTON_RELEASE_EVENT, button, x11rb::NONE, 0, 0); + } + }; + let vy = dy.round() as i32; + let vx = dx.round() as i32; + if vy != 0 { + emit(if vy < 0 { 4 } else { 5 }, vy.abs()); + } + if vx != 0 { + emit(if vx < 0 { 6 } else { 7 }, vx.abs()); + } + } + + fn screenshot_png(&self, display: u32) -> Result, String> { + // One X server, one root: only display 0 exists on this backend. + if display != 0 { + return Err(format!("Unknown display: {display}")); + } + let (w, h) = self.root_geometry()?; + let img = self + .conn + .get_image(ImageFormat::Z_PIXMAP, self.root, 0, 0, w, h, !0u32) + .map_err(|e| format!("get_image: {e}"))? + .reply() + .map_err(|e| format!("get_image reply: {e}"))?; + let mut data = img.data; + let expected = w as usize * h as usize * 4; + if data.len() != expected { + return Err(format!( + "unexpected image size {} for {}x{} (only 32-bpp roots are supported)", + data.len(), w, h + )); + } + // The agent needs to see the pointer; the stream's cursor settings do not apply here. + if self.has_xfixes { + if let Some(c) = self + .conn + .xfixes_get_cursor_image() + .ok() + .and_then(|c| c.reply().ok()) + { + if c.width > 0 && c.height > 0 { + let (img_x, img_y) = + super::cursor_image_origin(c.x, c.y, c.xhot, c.yhot, 0, 0); + super::overlay_cursor( + &mut data, + w as usize * 4, + w as i32, + h as i32, + c.width as i32, + c.height as i32, + &c.cursor_image, + img_x, + img_y, + ); + } + } + } + // The grab is BGRX; the padding byte is undefined for depth-24 roots, so alpha is + // forced opaque or the PNG would come out transparent. + for px in data.chunks_exact_mut(4) { + px.swap(0, 2); + px[3] = 0xFF; + } + encode_png_rgba(&data, w as u32, h as u32) + } + + fn cursor_pos(&self) -> Result<(f64, f64), String> { + let ptr = self + .conn + .query_pointer(self.root) + .map_err(|e| format!("query_pointer: {e}"))? + .reply() + .map_err(|e| format!("query_pointer reply: {e}"))?; + Ok((ptr.root_x as f64, ptr.root_y as f64)) + } + + fn resolve_keysyms(&self, keysyms: &[u32]) -> Vec<(u32, u32)> { + let mut cached = self.reverse_keymap.borrow_mut(); + let km = cached.get_or_insert_with(|| self.build_reverse_keymap()); + keysyms + .iter() + .map(|sym| km.by_sym.get(sym).copied().unwrap_or((0, 0))) + .collect() + } + + fn altgr_keycode(&self) -> u32 { + let mut cached = self.reverse_keymap.borrow_mut(); + cached.get_or_insert_with(|| self.build_reverse_keymap()).altgr_keycode + } + + fn with_transient_keysyms(&self, keysyms: &[u32], seq: &mut dyn FnMut(&HashMap)) { + // Dedup defensively; a duplicate would burn a spare keycode for nothing. + let mut unique: Vec = Vec::with_capacity(keysyms.len()); + for &s in keysyms { + if s != 0 && !unique.contains(&s) { + unique.push(s); + } + } + if unique.is_empty() { + seq(&HashMap::new()); + return; + } + if let Err(e) = self.type_with_transient_binds(&unique, seq) { + eprintln!("[ComputerUse] transient keysym bind failed ({e}); typing without it"); + seq(&HashMap::new()); + } + } +} diff --git a/pixelflux/src/x11/cursor.rs b/pixelflux/src/x11/cursor.rs index 394cbe0..970f01a 100644 --- a/pixelflux/src/x11/cursor.rs +++ b/pixelflux/src/x11/cursor.rs @@ -210,9 +210,18 @@ fn monitor_thread(stop: Arc, wake_win: Arc) { } let mut last: Option = fetch_payload(&conn); deliver(last.as_ref()); - // A registration that raced setup may have requested a replay before the wake window - // existed; the current cursor was just delivered, so the request is satisfied. - REPLAY.store(false, Ordering::Release); + // A registration that raced setup may have requested a replay before the wake + // window existed (its wake was skipped). The deliver above satisfied it only if + // the callback was already visible and the fetch produced a payload, so consume + // the flag and re-deliver rather than assume: an unconditional clear leaves that + // callback cursor-less until the next real cursor change. From here on the wake + // window exists, so later requests always reach the loop below. + if REPLAY.swap(false, Ordering::AcqRel) { + if last.is_none() { + last = fetch_payload(&conn); + } + deliver(last.as_ref()); + } loop { let event = match conn.wait_for_event() { Ok(ev) => ev, @@ -315,7 +324,9 @@ fn deliver(payload: Option<&Payload>) { }; if let Some(cb) = cb { let py_bytes = PyBytes::new(py, png); - let _ = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)); + if let Err(e) = cb.call1(py, (*msg_type, py_bytes, *hot_x, *hot_y)) { + e.print(py); + } } }); } @@ -369,6 +380,12 @@ fn cursor_to_png(img: &GetCursorImageReply, cap: i32) -> (&'static str, Vec, hot_y = (hot_y as f32 * scale) as i32; } crate::unpremultiply_rgba(&mut image); + // A hotspot can lie outside the visible bbox (its neighborhood was cropped as + // fully transparent). Consumers treat the hotspot as an offset INTO the + // bitmap, so clamp to the cropped bounds instead of emitting off-image + // coordinates. + let hot_x = hot_x.clamp(0, image.width() as i32 - 1); + let hot_y = hot_y.clamp(0, image.height() as i32 - 1); let mut png = Vec::new(); match image.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png) { Ok(()) => ("png", png, hot_x, hot_y), @@ -406,16 +423,35 @@ mod tests { assert_eq!(t, "hide"); } - /// The image is cropped to its visible bbox and the hotspot re-based to the crop: one - /// opaque pixel at (2,1) with hotspot (3,3) yields a 1x1 PNG with hotspot (1,2). + /// The image is cropped to its visible bbox and the hotspot re-based to the crop: + /// a 2x2 visible block at (1,1)..(2,2) with hotspot (2,2) yields a 2x2 PNG with + /// hotspot (1,1). #[test] fn crop_rebases_hotspot() { let mut px = vec![0u32; 16]; - px[1 * 4 + 2] = 0xFF00_0000; - let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + for (x, y) in [(1, 1), (2, 1), (1, 2), (2, 2)] { + px[y * 4 + x] = 0xFF00_0000; + } + let (t, data, hx, hy) = cursor_to_png(&reply(4, 4, 2, 2, px), 32); assert_eq!(t, "png"); assert!(!data.is_empty()); - assert_eq!((hx, hy), (1, 2)); + assert_eq!((hx, hy), (1, 1)); + } + + /// A hotspot whose neighborhood was cropped away as transparent is clamped into + /// the emitted bitmap — consumers use it as an offset INTO the image, and the + /// Wayland path never emits out-of-bounds hotspots either. + #[test] + fn out_of_bbox_hotspot_clamped() { + let mut px = vec![0u32; 16]; + px[1 * 4 + 2] = 0xFF00_0000; + // Visible pixel at (2,1) only. Hotspot (0,0): rebased (-2,-1) -> (0,0). + let (t, _, hx, hy) = cursor_to_png(&reply(4, 4, 0, 0, px.clone()), 32); + assert_eq!(t, "png"); + assert_eq!((hx, hy), (0, 0)); + // Hotspot (3,3): rebased (1,2) past the 1x1 crop -> clamps to (0,0). + let (_, _, hx, hy) = cursor_to_png(&reply(4, 4, 3, 3, px), 32); + assert_eq!((hx, hy), (0, 0)); } /// Premultiplied color becomes straight alpha in the PNG: a half-alpha pixel stored diff --git a/pixelflux/src/x11/mod.rs b/pixelflux/src/x11/mod.rs index a664bbe..4e6a49b 100644 --- a/pixelflux/src/x11/mod.rs +++ b/pixelflux/src/x11/mod.rs @@ -33,12 +33,13 @@ use x11rb::protocol::xfixes::ConnectionExt as XfixesExt; use x11rb::protocol::xproto::{ConnectionExt as XprotoExt, ImageFormat}; use x11rb::rust_connection::RustConnection; -use crate::encoders::overlay::WatermarkLocation; +use crate::encoders::overlay::blend_pixel; use crate::encoders::software::EncodedStripe; use crate::pipeline::X11Pipeline; use crate::recording_sink::RecordingSink; use crate::RustCaptureSettings; +pub mod computer_use; pub mod cursor; /// Cross-thread controls for a running capture: a bag of atomics (plus two mutex-guarded @@ -261,28 +262,6 @@ fn grab_frame( Ok(()) } -/// Alpha-blend a source pixel (pre-split into r,g,b,a) over a BGRA destination pixel. -/// -/// Cursor and watermark pixels are overwhelmingly either fully opaque or fully transparent, and this -/// runs per pixel per frame on the CPU, so the two extremes are special-cased to skip the blend -/// arithmetic entirely: an opaque source (`a == 255`) simply overwrites, a fully transparent source -/// (`a == 0`) is left as-is, and only genuine edge pixels pay for the integer source-over. In every -/// case only the B / G / R bytes are written; the destination's alpha byte is left as the capture -/// delivered it. -#[inline] -fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { - if a == 255 { - dst[0] = b; - dst[1] = g; - dst[2] = r; - } else if a > 0 { - let ia = 255 - a as u32; - dst[0] = ((b as u32 * a as u32 + dst[0] as u32 * ia) / 255) as u8; - dst[1] = ((g as u32 * a as u32 + dst[1] as u32 * ia) / 255) as u8; - dst[2] = ((r as u32 * a as u32 + dst[2] as u32 * ia) / 255) as u8; - } -} - /// Frame-space top-left of the cursor image, given the XFixes hotspot position. /// /// XFixes reports the cursor position at its HOTSPOT; X draws the image with its top-left at @@ -290,7 +269,7 @@ fn blend_pixel(dst: &mut [u8], r: u8, g: u8, b: u8, a: u8) { /// The result may go negative near the frame edges, which `overlay_cursor` clips per pixel to match /// the server's own edge clipping. #[inline] -fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { +pub(crate) fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: i32) -> (i32, i32) { (x as i32 - xhot as i32 - cap_x, y as i32 - yhot as i32 - cap_y) } @@ -298,7 +277,7 @@ fn cursor_image_origin(x: i16, y: i16, xhot: u16, yhot: u16, cap_x: i32, cap_y: /// at `(img_x, img_y)`, blending each pixel through `blend_pixel` with per-pixel bounds clipping so /// an image straddling a frame edge writes only its in-frame portion. #[allow(clippy::too_many_arguments)] -fn overlay_cursor( +pub(crate) fn overlay_cursor( frame: &mut [u8], stride: usize, frame_w: i32, @@ -330,147 +309,6 @@ fn overlay_cursor( } } -/// CPU watermark: the raw RGBA pixels (row-major, `w*h*4`) in host memory plus the -/// placement / animation state, blended directly into the captured BGRA frame. -/// -/// The blend is on the CPU because the frame is already sitting in host shm memory before it reaches -/// any encoder, so stamping the overlay there is both the cheapest place to do it and the one place -/// it applies identically regardless of which encoder (hardware or software) runs downstream. The -/// pixels are kept as RGBA straight from `image`'s decode and converted per pixel as they are blended. -struct X11Watermark { - pixels: Vec, - w: i32, - h: i32, - pos_x: i32, - pos_y: i32, - sub_x: f64, - sub_y: f64, - vel_x: f64, - vel_y: f64, - loaded: bool, -} - -impl X11Watermark { - /// Load the watermark image at `path` into host RGBA, or return an unloaded stub. - /// - /// An empty path or any decode failure yields `loaded == false`, which every method treats as a - /// no-op, so a missing or broken watermark simply disables the overlay. The bouncing-animation - /// velocities are seeded here. - fn load(path: &str) -> Self { - let mut wm = Self { - pixels: Vec::new(), - w: 0, - h: 0, - pos_x: 0, - pos_y: 0, - sub_x: 0.0, - sub_y: 0.0, - vel_x: 2.0, - vel_y: 2.0, - loaded: false, - }; - if path.is_empty() { - return wm; - } - if let Ok(img) = image::open(std::path::Path::new(path)) { - let rgba = img.to_rgba8(); - wm.w = rgba.width() as i32; - wm.h = rgba.height() as i32; - wm.pixels = rgba.into_vec(); - wm.loaded = wm.w > 0 && wm.h > 0; - } - wm - } - - /// Set the watermark's top-left placement for this frame from the location enum; a stub - /// (unloaded) watermark returns immediately. - /// - /// The fixed corners and center are direct arithmetic. The animated mode advances a bouncing - /// position and reflects velocity off each frame edge, accumulating in the fractional `sub_x` / - /// `sub_y` rather than the integer `pos_x` / `pos_y`: a sub-pixel-per-frame velocity has to - /// survive between frames or the motion would either stall or snap by whole pixels, so the float - /// carries the remainder and `pos_x` / `pos_y` take its floor for the actual blit. - fn update_position(&mut self, frame_w: i32, frame_h: i32, loc_enum: i32) { - if !self.loaded { - return; - } - match WatermarkLocation::from(loc_enum) { - WatermarkLocation::TL => { - self.pos_x = 0; - self.pos_y = 0; - } - WatermarkLocation::TR => { - self.pos_x = frame_w - self.w; - self.pos_y = 0; - } - WatermarkLocation::BL => { - self.pos_x = 0; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::BR => { - self.pos_x = frame_w - self.w; - self.pos_y = frame_h - self.h; - } - WatermarkLocation::MI => { - self.pos_x = (frame_w - self.w) / 2; - self.pos_y = (frame_h - self.h) / 2; - } - WatermarkLocation::AN => { - self.sub_x += self.vel_x; - self.sub_y += self.vel_y; - if self.sub_x <= 0.0 { - self.sub_x = 0.0; - self.vel_x = self.vel_x.abs(); - } else if self.sub_x + self.w as f64 >= frame_w as f64 { - self.sub_x = (frame_w - self.w) as f64; - self.vel_x = -self.vel_x.abs(); - } - if self.sub_y <= 0.0 { - self.sub_y = 0.0; - self.vel_y = self.vel_y.abs(); - } else if self.sub_y + self.h as f64 >= frame_h as f64 { - self.sub_y = (frame_h - self.h) as f64; - self.vel_y = -self.vel_y.abs(); - } - self.pos_x = self.sub_x as i32; - self.pos_y = self.sub_y as i32; - } - WatermarkLocation::None => (), - } - } - - /// Alpha-blend the loaded watermark into the BGRA frame at its current position; a no-op - /// when unloaded. Clips per pixel at the frame bounds because the animated position — or a - /// watermark larger than the capture — can leave part of the image off-frame, and only the - /// in-frame portion may be written. - fn blend_into(&self, frame: &mut [u8], stride: usize, frame_w: i32, frame_h: i32) { - if !self.loaded { - return; - } - for y in 0..self.h { - let ty = self.pos_y + y; - if ty < 0 || ty >= frame_h { - continue; - } - for x in 0..self.w { - let tx = self.pos_x + x; - if tx < 0 || tx >= frame_w { - continue; - } - let src = ((y * self.w + x) * 4) as usize; - let (r, g, b, a) = ( - self.pixels[src], - self.pixels[src + 1], - self.pixels[src + 2], - self.pixels[src + 3], - ); - let off = ty as usize * stride + tx as usize * 4; - blend_pixel(&mut frame[off..off + 4], r, g, b, a); - } - } - } -} - /// A captured raw BGRA frame held in a pooled shm surface, ready to encode. /// /// It carries the surface pointer plus geometry so the encode thread reads the pixels directly (no @@ -719,7 +557,7 @@ where .is_some_and(|pl| pl.reshape(&psettings, size_changed)); if !reshaped { drop(pipeline.take()); - pipeline = Some(X11Pipeline::new(psettings.clone(), recording_sink.clone())); + pipeline = Some(X11Pipeline::new(psettings.clone())); if let Some(pl) = &pipeline { let enc_name = pl.encoder_name(); let mut log_msg = format!( @@ -749,7 +587,13 @@ where } let pl = pipeline.as_mut().unwrap(); - if controls.force_idr.swap(false, Ordering::Relaxed) { + // A recorder connecting to the socket sink needs a fresh decode entry point; it is + // folded into the same standard request-IDR path as a client-driven force_idr. + let sink_idr = recording_sink + .as_ref() + .map(|s| s.should_force_idr()) + .unwrap_or(false); + if controls.force_idr.swap(false, Ordering::Relaxed) || sink_idr { pl.request_idr(); } if controls.rate_dirty.swap(false, Ordering::Acquire) { @@ -771,13 +615,9 @@ where if !stripes.is_empty() { frame_count += 1; stripe_count += stripes.len() as u64; + // IDR arming is consumed inside X11Pipeline::process; this tap only fans out. if let Some(ref socket) = recording_sink { - for stripe in &stripes { - let _ = socket.write_encoded_frame(stripe); - } - if socket.should_force_idr() { - controls.force_idr.store(true, Ordering::Relaxed); - } + socket.write_frame(&stripes, psettings.height); } on_frame(stripes); } @@ -894,7 +734,10 @@ where } let pool = Arc::new(FramePool::new(POOL_N)); - let mut watermark = X11Watermark::load(&settings.watermark_path); + let mut watermark = crate::encoders::overlay::OverlayState::default(); + if !settings.watermark_path.is_empty() { + watermark.load_watermark(&settings.watermark_path, 1.0); + } let enc_pool = pool.clone(); let enc_controls = controls.clone(); @@ -1058,9 +901,9 @@ where } } - if watermark.loaded { + if watermark.is_active() { watermark.update_position(frame_w, frame_h, settings.watermark_location_enum); - watermark.blend_into(buf, stride, frame_w, frame_h); + watermark.blend_bgra(buf, stride, frame_w, frame_h); } let published = pool.publish(