From 5f98d948026d5f43f1418db9c129d1647e4681f1 Mon Sep 17 00:00:00 2001 From: iamnbutler Date: Mon, 5 Oct 2026 08:25:23 -0400 Subject: [PATCH] feat(desktop): add targeted click variants and screenshot points --- .../native/sources/client/actions.swift | 59 +++++++++++++++++-- apps/host/src/desktop.ts | 29 ++++++++- docs/architecture.md | 5 +- docs/desktop-tools.md | 22 ++++--- packages/channel/src/desktop.ts | 28 +++++++-- 5 files changed, 123 insertions(+), 20 deletions(-) diff --git a/apps/desktop/native/sources/client/actions.swift b/apps/desktop/native/sources/client/actions.swift index 07ee8d33..9fe40f7a 100644 --- a/apps/desktop/native/sources/client/actions.swift +++ b/apps/desktop/native/sources/client/actions.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation import PeekabooAutomationKit import PeekabooBridge @@ -17,6 +18,8 @@ private struct ActionRequest: Decodable { let prefix: String? let suffix: String? let selection: String? + let point: PointerPoint? + let kind: String? func validate() throws { guard !snapshot.isEmpty, snapshot.utf16.count <= 256 else { @@ -28,8 +31,22 @@ private struct ActionRequest: Decodable { if op != .select, prefix != nil || suffix != nil || selection != nil { throw ActionError("Only text selection accepts prefix, suffix, or selection.") } + if op != .click, kind != nil || point != nil { + throw ActionError("Only clicks accept a click kind or target point.") + } switch op { - case .click, .type, .select: + case .click: + guard (element == nil) != (point == nil), text == nil, key == nil else { + throw ActionError("Choose exactly one observed element ID or normalized screenshot point.") + } + if let element, element.isEmpty || element.utf16.count > 256 { + throw ActionError("Choose a literal element ID from the inspected snapshot.") + } + try point?.validate() + guard kind == nil || pointerClicks.contains(kind!) else { + throw ActionError("Choose single, double, right, middle, or triple click.") + } + case .type, .select: guard let element, !element.isEmpty, element.utf16.count <= 256, key == nil else { throw ActionError("Choose a literal element ID from the inspected snapshot.") } @@ -46,8 +63,6 @@ private struct ActionRequest: Decodable { if let selection, TextSelectionType(rawValue: selection) == nil { throw ActionError("Choose text, cursor_before, or cursor_after selection.") } - } else if text != nil { - throw ActionError("A click does not accept text.") } case .key: guard element == nil, text == nil, let key, keyboardKeys.contains(key) else { @@ -63,6 +78,28 @@ private struct ActionRequest: Decodable { } } +private struct PointerPoint: Decodable { + let x: Double + let y: Double + + func validate() throws { + guard x.isFinite, y.isFinite, (0..<1).contains(x), (0..<1).contains(y) else { + throw ActionError("Screenshot point coordinates must be at least 0 and less than 1.") + } + } + + func mapped(in authority: SnapshotTargetReceipt.CoordinateAuthority) throws -> CGPoint { + let point = try CaptureCoordinateMapper.globalPoint( + for: CGPoint(x: x, y: y), in: .normalized, context: authority.context + ) + guard authority.target.bounds.contains(point) else { + throw ActionError("The screenshot point is outside its exact captured window.") + } + return point + } +} + +private let pointerClicks: Set = ["single", "double", "right", "middle", "triple"] private let keyboardModifiers: Set = ["command", "control", "option", "shift"] private let keyboardKeys: Set = { var keys: Set = [ @@ -156,14 +193,26 @@ func nativeAction(_ client: PeekabooBridgeClient) async throws -> Data { if request.op == .key, context.focusedElement == nil { throw ActionError("The snapshot has no exact focused control. Click a control, then inspect the window again.") } + var point: CGPoint? + if request.point != nil { + let window = try UIAutomationTarget.ExactWindow(identity: identity, bounds: bounds) + // Normalized coordinates survive host image resizing; authority stays in the bridge's capture. + let authority = try SnapshotTargetReceiptPlanner.assemble( + snapshotID: request.snapshot, detectionResult: detection + ).receipt.requireCoordinateAuthority() + guard authority.target == window, let captured = authority.context.logicalBounds, + window.bounds.contains(captured), !detection.screenshotPath.isEmpty + else { throw ActionError("This observation has no pixel-backed coordinate authority for its exact window.") } + point = try request.point?.mapped(in: authority) + } lease = try await client.beginSnapshotMutation(snapshotId: request.snapshot) invoked = true let evidence: ActionEvidence switch request.op { case .click: evidence = try await ActionEvidence(client.clickWithOutcome( - target: .elementId(request.element!), - clickType: .single, + target: point.map(ClickTarget.coordinates) ?? .elementId(request.element!), + clickType: request.kind.flatMap(ClickType.init(rawValue:)) ?? .single, snapshotId: request.snapshot, windowEvidence: .init(identity: identity, bounds: bounds), allowsAccessibilityValueDelivery: true diff --git a/apps/host/src/desktop.ts b/apps/host/src/desktop.ts index 2458e3a4..b1832f82 100644 --- a/apps/host/src/desktop.ts +++ b/apps/host/src/desktop.ts @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import { type Desktop, + DESKTOP_CLICKS, DESKTOP_KEYS, DESKTOP_MODIFIERS, DESKTOP_SELECTIONS, @@ -220,7 +221,7 @@ async function inspect( width, height, note: - "Screenshot resized; Accessibility bounds remain in their original coordinate system.", + "Screenshot resized; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.", }, }, "ui_elements"); return { text, image }; @@ -241,6 +242,17 @@ async function inspect( } } +function validatePoint(point: unknown) { + if (!point || typeof point !== "object" || !("x" in point) || !("y" in point)) { + throw new Error("Choose a normalized screenshot point with x and y coordinates."); + } + for (const value of [point.x, point.y]) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value >= 1) { + throw new Error("Screenshot point coordinates must be at least 0 and less than 1."); + } + } +} + async function availability( request: Extract, signal: AbortSignal, @@ -303,6 +315,21 @@ function validateAction(request: DesktopAction) { if (typeof request.snapshot !== "string" || !request.snapshot || request.snapshot.length > 256) { throw new Error("Use the snapshot_id from a fresh desktop_inspect result."); } + if (request.op === "click") { + if ((request.element === undefined) === (request.point === undefined)) { + throw new Error("Choose exactly one observed element ID or normalized screenshot point."); + } + if (request.point !== undefined) validatePoint(request.point); + else if ( + typeof request.element !== "string" || !request.element || request.element.length > 256 + ) { + throw new Error("Choose an element ID from the inspected snapshot."); + } + if (request.kind !== undefined && !DESKTOP_CLICKS.includes(request.kind)) { + throw new Error("Choose single, double, right, middle, or triple click."); + } + return; + } if (request.op === "key") { if (!DESKTOP_KEYS.includes(request.key)) throw new Error("Choose one supported key."); const modifiers = request.modifiers; diff --git a/docs/architecture.md b/docs/architecture.md index df5cdc97..e6251407 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -177,8 +177,9 @@ Agents inspect and operate native windows through an injected desktop capability Peekaboo's native bridge inside Ace's UI process, which owns macOS Accessibility, Screen Recording, and Event Synthesizing permissions. The host invokes a bundled client over a local Unix socket. The bridge accepts only the client's exact identifier signed by Ace's team; the client verifies the host's -signing team. Tools expose application and window inventories, observation, single element clicks, -replacement of editable field values, text selection, and keys or shortcuts. Actions use background delivery bound to a +signing team. Tools expose application and window inventories, observation, element or screenshot +clicks, replacement of editable field values, text selection, and keys or shortcuts. Screenshot +points are normalized and mapped through the snapshot's native capture geometry. Actions use background delivery bound to a snapshot's exact process, window, and controls; every dispatched action requires a new observation. Pi records their intent and never automatically replays an interrupted action. Results distinguish completed operations, refusals before dispatch, and uncertain delivery. The host bounds diff --git a/docs/desktop-tools.md b/docs/desktop-tools.md index 92313fd2..0ac5954c 100644 --- a/docs/desktop-tools.md +++ b/docs/desktop-tools.md @@ -11,7 +11,7 @@ same result appears in the chat's expandable tool output, including after reopen Native desktop tools require macOS 15 or later. Ace embeds Peekaboo's native library; there is no separate Peekaboo installation or permission grant. In Ace Settings, open This Mac and enable Accessibility and Screen Recording. macOS grants those permissions to Ace on that machine. -Keyboard event delivery also requires Event Synthesizing, which Ace reports and requests separately. +Keyboard and pointer event delivery also require Event Synthesizing, which Ace reports and requests separately. Clicking Accessibility controls, selecting text, and replacing field values use Accessibility permission. Keep Ace open on the machine running the tools. Closing its window is fine, but quitting Ace @@ -34,7 +34,8 @@ find the client alongside Ace Helper automatically. Set `mode` to `pixels` for an explicit read-only screenshot when Accessibility is unavailable. Pixel inspection verifies the same exact target and image content but returns no reusable action snapshot or element IDs. The default mode is `accessibility`. -- `desktop_click` clicks one observed Accessibility element once. +- `desktop_click` clicks one observed Accessibility element or screenshot point. `kind` defaults + to `single`; `double`, `right`, `middle`, and `triple` are also supported. - `desktop_type` replaces the entire string value of one observed editable Accessibility element. It does not append text, send keystrokes, or use the clipboard. Fields that do not support this operation are refused. @@ -55,8 +56,13 @@ Use `is_active` only when `is_active_known` is true, and `is_hidden` only when inventory completeness and warnings. Choose the application from the inventory and the window from that application's window list. -Inspect the window before acting. Pass its `snapshot_id` as `snapshot`; click, type, and select also take -the literal `element` ID from that observation. The bridge binds the snapshot to the application +Inspect the window before acting. Pass its `snapshot_id` as `snapshot`. Type and select take the +literal `element` ID from that observation. Click takes exactly one `element` or `point`. +Points use normalized image coordinates: `x` is the fraction from the screenshot's left edge, +`y` from its top edge, each at least 0 and less than 1. For example, `{ "x": 0.5, "y": 0.5 }` +is its center. Normalized points retain their meaning when Ace resizes the screenshot. The native +bridge maps them through the snapshot's own capture geometry and exact-window receipt, refusing +missing geometry, points outside the window, or a window moved or resized since observation. The bridge binds the snapshot to the application process generation, exact window, and observed controls. Keys additionally require the same focused control. Selecting text does not activate its window; inspect the current focus before sending keys. A stale, missing, disabled, or unsupported target is refused instead of sending @@ -69,9 +75,11 @@ completed action and explains the observation failure; it does not imply that th be repeated. These tools use targeted background delivery. They do not activate an app or fall back to global -mouse or keyboard input when a background route is unavailable. The target app can still respond -by changing its own state or opening a window. Literal insertion, pixel clicks, scrolling, drag-and-drop, clipboard -operations, and app or window management remain later slices of +mouse or keyboard input when a background route is unavailable. Pointer actions do not move the +physical pointer. The target app can still respond by changing its own state or opening a window. +Modifier-clicks and long presses need a separate foreground interaction contract; they are not +emulated with held keys or mouse buttons across calls. Literal insertion, scrolling, drag-and-drop, +clipboard operations, foreground interaction, and app or window management remain later slices of [native computer use](https://github.com/githubnext/ace2/issues/8). Captures are resized and compressed before entering pi's existing channel history. Text and diff --git a/packages/channel/src/desktop.ts b/packages/channel/src/desktop.ts index c2bffc62..df36f32c 100644 --- a/packages/channel/src/desktop.ts +++ b/packages/channel/src/desktop.ts @@ -14,7 +14,13 @@ export type DesktopRequest = | { op: "apps" } | { op: "windows"; pid: number } | { op: "inspect"; pid: number; window: number; mode?: "accessibility" | "pixels" } - | { op: "click"; snapshot: string; element: string } + | { + op: "click"; + snapshot: string; + element?: string; + point?: DesktopPoint; + kind?: DesktopClick; + } | { op: "type"; snapshot: string; element: string; text: string } | { op: "select"; @@ -27,6 +33,10 @@ export type DesktopRequest = } | { op: "key"; snapshot: string; key: DesktopKey; modifiers?: DesktopModifier[] }; +export type DesktopPoint = { x: number; y: number }; +export const DESKTOP_CLICKS = ["single", "double", "right", "middle", "triple"] as const; +export type DesktopClick = (typeof DESKTOP_CLICKS)[number]; + export const DESKTOP_KEYS = [ "enter", "tab", @@ -136,6 +146,10 @@ export function desktop(execute: Desktop): Extension { }; const snapshot = Type.String({ minLength: 1, maxLength: 256 }); const element = Type.String({ minLength: 1, maxLength: 256 }); + const point = Type.Object({ + x: Type.Number({ minimum: 0, exclusiveMaximum: 1 }), + y: Type.Number({ minimum: 0, exclusiveMaximum: 1 }), + }); return defineExtension({ name: "ace-desktop", tools: [ @@ -172,12 +186,16 @@ export function desktop(execute: Desktop): Extension { defineTool({ name: "desktop_click", description: - "Click one Accessibility element from a fresh desktop_inspect result on this channel's execution host. Pass its snapshot_id as snapshot and its element ID as element. The snapshot binds the action to that application process and window and is single-use. Stale or unsupported targets are refused. Inspect again after the action; never repeat an interrupted action without checking the current state.", - parameters: Type.Object({ snapshot, element }), + "Click one element or screenshot point from a fresh desktop_inspect result on this channel's execution host. Pass snapshot_id as snapshot and exactly one of the literal element ID or point. A point uses normalized image coordinates: x is the fraction from the screenshot's left edge, y from its top edge, each >= 0 and < 1. kind defaults to single; double, right, middle, and triple are also supported. Coordinates stay bound to the captured window, even when the screenshot was resized. Stale or unsupported targets are refused without activation or global input. The snapshot is single-use; inspect again after the action and never blindly repeat interrupted input.", + parameters: Type.Object({ + snapshot, + element: Type.Optional(element), + point: Type.Optional(point), + kind: Type.Optional(Type.Union(DESKTOP_CLICKS.map((value) => Type.Literal(value)))), + }), replay: "unsafe", executionMode: "sequential", - execute: async ({ snapshot, element }, api, context) => - act({ op: "click", snapshot, element }, api, context), + execute: async (args, api, context) => act({ op: "click", ...args }, api, context), }), defineTool({ name: "desktop_type",