From b3871069e5ec63aa697ab94eb4011d72def797ec Mon Sep 17 00:00:00 2001 From: zerob13 Date: Tue, 28 Jul 2026 12:58:02 +0800 Subject: [PATCH 1/3] docs(pip): define computer use snapshots --- .../nativekit-agent-browser-pip/spec.md | 6 + docs/features/agent-browser-pip/spec.md | 6 + .../computer-use-snapshot-pip/plan.md | 310 +++++++++ .../computer-use-snapshot-pip/spec.md | 601 ++++++++++++++++++ .../computer-use-snapshot-pip/tasks.md | 135 ++++ 5 files changed, 1058 insertions(+) create mode 100644 docs/features/computer-use-snapshot-pip/plan.md create mode 100644 docs/features/computer-use-snapshot-pip/spec.md create mode 100644 docs/features/computer-use-snapshot-pip/tasks.md diff --git a/docs/architecture/nativekit-agent-browser-pip/spec.md b/docs/architecture/nativekit-agent-browser-pip/spec.md index 462c93ee7..69c6851e9 100644 --- a/docs/architecture/nativekit-agent-browser-pip/spec.md +++ b/docs/architecture/nativekit-agent-browser-pip/spec.md @@ -56,6 +56,12 @@ contracts remain in force. The current renderer Canvas remains the compatibility GitHub issue sync was not requested and was not performed. +On 2026-07-28, a +[Computer Use latest-snapshot PiP extension](../../features/computer-use-snapshot-pip/spec.md) was +proposed. It plans to extract the process-global NativeKit lifecycle into one shared PiP +coordinator while preserving this document's Browser ownership and controls. Until that proposal is +implemented, the Browser-only architecture documented here remains current. + ## Counterpoint The NativeKit path will make panel dragging fluid and allow the panel to leave the DeepChat window. diff --git a/docs/features/agent-browser-pip/spec.md b/docs/features/agent-browser-pip/spec.md index b5b471222..4d67bf744 100644 --- a/docs/features/agent-browser-pip/spec.md +++ b/docs/features/agent-browser-pip/spec.md @@ -16,6 +16,12 @@ Canvas described here remains the compatibility fallback. This document's page o background viewport, read-only preview, panel handoff, and run-scoped dismissal contracts remain authoritative. +On 2026-07-28, a +[Computer Use latest-snapshot PiP extension](../computer-use-snapshot-pip/spec.md) was proposed. +That proposal shares only process-global NativeKit presentation ownership; it does not generalize +Browser page, capture, panel-handoff, or public feature contracts. Until implemented, it does not +change the Browser behavior documented here. + The referenced screenshot was not available in the current task context, so the interaction and layout are specified here while exact visual styling remains provisional. diff --git a/docs/features/computer-use-snapshot-pip/plan.md b/docs/features/computer-use-snapshot-pip/plan.md new file mode 100644 index 000000000..9b7201aee --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/plan.md @@ -0,0 +1,310 @@ +# Computer Use Snapshot Picture-in-Picture Implementation Plan + +## Status + +Proposed on 2026-07-28. No implementation work is included in this document-only change. + +CUA runtime, plugin, policy, skill, and asset updates are explicitly out of scope. The plan treats +the current successful `get_window_state` image result as an immutable input. + +## Delivery Strategy + +Establish one process-global PiP owner before adding the Computer Use presenter. NativeKit toolbar +configuration is global, so implementing a second standalone overlay adapter first would create a +known Browser/Computer race. + +Then carry existing Agent run identity through the MCP execution path and observe only resolved, +trusted CUA calls. Build the snapshot transform and renderer fallback after those identities are +available. Finish with arbitration, lifecycle hardening, and Browser regression verification. + +```text +shared NativeKit owner + | + +--> existing Browser producer + | + +--> Computer Use observer -> presenter -> snapshot producer + | + native or Canvas surface +``` + +## Phase 1: Shared PiP Ownership + +### 1. Extract the process-global overlay owner + +Refactor `AgentBrowserNativeOverlay` so NativeKit lifecycle and host synchronization are owned by +one coordinator instantiated in `src/main/app/composition.ts`. + +Keep the extraction narrow: + +- one dynamic NativeKit import and capability result; +- one `overlay.start()` and `overlay.stop()` lifecycle; +- one host attachment and host/display listener set; +- one visible presentation; +- toolbar profiles for Browser and Computer Use; +- source-specific activation/control callbacks; +- common visibility, removal, prepaint, and latency behavior. + +The Browser presenter remains authoritative for Browser page/run/capture state. Migrate its current +native adapter calls to the coordinator without changing Browser contracts or fallback behavior. + +### 2. Add explicit toolbar profiles + +Configure: + +- Browser: **Open in panel**, then **Close**; activation opens the Browser panel. +- Computer Use: **Close** only; activation is ignored. + +Switch profiles before presenting a source. Hide/remove the previous presentation during a profile +change so controls can never act on the wrong target. + +### 3. Add claim arbitration + +Represent the latest retained claim per source and host/session with run, target identity, and a +monotonic activity sequence. + +- Explicit Browser tool activity claims Browser. +- An actual resolved CUA `get_window_state` invocation claims Computer Use. +- Frame updates are refreshes and cannot claim. +- A background or inactive host/session claim cannot preempt the foreground active session. +- A producer may present only while its claim is the latest eligible sequence for the foreground + active host/session. +- Focus/session changes reevaluate retained sequences without manufacturing new activity. +- Dismissal applies to the current shared session/run. + +Add coordinator unit tests before wiring the Computer Use image path. + +## Phase 2: MCP Execution Identity + +### 4. Propagate `runId` + +Extend MCP call options through: + +1. `McpServicePort.callTool`. +2. `McpService.callTool`. +3. `ToolManager.callTool`. +4. Narrow test doubles and call sites that implement those contracts. + +Update `ToolService` to forward `ToolCallOptions.runId` on the MCP branch alongside access and abort +metadata. Do not add it to `MCPToolCall`, serialize it to the MCP server, or place it in tool +arguments. + +Add a focused regression test proving that an Agent run ID reaches `ToolManager` for MCP calls and +that the outgoing CUA arguments remain byte-for-byte equivalent. + +### 5. Introduce a narrow Computer Use observer + +Define an injected main-process observer port with `started`, `completed`, and `failed` callbacks. +Call it only after `ToolManager` has: + +- resolved the real MCP client and original tool name; +- loaded and validated the server config; +- completed access and permission checks; +- repaired and prepared tool arguments; +- identified the plugin source through `ownerPluginId` or `sourceId`. + +Emit `started` immediately before actual client invocation. Emit one terminal callback without +changing response mapping or error propagation. + +Calls lacking run/session identity and non-CUA calls should produce no preview callbacks. + +## Phase 3: Computer Use Snapshot Presenter + +### 6. Add presenter state and lifecycle + +Create `ComputerUsePreviewPresenter` in `src/main/desktop/computerUse/` and instantiate it once from +the application composition root. + +Track: + +- host window and active chat session; +- run and tool call; +- `pid` and `windowId`; +- epoch and active coordinator claim; +- renderer preview mode; +- dismissed run; +- current valid encoded frame; +- transform in-flight and latest pending result. + +On a new run or target, advance the epoch, hide/remove the old presentation, release the old frame, +and wait for the first valid new frame. + +### 7. Implement snapshot recognition + +On observer callbacks: + +- require the trusted Computer Use plugin identity; +- require original tool name `get_window_state`; +- parse finite positive integer `pid` and `window_id`; +- claim the Computer Use source at invocation start; +- accept only a successful result containing a supported inline image; +- retain the prior valid same-target frame on failure; +- discard results whose session, run, target, tool call, claim, or epoch is stale. + +Use shared utilities only where an existing utility already matches the raw inline MCP image +contract. Do not route through cached message-preview artifacts. + +### 8. Build the bounded image pipeline + +In main: + +1. Select the first supported PNG/JPEG inline image block. +2. Validate base64 and a 16 MiB encoded-input ceiling. +3. Decode and validate dimensions no larger than 8192 x 8192. +4. Resize proportionally within 480 x 300 without upscaling. +5. Encode JPEG quality 72. +6. Reject output larger than 512 KiB. +7. Revalidate epoch and claim. +8. Present to NativeKit or emit the Canvas fallback frame. + +Keep one transform in flight and one replaceable latest pending result. Add rate-limited, +metadata-only warnings for validation and latency failures. + +## Phase 4: Typed Desktop Contracts and Renderer Fallback + +### 9. Add routes and events + +Add Computer Use preview schemas beside the existing desktop/browser contract organization: + +- `computerUse.setPreviewMode`; +- `computerUse.dismissPreview`; +- `computerUse.preview.frame`. + +Expose them through the existing typed preload/client boundary. Validate route sender and session +binding in main. Emit frame events only for Canvas fallback. + +### 10. Add the renderer controller + +Mount `AgentComputerUsePiP.vue` beside `AgentBrowserPiP.vue` in `ChatTabView.vue`. + +Derive: + +- active chat session; +- working versus terminal session state; +- current route; +- host focus. + +Send `eligible`, `suspended`, or `stopped` only when the derived mode changes. On native surfaces, +render nothing and never subscribe to image payloads. + +### 11. Add the Canvas fallback + +For fallback only: + +- render one bounded Canvas card; +- expose one existing translated **Close** button; +- make non-control surface points draggable with pointer capture; +- decode a frame before replacing current Canvas pixels; +- reject stale session/run/sequence events; +- release image resources on replacement and unmount; +- forward no input to the target application. + +Do not add a generic Browser/Computer Vue component unless the completed implementation has +meaningful stable duplication. + +## Phase 5: Lifecycle, Privacy, and Failure Hardening + +### 12. Complete foreground lifecycle + +Use existing window/session binding and host listeners to: + +- hide on blur, hide, minimize, inactive route, or suspended mode; +- restore only when the same current claim remains eligible; +- remove on stopped session, terminal run, target change, host close, or shutdown; +- prevent one host/session from seeing another session's retained frame. + +When the target application comes to the foreground, DeepChat loses focus and the preview hides. + +### 13. Enforce run-scoped dismissal + +Validate exact session and run on close. Suppress all later frames and source changes for that run +without canceling tools. Clear suppression only for a later run or terminal cleanup. + +### 14. Enforce memory-only image handling + +Audit logs, events, caches, and teardown: + +- no base64 or pixel buffer logging; +- no disk writes or settings persistence; +- no generic MCP image broadcast; +- no renderer image payload on native path; +- prompt release of stale buffers, Canvas resources, and native presentations. + +## Phase 6: Verification and Rollout + +### 15. Add focused automated coverage + +Add the smallest tests for: + +- shared coordinator profile switching, claims, dismissal, and stale refresh rejection; +- Browser toolbar/activation/dismissal regression; +- MCP `runId` propagation and unchanged outgoing arguments; +- trusted source recognition and permission/non-invocation paths; +- successful, failed, aborted, and out-of-order observer callbacks; +- presenter run/target/epoch transitions; +- image validation, resize, output limit, and latest-wins behavior; +- native delivery versus Canvas-only frame publication; +- renderer focus/session modes, close, frame retention, and cleanup. + +### 16. Measure performance + +Instrument metadata-only durations for result receipt, transform completion, and NativeKit push. +Verify: + +- zero added capture calls; +- zero idle polling; +- one in-flight transform; +- p95 result-to-visible latency at or below 150 ms; +- no renderer image traffic on native surface; +- no unbounded buffer retention. + +### 17. Run platform QA + +Verify at least: + +- one packaged native-overlay runtime; +- one packaged Canvas-fallback runtime; +- host focus/minimize/restore; +- target application foreground transition; +- Browser-to-Computer and Computer-to-Browser claims; +- native toolbar profile switching; +- run close/restart and session switching; +- malformed and oversized image handling. + +Keep the feature behind an internal gate until both surface paths pass if the platform matrix cannot +complete in the implementation change. + +### 18. Close documentation + +After implementation: + +- update `spec.md` status and record material deviations; +- update the linked Browser NativeKit architecture with the final shared-owner contract; +- mark completed tasks in `tasks.md`; +- include the BEFORE/AFTER ASCII layout in the PR; +- record remaining packaged platform QA explicitly. + +## Verification Commands + +Run after implementation: + +```bash +pnpm run format +pnpm run i18n +pnpm run lint +pnpm run typecheck +pnpm test -- test/main +pnpm test -- test/renderer +``` + +Prefer narrower test file invocations during development. Run the complete affected main and +renderer suites before handoff because the change crosses MCP execution, desktop presentation, and +renderer lifecycle boundaries. + +## Rollback + +The Computer Use observer and presenter must be independently disableable. Disabling them restores +the previous behavior without changing CUA tool execution. + +If the shared coordinator causes a Browser regression, revert the coordinator integration as one +unit and keep the run-propagation changes only if their focused tests pass and no public behavior +changes. Do not fall back to two independent NativeKit owners. diff --git a/docs/features/computer-use-snapshot-pip/spec.md b/docs/features/computer-use-snapshot-pip/spec.md new file mode 100644 index 000000000..ef2fe73e1 --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/spec.md @@ -0,0 +1,601 @@ +# Computer Use Snapshot Picture-in-Picture Spec + +## Status + +Proposed on 2026-07-28. Implementation has not started. + +This proposal adds a read-only picture-in-picture surface for the current Computer Use target by +reusing successful `get_window_state` image results that the Agent already requested. It does not +add a capture loop, change the CUA runtime, enable the upstream experimental PiP, or modify CUA +plugin policies and skill instructions. + +The existing +[Agent Browser NativeKit architecture](../../architecture/nativekit-agent-browser-pip/spec.md) +remains the current implementation. This proposal extends its process-global native overlay into a +shared, single-PiP coordinator while preserving Browser-specific behavior. + +GitHub issue sync was not requested and was not performed. + +## Counterpoint + +Calling this surface "live Computer Use PiP" would promise behavior the available data source does +not provide. A Computer Use tool result contains a point-in-time image only when the Agent calls +`get_window_state`; no frame exists between those calls. + +Polling `get_window_state` merely to animate PiP is the wrong tradeoff. The call also performs +accessibility-state work and updates the driver's element cache, so extra polling would consume +driver, IPC, and model-adjacent resources while changing the semantics of the automation flow. + +The correct first version is therefore an honest **latest snapshot** surface: + +- zero additional operating-system captures; +- first appearance only after a valid snapshot exists; +- immediate replacement when the next valid snapshot arrives; +- retention of the last valid current-run image between snapshots; +- no video, freshness animation, or fake activity. + +## Feasibility Assessment + +The snapshot PiP is feasible without changing CUA. + +| Question | Finding | Consequence | +| --- | --- | --- | +| Is a usable image already available? | Successful `get_window_state` results contain an inline MCP image block. | Reuse that image; add no capture API or polling. | +| Can the existing PiP surface be reused? | NativeKit accepts PNG/JPEG data URLs and supports repeated toolbar reconfiguration. | Share one native owner and switch explicit Browser/Computer profiles. | +| Can a result be tied to the correct Agent run? | The Agent runtime has a stable `runId`, but the MCP branch currently drops it. | Forward internal execution metadata through the MCP service path. | +| Can stale pixels be isolated? | ToolManager knows the resolved plugin source, original tool, prepared target arguments, and tool-call identity. | Observe at that boundary and validate run, target, claim, and epoch before display. | +| Does every platform have a native overlay? | No; the existing NativeKit matrix has Windows arm64, native Wayland, and load-failure gaps. | Preserve a small renderer Canvas fallback. | + +The main engineering risk is not screenshot performance. It is ownership: NativeKit is +process-global, while Browser and Computer Use have different controls and lifecycles. A shared +coordinator is therefore required for correctness. With that boundary in place, the Computer Use +path is a bounded image-result consumer rather than a new screen-capture subsystem. + +## Problem + +When an Agent operates a desktop application through Computer Use, the user can see tool calls in +the conversation but cannot see the target application's latest state without switching away from +DeepChat. That makes long-running automation difficult to supervise. + +The existing Agent Browser PiP solves a related problem, but it cannot be copied directly: + +- Browser owns a continuously capturable `WebContentsView`; Computer Use receives discrete MCP + image results from an external application. +- Browser PiP offers **Open in panel**; Computer Use has no DeepChat side panel to open. +- NativeKit exposes one process-global overlay manager whose toolbar is configured by + `overlay.start()`. Two independent adapters would race and replace each other's configuration. +- Browser emits frames continuously at a bounded rate; Computer Use must not add background polling + to simulate that behavior. + +## User Experience + +### Before + +```text +┌──────────────────────────────────────┐ +│ DeepChat │ +│ │ +│ Agent tool calls only; the target │ +│ application state is not visible. │ +└──────────────────────────────────────┘ +``` + +### After + +```text +┌──────────────────────────────────────┐ ┌────────────────── × ┐ +│ DeepChat │ │ │ +│ │ │ latest target app │ +│ Agent is working │ │ snapshot │ +└──────────────────────────────────────┘ └────────────────────┘ +``` + +The Computer Use PiP: + +- shows only the latest valid snapshot for the active target window; +- is draggable from any non-control point; +- has one **Close** button and no expand, open-panel, title, status, or action controls; +- is read-only and never forwards pointer, wheel, keyboard, or focus input to the target + application; +- closes only the preview for the current run and never closes the target application, cancels the + Agent, or interrupts a tool; +- disappears while DeepChat is not the foreground application, allowing the user to inspect the + real target application instead. + +The existing `common.close` translation is sufficient. This feature adds no user-facing copy. + +## Goals + +- Let users supervise the current Computer Use target without leaving DeepChat. +- Reuse the inline image already returned by a successful CUA `get_window_state` call. +- Show the first PiP only after a valid current-run image has been decoded. +- Update the PiP after each later valid snapshot for the same active target. +- Keep the last valid image visible between snapshots without showing stale images from another + run, session, or target. +- Prefer the existing NativeKit overlay on supported runtimes and preserve the renderer Canvas + fallback elsewhere. +- Share one process-global native overlay safely between Browser and Computer Use. +- Preserve Browser PiP's existing **Open in panel** plus **Close** controls. +- Give Computer Use exactly one **Close** control and ignore native activation. +- Keep image bytes in the main process on the native path. +- Bound image input, transform work, output size, and queued work. +- Hide and remove the surface deterministically across focus, route, session, run, target, and + shutdown lifecycle edges. + +## Non-Goals + +- Updating the CUA runtime, driver, plugin manifest, policies, skill, or packaged assets. +- Enabling or integrating the upstream CUA experimental PiP. +- Adding a CUA screenshot hook, daemon socket, post-action callback, or driver patch. +- Polling `get_window_state` to produce a 2-4 FPS stream. +- Capturing external windows through Electron `desktopCapturer` or a new native capture module. +- Presenting video, shared textures, WebRTC, or a GPU surface. +- Interacting with the target application through the PiP. +- Opening a side panel or adding an expand button for Computer Use. +- Displaying more than one PiP at a time. +- Persisting PiP position, size, image, target identity, or dismissal across app restarts. +- Publishing raw Computer Use images through a generic MCP result event. +- Generalizing all tool previews into a public framework. + +## Terminology + +- **Computer Use source**: the trusted plugin-owned MCP server identified by + `ownerPluginId` or `sourceId`, not by a user-editable server name. +- **Snapshot call**: a CUA `get_window_state` tool invocation with valid `pid` and `window_id` + arguments. +- **Snapshot frame**: the first supported inline image block in a successful snapshot result. +- **Target**: the tuple `(pid, windowId)` within one chat session and Agent run. +- **Run**: the stable Agent execution identity already carried as `ToolCallOptions.runId`. +- **Claim**: explicit Browser tool activity or a Computer Use snapshot-call start that requests + ownership of the single PiP surface. +- **Refresh**: a new frame for the source that already owns the PiP. A refresh is not a claim. +- **Eligible**: the renderer session is active and working; the presenter may show a valid frame + when the host is foreground. +- **Suspended**: retain current target and frame but hide the surface temporarily. +- **Stopped**: remove the presentation and target state for the session. + +`sessionId` in the preview contract is the current chat `conversationId`. + +## Current Repository Contract + +- `YoBrowserPresenter` owns Browser capture, preview mode, and frame delivery. +- `AgentBrowserNativeOverlay` owns the current NativeKit integration and configures + **Open in panel** plus **Close** during `overlay.start()`. +- NativeKit provides one process-global overlay manager. Starting a second independent adapter + would replace global toolbar configuration. +- `AgentBrowserPiP.vue`, mounted by `ChatTabView.vue`, coordinates Browser preview eligibility and + renders the Canvas fallback. +- The Agent runtime already passes `io.requestId` as `ToolCallOptions.runId`. +- `ToolService` forwards `runId` to built-in Agent tools but currently drops it on the MCP path + before `McpService.callTool`. +- `ToolManager` resolves the actual MCP client, original tool name, plugin identity, repaired + arguments, permissions, and prepared arguments before invoking the tool. +- CUA recognition already uses plugin ownership metadata rather than the server display name. +- A successful MCP image block has `type: 'image'`, base64 `data`, and `mimeType`. +- The generic `mcp.toolCall.result` publication does not carry the complete run, source, and target + identity needed for safe preview routing and must not become a pixel transport. +- `toolCallImagePreviews` caches result images for conversation rendering after tool execution. PiP + must consume the inline result before that persistence path and must not create another cached + copy. + +## Product Contract + +### Frame Source + +Only an actual invocation that meets every condition may update the preview: + +1. The resolved MCP client belongs to the official Computer Use plugin. +2. The resolved original tool name is `get_window_state`. +3. The prepared arguments contain finite positive integer `pid` and `window_id` values. +4. The tool result is not an error. +5. The result contains a PNG or JPEG inline image block. +6. The call still matches the active session, run, target, tool call, and presenter epoch when + asynchronous transform work completes. + +Text, resources, cached preview references, permission responses, failed calls, and images from +other CUA tools are not frame sources. + +### First Frame and Refresh + +- A snapshot call start claims Computer Use ownership immediately before MCP invocation, after + permission checks and argument preparation. +- Claiming a new run or target hides and removes the prior presentation before any asynchronous + result can arrive. +- The PiP does not appear until the first valid frame for that claim is decoded and transformed. +- A later valid snapshot for the same run and target replaces the visible frame. +- A failed snapshot keeps the last valid frame for the same active run and target. +- A failed or invalid result never replaces a valid frame with an error image or placeholder. +- Results from a superseded call, run, target, session, or epoch are discarded. + +### Eligibility and Visibility + +The Computer Use PiP may be visible only when all of these are true: + +- its chat route and session are active in the host renderer; +- the session status is `working`; +- the host window exists, is shown, is not minimized, and is focused; +- Computer Use currently owns the shared PiP claim; +- a valid frame exists for the active run and target; +- the current run has not been dismissed. + +Side-panel visibility is irrelevant to Computer Use PiP. It remains relevant to Browser PiP under +the existing Browser contract. + +Temporary host blur, hide, minimize, route switch, or renderer suspension hides the surface without +discarding the valid current target frame. Session terminal state, teardown, new run, target change, +host destruction, or app shutdown removes the presentation. + +### Dismissal + +- **Close** records dismissal for the active session and run. +- Dismissal hides/removes the current shared presentation without changing the tool or target + application. +- Later frames from the dismissed run remain suppressed. +- A later Agent run clears the suppression and may show a new preview after its first valid frame. +- Dismissal is shared at the one-PiP coordinator level so a Browser/Computer source transition + within the same run cannot immediately reopen a surface the user just closed. + +## Target Architecture + +```text +Agent runtime + | + | ToolCallOptions.runId + v +ToolService -> McpService -> ToolManager + | + | resolved trusted CUA client + | original tool + prepared args + v + ComputerUsePreviewObserver + | started | completed/failed + v + ComputerUsePreviewPresenter + | + | validate -> decode -> resize -> JPEG + v + AgentPreviewCoordinator (one per process) + | claim/arbitrate/lifecycle + | + +------+------------------+ + | | + v v + NativeKit overlay renderer Canvas fallback + main-process image typed bounded frame event + Computer toolbar: Close Computer toolbar: Close +``` + +### Shared Native Overlay Ownership + +Extract the process-global concerns currently inside `AgentBrowserNativeOverlay` into one +`AgentPreviewCoordinator` (or equivalently focused shared overlay adapter), instantiated once in +`src/main/app/composition.ts`. + +The shared owner is responsible for: + +- one NativeKit `start()` / `stop()` lifecycle; +- capability detection and Canvas fallback selection; +- one attached host and one visible presentation; +- toolbar profile switching before a source is presented; +- display, host move/resize, focus, show/hide, minimize/restore, and close listeners; +- one eligible source claim and monotonic claim sequence per host/session; +- shared run-scoped dismissal; +- prepaint-before-show ordering; +- NativeKit latency instrumentation and failure fallback. + +Source profiles remain explicit: + +| Source | Toolbar | Native activation | Source action | +| --- | --- | --- | --- | +| Browser | **Open in panel**, **Close** | Open Browser panel | Browser presenter callback | +| Computer Use | **Close** | Ignore | Computer presenter dismissal | + +Do not instantiate two adapters that independently call `overlay.start()`. Do not broaden Browser +routes, frame contracts, or page ownership into a generic public preview API. + +### Source Arbitration + +There is one visible PiP per DeepChat process. + +- Claims from an inactive or background host/session never preempt the foreground active session. +- Within the foreground active host/session, the most recent explicit source claim wins. +- Browser tool activity is a Browser claim. +- An actual CUA `get_window_state` invocation start is a Computer Use claim. +- Frame arrival and periodic Browser capture are refreshes, not claims. +- A refresh may update only the source that still owns the claim. +- A source losing ownership retains only the state needed by its existing lifecycle; it must not + remain visible. +- Reclaim requires later explicit tool activity. +- When focus or active session changes, the coordinator reevaluates the latest retained claim for + that newly eligible host/session instead of assigning a new activity sequence. + +This distinction prevents Browser's 4 FPS capture loop from continuously stealing the PiP back from +a newly active Computer Use target, and prevents a background Agent from hiding or leaking into the +foreground session. + +### MCP Run Identity and Observer + +Add `runId?: string` to the internal MCP call options and forward it through: + +```text +ToolService.callTool + -> McpServicePort.callTool + -> McpService.callTool + -> ToolManager.callTool +``` + +The value is execution metadata only. Never inject `runId` into the CUA tool arguments. + +Inject a narrow `ComputerUsePreviewObserver` into `ToolManager`. It receives only resolved +execution facts: + +```ts +type ComputerUsePreviewCall = { + conversationId: string + runId: string + toolCallId: string + toolName: string + args: Record + source: { + serverName: string + ownerPluginId?: string + sourceId?: string + } +} + +interface ComputerUsePreviewObserver { + started(call: ComputerUsePreviewCall): void + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void + failed(call: ComputerUsePreviewCall, error: unknown): void +} +``` + +Invoke `started` immediately before the resolved MCP client call. Invoke exactly one terminal +callback after success or failure. The observer must not alter the MCP response, permission flow, +abort semantics, tool latency, or error propagation. + +Calls without a conversation ID or run ID remain valid MCP calls but are not preview-eligible. + +### Computer Use Presenter + +Add a main-process `ComputerUsePreviewPresenter` that owns: + +```ts +type ComputerUsePreviewTarget = { + hostWindowId: number + sessionId: string + runId: string + toolCallId: string + pid: number + windowId: number + epoch: number + dismissedRunId: string | null +} +``` + +The presenter: + +- recognizes snapshot calls and target changes; +- advances `epoch` before each new run or target; +- resolves the host from the active renderer/session binding instead of trusting renderer-supplied + window IDs; +- validates and decodes supported inline images in the main process; +- resizes and encodes one bounded JPEG; +- keeps at most one transform in flight per active target; +- drops queued intermediate work and retains only the latest pending result; +- revalidates session, run, target, tool call, claim, and epoch before presentation; +- sends a data URL directly to NativeKit or a typed byte event only to the Canvas fallback; +- owns stop, suspend, resume, dismiss, and shutdown cleanup. + +### Renderer Coordination and Fallback + +Add a focused `AgentComputerUsePiP.vue` beside `AgentBrowserPiP.vue` in `ChatTabView.vue`. + +On the native path it is a headless lifecycle controller: + +- derive active session and `working` status from existing stores; +- observe host focus through the existing `WindowClient`; +- call typed mode routes only when derived state changes; +- render no image DOM and receive no frame bytes. + +On the fallback path it renders a small Canvas card with exactly one **Close** button. It retains +the previous Canvas pixels until a fully decoded new frame is ready and revokes all object URLs and +image resources on replacement/unmount. + +Keep the Computer Use component separate from `AgentBrowserPiP.vue`. Extract shared renderer logic +only if implementation reveals stable duplication; the source rules and controls are intentionally +different. + +## Typed Contracts + +Add focused Computer Use preview contracts rather than exposing generic MCP results. + +### Routes + +```ts +type ComputerUsePreviewMode = 'eligible' | 'suspended' | 'stopped' + +type SetComputerUsePreviewModeInput = { + sessionId: string + mode: ComputerUsePreviewMode +} + +type SetComputerUsePreviewModeOutput = { + updated: boolean + surface: 'native-overlay' | 'renderer-canvas' | 'none' +} + +type DismissComputerUsePreviewInput = { + sessionId: string + runId: string +} + +type DismissComputerUsePreviewOutput = { + dismissed: boolean +} +``` + +- `computerUse.setPreviewMode`: validate the route sender against the active session binding. +- `computerUse.dismissPreview`: dismiss only an exact current session/run match. + +`eligible` means the active renderer session is working. Native host focus listeners remain +authoritative for actual visibility. `suspended` retains the current valid frame while hiding it. +`stopped` removes presenter state for that session. + +### Fallback Event + +```ts +type ComputerUsePreviewFrame = { + sessionId: string + runId: string + sequence: number + width: number + height: number + mimeType: 'image/jpeg' + data: Uint8Array + timestamp: number +} +``` + +Publish `computerUse.preview.frame` only when the selected surface is `renderer-canvas`. The native +path must not send image bytes to the renderer. + +## Image Pipeline + +### Validation + +- Accept only inline PNG and JPEG image blocks. +- Reject unsupported MIME types, malformed base64, empty data, and invalid dimensions. +- Cap encoded input at 16 MiB before decode. +- Cap decoded width and height at 8192 px. +- Never use image metadata, target window title, or application name as user-visible or logged + content. + +### Transform + +- Preserve aspect ratio within 480 x 300. +- Never upscale the source. +- Encode JPEG at quality 72. +- Enforce a 512 KiB encoded output ceiling. +- Keep one transform in flight for the active target. +- If another valid result arrives during transform, retain only the latest pending result. +- Validate the active epoch after decode and again before presentation. + +The NativeKit surface continues to render with a maximum edge of 360 DIP. The larger encoded frame +retains useful detail on scaled displays without increasing overlay geometry. + +### Memory and Privacy + +- Keep frames in memory only. +- Do not write snapshots to disk. +- Do not add snapshots to settings, telemetry, crash metadata, or logs. +- Do not log base64, pixel buffers, window titles, or full CUA arguments. +- Release superseded buffers and renderer resources promptly. +- Reuse the original MCP response for conversation rendering; PiP must not create a second cached + artifact. + +## Platform Behavior + +The feature follows the existing NativeKit support matrix: + +| Runtime | Surface | +| --- | --- | +| macOS arm64/x64 | Native overlay | +| Windows x64 | Native overlay | +| Windows arm64 | Renderer Canvas fallback | +| Linux x64/arm64 under X11 or supported XWayland | Native overlay | +| Linux native Wayland | Renderer Canvas fallback | +| Missing/corrupt addon or native startup failure | Renderer Canvas fallback | + +No platform capture permission is added because the feature consumes the image already returned by +the Computer Use tool. Existing CUA permission behavior is unchanged. + +If NativeKit fails after initialization, hide/remove the native presentation and publish the +retained valid current frame to Canvas fallback without interrupting the Agent. + +## Performance Budget + +- Additional operating-system captures: zero. +- Additional idle polling: zero. +- Idle CPU when no result is being transformed: effectively zero. +- Maximum transforms in flight per active target: one. +- Pending-frame policy: latest wins; no unbounded queue. +- Valid result receipt to visible frame: p95 at or below 150 ms on the reference desktop. +- Synchronous NativeKit `pushImage()` warning threshold: 25 ms, preserving the Browser baseline. +- Encoded output: at most 512 KiB. +- Native path renderer image payloads: zero. + +These are acceptance budgets, not claims that every CUA action produces a snapshot. + +## Failure Behavior + +| Failure | Behavior | +| --- | --- | +| Permission response instead of invocation | Do not claim or show Computer Use PiP | +| Missing run/session identity | Execute tool normally; skip preview | +| Invalid `pid` or `window_id` | Execute tool normally; skip preview | +| Failed `get_window_state` | Retain valid frame for same target; never show error payload | +| Unsupported or malformed image | Ignore frame and retain prior valid frame | +| Oversized image | Reject frame and record a rate-limited metadata-only warning | +| Stale/out-of-order result | Drop by claim, epoch, run, target, and tool-call validation | +| Target changes | Hide/remove old presentation; wait for first valid new-target frame | +| Native overlay unavailable | Use Canvas fallback | +| Host loses focus | Hide while retaining current valid frame | +| Session/run stops | Remove presentation and release frame state | +| Presenter shutdown | Unsubscribe listeners, remove image, detach host, stop shared owner once | + +Preview failures never change the MCP tool result or Agent execution outcome. + +## Acceptance Criteria + +### User Behavior + +- A successful current-run `get_window_state` image produces one read-only latest-snapshot PiP when + the active DeepChat chat is foreground and working. +- The PiP is never blank and appears only after its first valid frame. +- Later valid snapshots for the same target replace the visible image. +- The Computer Use PiP contains exactly one **Close** button. +- Dragging is native on supported platforms and renderer-local on fallback platforms. +- Close suppresses only the current run and never changes the Agent or target application. +- A later run may show a new PiP after its first valid snapshot. +- Host blur/hide/minimize and route/session switch hide the PiP without leaking another session's + pixels. +- Terminal run/session state removes the PiP. + +### Architecture and Safety + +- Browser and Computer Use share one process-global NativeKit owner. +- Browser keeps **Open in panel** plus **Close** and all existing handoff behavior. +- Computer Use ignores NativeKit activation and exposes no open-panel action. +- Source ownership changes only on explicit tool activity; frame refresh does not steal ownership. +- `runId` reaches `ToolManager` as execution metadata and is not added to MCP arguments. +- CUA identity is determined from resolved plugin ownership metadata. +- No generic MCP pixel event is introduced. +- Native frames do not enter the renderer. +- No new operating-system capture or CUA polling occurs. +- Oversize, malformed, stale, and out-of-order images cannot replace a current frame. +- All image bytes stay memory-only and are absent from logs and telemetry. + +### Verification + +- Unit tests cover foreground-scoped claim arbitration, run-scoped dismissal, target changes, stale + epochs, and lifecycle transitions. +- MCP integration tests cover run propagation, trusted CUA recognition, permission paths, valid + result observation, failure observation, and unchanged tool responses. +- Image pipeline tests cover PNG/JPEG, malformed base64, unsupported MIME, dimension/input/output + limits, aspect ratio, no upscale, and latest-wins scheduling. +- Renderer tests cover native headless mode, Canvas fallback frame retention, close, focus, + unmount cleanup, and session switching. +- Browser regression tests cover toolbar profile switching, activation, dismissal, capture refresh, + and panel handoff. +- Packaged QA covers the existing NativeKit native/fallback platform matrix. + +## Rollout + +Land behind an internal feature gate if cross-platform packaged QA cannot complete in the same +change. The gate must disable only the Computer Use preview observer/presenter; it must not alter +CUA tool execution or Browser PiP. + +Do not call the feature complete until Browser regression coverage and at least one native plus one +Canvas fallback packaged run pass. Update this status and the linked Browser NativeKit architecture +document after implementation. diff --git a/docs/features/computer-use-snapshot-pip/tasks.md b/docs/features/computer-use-snapshot-pip/tasks.md new file mode 100644 index 000000000..2f3abf992 --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/tasks.md @@ -0,0 +1,135 @@ +# Computer Use Snapshot Picture-in-Picture Tasks + +## Status + +Proposal only. All implementation tasks remain open. + +## Task List + +- [ ] T01 - Propagate Agent run identity through MCP execution + - Add `runId` to internal `McpServicePort`, `McpService`, and `ToolManager` call options. + - Forward `ToolCallOptions.runId` from the MCP branch in `ToolService`. + - Keep the value out of `MCPToolCall` serialization and CUA arguments. + - Add focused propagation and unchanged-arguments tests. + +- [ ] T02 - Extract the shared process-global PiP coordinator + - Move NativeKit lifecycle, capability detection, host/display synchronization, and presentation + ownership out of the Browser-only adapter. + - Instantiate one coordinator in the application composition root. + - Preserve the current Browser native and Canvas behavior. + - Keep one visible PiP invariant. + +- [ ] T03 - Add source profiles and arbitration + - Keep Browser **Open in panel** plus **Close** controls. + - Add a Computer Use profile with **Close** only and ignored activation. + - Make Browser tool activity and CUA snapshot-call starts claim their source. + - Prevent frame refreshes from stealing ownership. + - Prevent background host/session claims from preempting the foreground session. + - Add shared run-scoped dismissal. + +- [ ] T04 - Add the narrow Computer Use MCP observer + - Inject `started`, `completed`, and `failed` callbacks into `ToolManager`. + - Use resolved `ownerPluginId`/`sourceId`, original tool name, and prepared arguments. + - Observe only actual invocations after permission resolution. + - Preserve tool response, abort, progress, and error semantics. + +- [ ] T05 - Implement Computer Use target and run state + - Add `ComputerUsePreviewPresenter`. + - Track host, session, run, tool call, `pid`, `windowId`, epoch, claim, and dismissal. + - Remove the prior presentation on a new run or target. + - Reject stale and out-of-order terminal results. + +- [ ] T06 - Implement the bounded snapshot pipeline + - Accept successful `get_window_state` PNG/JPEG inline images only. + - Validate base64, input size, decoded dimensions, and output size. + - Resize within 480 x 300 without upscaling and encode JPEG quality 72. + - Keep one transform in flight and one latest pending result. + - Keep frames memory-only and logs pixel-free. + +- [ ] T07 - Add typed Computer Use preview contracts + - Add `computerUse.setPreviewMode`. + - Add `computerUse.dismissPreview`. + - Add Canvas-only `computerUse.preview.frame`. + - Expose routes/events through the typed preload client. + - Validate route sender and active session in main. + +- [ ] T08 - Add renderer lifecycle coordination + - Mount `AgentComputerUsePiP.vue` beside the Browser PiP controller. + - Derive active session, working state, route, and focus from existing stores/clients. + - Send `eligible`, `suspended`, and `stopped` transitions. + - Keep native mode headless and free of image events. + +- [ ] T09 - Add the Canvas fallback UI + - Render the latest valid snapshot with one **Close** button. + - Make non-control points draggable without forwarding target input. + - Retain old pixels until a new frame is fully decoded. + - Reject stale frame sequences and release renderer resources on teardown. + - Reuse `common.close` without adding new copy. + +- [ ] T10 - Complete lifecycle and privacy cleanup + - Hide on host blur/hide/minimize and inactive route/session. + - Remove on terminal run/session, target change, host close, and shutdown. + - Ensure another host/session can never see retained pixels. + - Ensure no disk, settings, telemetry, crash metadata, or generic MCP event stores frame bytes. + +- [ ] T11 - Add main-process regression coverage + - Test shared coordinator startup/shutdown, toolbar switching, arbitration, and dismissal. + - Test Browser activation, close, capture refresh, and panel handoff after extraction. + - Test CUA recognition, permission paths, run identity, target changes, and stale results. + - Test valid, malformed, unsupported, oversized, failed, and aborted image results. + - Test native/fallback routing and latest-wins scheduling. + +- [ ] T12 - Add renderer regression coverage + - Test native headless mode and absence of frame subscriptions. + - Test Canvas first frame, frame retention, sequence rejection, drag, and close. + - Test focus, minimize, session switch, terminal state, and unmount cleanup. + - Verify Computer Use exposes no expand/open-panel behavior. + +- [ ] T13 - Verify performance budgets + - Confirm zero additional operating-system captures and zero idle polling. + - Confirm one transform in flight with no unbounded queue. + - Measure p95 valid-result-to-visible latency against the 150 ms budget. + - Keep NativeKit slow-push warning behavior at 25 ms. + - Confirm native mode sends zero image bytes to the renderer. + +- [ ] T14 - Run packaged platform QA + - Verify one NativeKit native-overlay runtime. + - Verify one renderer Canvas-fallback runtime. + - Exercise host focus/minimize/restore and target-app foreground behavior. + - Exercise Browser/Computer source transitions and toolbar profile changes. + - Exercise close, later run, route/session switch, and app shutdown. + +- [ ] T15 - Run repository verification + - Run `pnpm run format`. + - Run `pnpm run i18n`. + - Run `pnpm run lint`. + - Run `pnpm run typecheck`. + - Run focused main and renderer tests, then the complete affected suites. + +- [ ] T16 - Close implementation documentation + - Update the proposal status and record deviations. + - Update the Browser NativeKit architecture with the implemented shared-owner contract. + - Mark only verified tasks complete. + - Include BEFORE/AFTER ASCII layouts and platform QA status in the PR. + +## Implementation Order + +1. T01-T03 establish identity and safe process-global ownership. +2. T04-T06 connect trusted CUA results to a bounded main-process snapshot pipeline. +3. T07-T09 add typed renderer coordination and fallback UI. +4. T10-T12 harden lifecycle, privacy, and regressions. +5. T13-T16 measure, validate, and close documentation. + +## Done Definition + +- A current successful CUA `get_window_state` image can show one latest-snapshot PiP without any + extra capture or polling. +- Browser and Computer Use safely share one NativeKit owner and retain source-specific controls. +- Computer Use PiP has only **Close**, is read-only, and never affects the Agent or target + application. +- Run, target, session, host, and epoch validation prevent stale pixel disclosure. +- Native delivery avoids renderer image traffic; unsupported runtimes use the bounded Canvas + fallback. +- Image work is memory-only, bounded, latest-wins, and covered by focused tests. +- Browser PiP behavior remains regression-tested. +- Required checks and at least one native plus one fallback packaged QA run pass. From 861fd897793fffaad90db1c8b51d2a0dd508fdd4 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Tue, 28 Jul 2026 14:34:46 +0800 Subject: [PATCH 2/3] feat(pip): add computer use snapshots --- .../nativekit-agent-browser-pip/spec.md | 20 +- docs/features/agent-browser-pip/spec.md | 8 +- .../computer-use-snapshot-pip/plan.md | 30 +- .../computer-use-snapshot-pip/spec.md | 122 +++- .../computer-use-snapshot-pip/tasks.md | 69 +- src/main/app/composition.ts | 26 +- .../desktop/browser/YoBrowserPresenter.ts | 218 +++++-- .../ComputerUsePreviewPresenter.ts | 590 ++++++++++++++++++ .../AgentPreviewCoordinator.ts} | 338 +++++++--- src/main/desktop/routes.ts | 60 +- src/main/mcp/index.ts | 9 +- src/main/mcp/toolManager.ts | 269 +++++++- src/main/tool/index.ts | 1 + src/renderer/api/BrowserClient.ts | 17 +- src/renderer/api/ComputerUseClient.ts | 46 ++ .../src/apps/chat-main/ChatTabView.vue | 4 + .../components/browser/AgentBrowserPiP.vue | 27 +- .../computerUse/AgentComputerUsePiP.vue | 403 ++++++++++++ src/shared/contracts/events.ts | 9 + src/shared/contracts/events/browser.events.ts | 11 + .../contracts/events/computerUse.events.ts | 33 + src/shared/contracts/routes.ts | 9 + src/shared/contracts/routes/browser.routes.ts | 11 + .../contracts/routes/computerUse.routes.ts | 27 + src/shared/types/computerUse.ts | 8 + src/shared/types/desktop.ts | 12 + src/shared/types/mcp.ts | 1 + .../browser/AgentBrowserNativeOverlay.test.ts | 253 -------- .../browser/YoBrowserPresenter.test.ts | 68 +- .../ComputerUsePreviewPresenter.test.ts | 474 ++++++++++++++ .../preview/AgentPreviewCoordinator.test.ts | 344 ++++++++++ test/main/mcp/toolManager.test.ts | 525 +++++++++++++++- test/main/routes/contracts.test.ts | 39 ++ test/main/routes/dispatcher.test.ts | 75 ++- test/main/tool/toolService.test.ts | 21 +- test/renderer/api/clients.test.ts | 57 ++ .../components/AgentBrowserPiP.test.ts | 63 +- .../components/AgentComputerUsePiP.test.ts | 385 ++++++++++++ test/renderer/components/ChatTabView.test.ts | 6 + test/setup.ts | 6 + 40 files changed, 4163 insertions(+), 531 deletions(-) create mode 100644 src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts rename src/main/desktop/{browser/AgentBrowserNativeOverlay.ts => preview/AgentPreviewCoordinator.ts} (62%) create mode 100644 src/renderer/api/ComputerUseClient.ts create mode 100644 src/renderer/src/components/computerUse/AgentComputerUsePiP.vue create mode 100644 src/shared/contracts/events/computerUse.events.ts create mode 100644 src/shared/contracts/routes/computerUse.routes.ts create mode 100644 src/shared/types/computerUse.ts delete mode 100644 test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts create mode 100644 test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts create mode 100644 test/main/desktop/preview/AgentPreviewCoordinator.test.ts create mode 100644 test/renderer/components/AgentComputerUsePiP.test.ts diff --git a/docs/architecture/nativekit-agent-browser-pip/spec.md b/docs/architecture/nativekit-agent-browser-pip/spec.md index 69c6851e9..3a497ef37 100644 --- a/docs/architecture/nativekit-agent-browser-pip/spec.md +++ b/docs/architecture/nativekit-agent-browser-pip/spec.md @@ -58,9 +58,10 @@ GitHub issue sync was not requested and was not performed. On 2026-07-28, a [Computer Use latest-snapshot PiP extension](../../features/computer-use-snapshot-pip/spec.md) was -proposed. It plans to extract the process-global NativeKit lifecycle into one shared PiP -coordinator while preserving this document's Browser ownership and controls. Until that proposal is -implemented, the Browser-only architecture documented here remains current. +implemented. `AgentPreviewCoordinator` now owns the process-global NativeKit lifecycle and +arbitrates one Browser or Computer Use presentation. Browser retains its page, capture, panel +handoff, and **Open in panel** plus **Close** behavior; Computer Use contributes only bounded latest +snapshots with **Close**. ## Counterpoint @@ -253,21 +254,24 @@ focusless render-host BaseWindow -> Agent WebContentsView at 1280 x 800 - branches completed frames to exactly one surface; - stops capture and clears the surface on every existing lifecycle edge. -#### `AgentBrowserNativeOverlay` +#### `AgentPreviewCoordinator` -A focused main-process adapter owns only: +A focused process-global main-process coordinator owns only: - dynamic package loading and one-time capability detection; - `overlay.start()` / `stop()` lifecycle; - one active host and one active presentation; +- Browser and Computer Use toolbar profile switching; +- latest explicit source claim and shared run-scoped dismissal; - host move/resize/close synchronization; - JPEG `Buffer` to data-URL conversion; - prepaint-before-show ordering; -- mapping NativeKit `activate` and configured `control` IDs to the current logical - `{ sessionId, runId, windowId }`; +- mapping NativeKit activation and configured controls to the current source-specific handler; - timing counters around synchronous native calls. -It is not a general Presenter, registry, plugin system, or cross-native abstraction. +`YoBrowserPresenter` remains the Browser producer and `ComputerUsePreviewPresenter` remains the +Computer Use producer. The coordinator is not a public preview framework, plugin system, or +cross-native abstraction. #### Renderer diff --git a/docs/features/agent-browser-pip/spec.md b/docs/features/agent-browser-pip/spec.md index 4d67bf744..a24a4adc3 100644 --- a/docs/features/agent-browser-pip/spec.md +++ b/docs/features/agent-browser-pip/spec.md @@ -17,10 +17,10 @@ background viewport, read-only preview, panel handoff, and run-scoped dismissal authoritative. On 2026-07-28, a -[Computer Use latest-snapshot PiP extension](../computer-use-snapshot-pip/spec.md) was proposed. -That proposal shares only process-global NativeKit presentation ownership; it does not generalize -Browser page, capture, panel-handoff, or public feature contracts. Until implemented, it does not -change the Browser behavior documented here. +[Computer Use latest-snapshot PiP extension](../computer-use-snapshot-pip/spec.md) was implemented. +It shares only process-global NativeKit presentation ownership; it does not generalize Browser +page, capture, panel-handoff, or public feature contracts. Browser keeps **Open in panel** plus +**Close**, continuous bounded capture, and its existing Canvas compatibility behavior. The referenced screenshot was not available in the current task context, so the interaction and layout are specified here while exact visual styling remains provisional. diff --git a/docs/features/computer-use-snapshot-pip/plan.md b/docs/features/computer-use-snapshot-pip/plan.md index 9b7201aee..2047f1bc7 100644 --- a/docs/features/computer-use-snapshot-pip/plan.md +++ b/docs/features/computer-use-snapshot-pip/plan.md @@ -2,10 +2,16 @@ ## Status -Proposed on 2026-07-28. No implementation work is included in this document-only change. +Implemented on branch `codex/computer-use-snapshot-pip` on 2026-07-28. -CUA runtime, plugin, policy, skill, and asset updates are explicitly out of scope. The plan treats -the current successful `get_window_state` image result as an immutable input. +Phases 1-5 are implemented with focused automated coverage, and the complete renderer suite passes. +The complete main suite retains one isolated, unrelated provider-metadata expectation failure. +Performance measurement and packaged native and fallback platform QA remain open; the exact +remaining work is recorded in `tasks.md`. + +CUA runtime, plugin, policy, skill, and asset updates are explicitly out of scope. The plan consumes +successful Agent-requested snapshots plus one bounded private `get_window_state` result after an +eligible successful `click`. ## Delivery Strategy @@ -107,6 +113,22 @@ changing response mapping or error propagation. Calls lacking run/session identity and non-CUA calls should produce no preview callbacks. +### 5a. Add a private post-click refresh + +After a successful exact `click` on the resolved official CUA client: + +- require valid current session/run plus positive integer `pid` and `window_id`; +- synchronously ask the presenter whether the same target is PiP-eligible and not dismissed; +- asynchronously call `get_window_state` on that already resolved client with only + `{ pid, window_id }`; +- send the private result only through the preview observer; +- never publish or return the private result to the Agent or conversation; +- retain the original click response and latency behavior if capture fails. + +Do not trigger on permission responses, failed clicks, stale targets, `right_click`, +`double_click`, inactive PiP, or arbitrary servers. Do not add timers, polling, or CUA runtime +changes. + ## Phase 3: Computer Use Snapshot Presenter ### 6. Add presenter state and lifecycle @@ -250,7 +272,7 @@ Add the smallest tests for: Instrument metadata-only durations for result receipt, transform completion, and NativeKit push. Verify: -- zero added capture calls; +- at most one added capture call per eligible successful `click` and none while PiP is ineligible; - zero idle polling; - one in-flight transform; - p95 result-to-visible latency at or below 150 ms; diff --git a/docs/features/computer-use-snapshot-pip/spec.md b/docs/features/computer-use-snapshot-pip/spec.md index ef2fe73e1..b0d224dab 100644 --- a/docs/features/computer-use-snapshot-pip/spec.md +++ b/docs/features/computer-use-snapshot-pip/spec.md @@ -2,33 +2,47 @@ ## Status -Proposed on 2026-07-28. Implementation has not started. +Implemented on branch `codex/computer-use-snapshot-pip` on 2026-07-28. -This proposal adds a read-only picture-in-picture surface for the current Computer Use target by -reusing successful `get_window_state` image results that the Agent already requested. It does not -add a capture loop, change the CUA runtime, enable the upstream experimental PiP, or modify CUA -plugin policies and skill instructions. +Format, i18n, lint, typecheck, focused main/renderer coverage, and the complete renderer suite pass. +The complete main suite has one unrelated provider-metadata expectation failure in +`test/main/provider/modelConfig.test.ts`; the same test fails in isolation and neither it nor its +implementation is changed by this feature. Packaged native-overlay and Canvas-fallback QA, +cross-platform interaction checks, and the 150 ms p95 result-to-visible measurement remain open and +are tracked in `tasks.md`. + +This feature adds a read-only picture-in-picture surface for the current Computer Use target by +reusing successful `get_window_state` image results. In addition to Agent-requested snapshots, one +eligible successful `click` may schedule one private `get_window_state` refresh whose result is +consumed only by PiP. It does not add a capture loop, change the CUA runtime, enable the upstream +experimental PiP, or modify CUA plugin policies and skill instructions. The existing [Agent Browser NativeKit architecture](../../architecture/nativekit-agent-browser-pip/spec.md) -remains the current implementation. This proposal extends its process-global native overlay into a -shared, single-PiP coordinator while preserving Browser-specific behavior. +now uses the implemented process-global, single-PiP coordinator shared with Computer Use while +preserving Browser-specific behavior. GitHub issue sync was not requested and was not performed. ## Counterpoint Calling this surface "live Computer Use PiP" would promise behavior the available data source does -not provide. A Computer Use tool result contains a point-in-time image only when the Agent calls -`get_window_state`; no frame exists between those calls. +not provide. A Computer Use tool result contains a point-in-time image only when +`get_window_state` runs; no frame exists between those calls. Polling `get_window_state` merely to animate PiP is the wrong tradeoff. The call also performs accessibility-state work and updates the driver's element cache, so extra polling would consume driver, IPC, and model-adjacent resources while changing the semantics of the automation flow. -The correct first version is therefore an honest **latest snapshot** surface: +An event-driven refresh after a successful `click` is a narrower tradeoff than polling: it adds at +most one capture at a user-visible state boundary, only while PiP is eligible, and does not put the +private result on the Agent path. `get_window_state` still refreshes the CUA accessibility cache, so +Agents must continue to obtain their own current state before selecting a later element index. + +The correct version is therefore an honest **latest snapshot** surface: -- zero additional operating-system captures; +- no idle or timer-driven captures; +- at most one private snapshot request after each eligible successful `click`; - first appearance only after a valid snapshot exists; - immediate replacement when the next valid snapshot arrives; - retention of the last valid current-run image between snapshots; @@ -40,7 +54,7 @@ The snapshot PiP is feasible without changing CUA. | Question | Finding | Consequence | | --- | --- | --- | -| Is a usable image already available? | Successful `get_window_state` results contain an inline MCP image block. | Reuse that image; add no capture API or polling. | +| Is a usable image already available? | Successful `get_window_state` results contain an inline MCP image block. | Reuse Agent-requested results and issue only an event-driven private refresh after eligible clicks; add no capture API or polling. | | Can the existing PiP surface be reused? | NativeKit accepts PNG/JPEG data URLs and supports repeated toolbar reconfiguration. | Share one native owner and switch explicit Browser/Computer profiles. | | Can a result be tied to the correct Agent run? | The Agent runtime has a stable `runId`, but the MCP branch currently drops it. | Forward internal execution metadata through the MCP service path. | | Can stale pixels be isolated? | ToolManager knows the resolved plugin source, original tool, prepared target arguments, and tool-call identity. | Observe at that boundary and validate run, target, claim, and epoch before display. | @@ -64,8 +78,8 @@ The existing Agent Browser PiP solves a related problem, but it cannot be copied - Browser PiP offers **Open in panel**; Computer Use has no DeepChat side panel to open. - NativeKit exposes one process-global overlay manager whose toolbar is configured by `overlay.start()`. Two independent adapters would race and replace each other's configuration. -- Browser emits frames continuously at a bounded rate; Computer Use must not add background polling - to simulate that behavior. +- Browser emits frames continuously at a bounded rate; Computer Use uses discrete Agent-requested + and post-click snapshots and must not add background polling to simulate that behavior. ## User Experience @@ -102,12 +116,22 @@ The Computer Use PiP: - disappears while DeepChat is not the foreground application, allowing the user to inspect the real target application instead. +Post-click freshness changes without changing the layout: + +```text +BEFORE click -> Agent receives click result -> PiP retains the prior snapshot +AFTER click -> Agent receives click result + `-> private get_window_state -> PiP replaces the snapshot +``` + The existing `common.close` translation is sufficient. This feature adds no user-facing copy. ## Goals - Let users supervise the current Computer Use target without leaving DeepChat. -- Reuse the inline image already returned by a successful CUA `get_window_state` call. +- Reuse inline images returned by successful CUA `get_window_state` calls. +- Refresh PiP after an eligible successful `click` without exposing the private snapshot result to + the Agent, conversation, or generic MCP result event. - Show the first PiP only after a valid current-run image has been decoded. - Update the PiP after each later valid snapshot for the same active target. - Keep the last valid image visible between snapshots without showing stale images from another @@ -126,7 +150,7 @@ The existing `common.close` translation is sufficient. This feature adds no user - Updating the CUA runtime, driver, plugin manifest, policies, skill, or packaged assets. - Enabling or integrating the upstream CUA experimental PiP. -- Adding a CUA screenshot hook, daemon socket, post-action callback, or driver patch. +- Adding a CUA driver screenshot hook, daemon socket, callback, or runtime patch. - Polling `get_window_state` to produce a 2-4 FPS stream. - Capturing external windows through Electron `desktopCapturer` or a new native capture module. - Presenting video, shared textures, WebRTC, or a GPU surface. @@ -143,6 +167,8 @@ The existing `common.close` translation is sufficient. This feature adds no user `ownerPluginId` or `sourceId`, not by a user-editable server name. - **Snapshot call**: a CUA `get_window_state` tool invocation with valid `pid` and `window_id` arguments. +- **Private post-click snapshot**: an asynchronous `get_window_state` invocation made through the + same resolved official CUA client after a successful `click`; its result is PiP-only. - **Snapshot frame**: the first supported inline image block in a successful snapshot result. - **Target**: the tuple `(pid, windowId)` within one chat session and Agent run. - **Run**: the stable Agent execution identity already carried as `ToolCallOptions.runId`. @@ -156,7 +182,7 @@ The existing `common.close` translation is sufficient. This feature adds no user `sessionId` in the preview contract is the current chat `conversationId`. -## Current Repository Contract +## Baseline Repository Contract - `YoBrowserPresenter` owns Browser capture, preview mode, and frame delivery. - `AgentBrowserNativeOverlay` owns the current NativeKit integration and configures @@ -178,11 +204,28 @@ The existing `common.close` translation is sufficient. This feature adds no user must consume the inline result before that persistence path and must not create another cached copy. +## Implemented Deviations + +- Added source-specific `computerUse.preview.surface.changed` and + `browser.preview.surface.changed` events. They carry only bounded identity and surface metadata, + never image bytes, and let renderers react to native failure and cross-source arbitration without + polling. +- Added an `epoch` to Computer Use surface and frame events so a later target in the same run cannot + accept an in-flight frame from the prior target. +- The shared coordinator permanently falls back to Canvas after a NativeKit load, startup, toolbar, + host-attach, or frame-push failure for the current process. It does not repeatedly retry a failed + native path during Agent work. +- An eligible successful official CUA `click` schedules one non-blocking private + `get_window_state({ pid, window_id })` call. The private response is routed only to the preview + observer and is never published as an MCP result or returned to the Agent. +- No CUA runtime, driver, plugin manifest, policy, skill, or packaged asset was changed. + ## Product Contract ### Frame Source -Only an actual invocation that meets every condition may update the preview: +Only an actual Agent-requested or private post-click invocation that meets every condition may +update the preview: 1. The resolved MCP client belongs to the official Computer Use plugin. 2. The resolved original tool name is `get_window_state`. @@ -195,6 +238,25 @@ Only an actual invocation that meets every condition may update the preview: Text, resources, cached preview references, permission responses, failed calls, and images from other CUA tools are not frame sources. +### Private Post-click Refresh + +- Only the exact resolved original tool `click` is eligible; `right_click`, `double_click`, and + other actions do not implicitly expand this contract. +- The click must succeed on the resolved official CUA client and carry finite positive integer + `pid` and `window_id` values. +- The presenter must confirm that PiP is eligible and that the click still matches its active + session, run, target, and non-dismissed state. +- The private call uses only `{ pid, window_id }`, starts asynchronously after click completion, and + does not delay or change the click response. +- The private call bypasses Agent-facing permission and result publication because the triggering + click has already passed access checks and the plugin policy explicitly allows + `get_window_state`. +- The private result enters only the same bounded preview observer and image pipeline. Its text, + accessibility tree, and other content are discarded. +- A private-call failure retains the last valid frame and never changes Agent execution. +- No private snapshot is scheduled while the preview is stopped, suspended, dismissed, stale, or + missing a valid active target. + ### First Frame and Refresh - A snapshot call start claims Computer Use ownership immediately before MCP invocation, after @@ -344,6 +406,7 @@ type ComputerUsePreviewCall = { } interface ComputerUsePreviewObserver { + shouldCaptureAfterClick?(call: ComputerUsePreviewCall): boolean started(call: ComputerUsePreviewCall): void completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void failed(call: ComputerUsePreviewCall, error: unknown): void @@ -351,8 +414,10 @@ interface ComputerUsePreviewObserver { ``` Invoke `started` immediately before the resolved MCP client call. Invoke exactly one terminal -callback after success or failure. The observer must not alter the MCP response, permission flow, -abort semantics, tool latency, or error propagation. +callback after success or failure. The optional post-click predicate is synchronous, failure-safe, +and may authorize only a PiP-eligible current target. Agent-requested observer work must not alter +the MCP response, permission flow, abort semantics, tool latency, or error propagation. Private +post-click work is asynchronous and must not delay or mutate the original click result. Calls without a conversation ID or run ID remain valid MCP calls but are not preview-eligible. @@ -493,6 +558,8 @@ retains useful detail on scaled displays without increasing overlay geometry. - Release superseded buffers and renderer resources promptly. - Reuse the original MCP response for conversation rendering; PiP must not create a second cached artifact. +- Discard every non-image field from a private post-click response; do not publish, cache, persist, + or return that response to the Agent. ## Platform Behavior @@ -507,15 +574,15 @@ The feature follows the existing NativeKit support matrix: | Linux native Wayland | Renderer Canvas fallback | | Missing/corrupt addon or native startup failure | Renderer Canvas fallback | -No platform capture permission is added because the feature consumes the image already returned by -the Computer Use tool. Existing CUA permission behavior is unchanged. +No platform capture permission is added because both Agent-requested and private snapshots use the +existing Computer Use tool. Existing CUA permission behavior is unchanged. If NativeKit fails after initialization, hide/remove the native presentation and publish the retained valid current frame to Canvas fallback without interrupting the Agent. ## Performance Budget -- Additional operating-system captures: zero. +- Additional captures: at most one per eligible successful `click`; zero when PiP is not eligible. - Additional idle polling: zero. - Idle CPU when no result is being transformed: effectively zero. - Maximum transforms in flight per active target: one. @@ -535,6 +602,7 @@ These are acceptance budgets, not claims that every CUA action produces a snapsh | Missing run/session identity | Execute tool normally; skip preview | | Invalid `pid` or `window_id` | Execute tool normally; skip preview | | Failed `get_window_state` | Retain valid frame for same target; never show error payload | +| Failed private post-click snapshot | Retain valid frame; do not change or delay the click result | | Unsupported or malformed image | Ignore frame and retain prior valid frame | | Oversized image | Reject frame and record a rate-limited metadata-only warning | | Stale/out-of-order result | Drop by claim, epoch, run, target, and tool-call validation | @@ -554,6 +622,8 @@ Preview failures never change the MCP tool result or Agent execution outcome. the active DeepChat chat is foreground and working. - The PiP is never blank and appears only after its first valid frame. - Later valid snapshots for the same target replace the visible image. +- An eligible successful `click` schedules a private PiP-only snapshot and may refresh the visible + image without adding screenshot content to the Agent response. - The Computer Use PiP contains exactly one **Close** button. - Dragging is native on supported platforms and renderer-local on fallback platforms. - Close suppresses only the current run and never changes the Agent or target application. @@ -572,7 +642,8 @@ Preview failures never change the MCP tool result or Agent execution outcome. - CUA identity is determined from resolved plugin ownership metadata. - No generic MCP pixel event is introduced. - Native frames do not enter the renderer. -- No new operating-system capture or CUA polling occurs. +- No idle CUA polling or new capture subsystem is introduced; private capture is bounded to one + asynchronous request per eligible successful `click`. - Oversize, malformed, stale, and out-of-order images cannot replace a current frame. - All image bytes stay memory-only and are absent from logs and telemetry. @@ -581,7 +652,8 @@ Preview failures never change the MCP tool result or Agent execution outcome. - Unit tests cover foreground-scoped claim arbitration, run-scoped dismissal, target changes, stale epochs, and lifecycle transitions. - MCP integration tests cover run propagation, trusted CUA recognition, permission paths, valid - result observation, failure observation, and unchanged tool responses. + result observation, post-click private capture isolation, failure observation, and unchanged tool + responses. - Image pipeline tests cover PNG/JPEG, malformed base64, unsupported MIME, dimension/input/output limits, aspect ratio, no upscale, and latest-wins scheduling. - Renderer tests cover native headless mode, Canvas fallback frame retention, close, focus, diff --git a/docs/features/computer-use-snapshot-pip/tasks.md b/docs/features/computer-use-snapshot-pip/tasks.md index 2f3abf992..be755ed6b 100644 --- a/docs/features/computer-use-snapshot-pip/tasks.md +++ b/docs/features/computer-use-snapshot-pip/tasks.md @@ -2,24 +2,26 @@ ## Status -Proposal only. All implementation tasks remain open. +Implementation, focused automated verification, and the complete renderer suite are complete. +The complete main suite retains one unrelated provider-metadata expectation failure. Performance +measurement and packaged platform QA are still open. ## Task List -- [ ] T01 - Propagate Agent run identity through MCP execution +- [x] T01 - Propagate Agent run identity through MCP execution - Add `runId` to internal `McpServicePort`, `McpService`, and `ToolManager` call options. - Forward `ToolCallOptions.runId` from the MCP branch in `ToolService`. - Keep the value out of `MCPToolCall` serialization and CUA arguments. - Add focused propagation and unchanged-arguments tests. -- [ ] T02 - Extract the shared process-global PiP coordinator +- [x] T02 - Extract the shared process-global PiP coordinator - Move NativeKit lifecycle, capability detection, host/display synchronization, and presentation ownership out of the Browser-only adapter. - Instantiate one coordinator in the application composition root. - Preserve the current Browser native and Canvas behavior. - Keep one visible PiP invariant. -- [ ] T03 - Add source profiles and arbitration +- [x] T03 - Add source profiles and arbitration - Keep Browser **Open in panel** plus **Close** controls. - Add a Computer Use profile with **Close** only and ignored activation. - Make Browser tool activity and CUA snapshot-call starts claim their source. @@ -27,90 +29,112 @@ Proposal only. All implementation tasks remain open. - Prevent background host/session claims from preempting the foreground session. - Add shared run-scoped dismissal. -- [ ] T04 - Add the narrow Computer Use MCP observer +- [x] T04 - Add the narrow Computer Use MCP observer - Inject `started`, `completed`, and `failed` callbacks into `ToolManager`. - Use resolved `ownerPluginId`/`sourceId`, original tool name, and prepared arguments. - Observe only actual invocations after permission resolution. - Preserve tool response, abort, progress, and error semantics. -- [ ] T05 - Implement Computer Use target and run state +- [x] T05 - Implement Computer Use target and run state - Add `ComputerUsePreviewPresenter`. - Track host, session, run, tool call, `pid`, `windowId`, epoch, claim, and dismissal. - Remove the prior presentation on a new run or target. - Reject stale and out-of-order terminal results. -- [ ] T06 - Implement the bounded snapshot pipeline +- [x] T06 - Implement the bounded snapshot pipeline - Accept successful `get_window_state` PNG/JPEG inline images only. - Validate base64, input size, decoded dimensions, and output size. - Resize within 480 x 300 without upscaling and encode JPEG quality 72. - Keep one transform in flight and one latest pending result. - Keep frames memory-only and logs pixel-free. -- [ ] T07 - Add typed Computer Use preview contracts +- [x] T07 - Add typed Computer Use preview contracts - Add `computerUse.setPreviewMode`. - Add `computerUse.dismissPreview`. - Add Canvas-only `computerUse.preview.frame`. - Expose routes/events through the typed preload client. - Validate route sender and active session in main. -- [ ] T08 - Add renderer lifecycle coordination +- [x] T08 - Add renderer lifecycle coordination - Mount `AgentComputerUsePiP.vue` beside the Browser PiP controller. - Derive active session, working state, route, and focus from existing stores/clients. - Send `eligible`, `suspended`, and `stopped` transitions. - Keep native mode headless and free of image events. -- [ ] T09 - Add the Canvas fallback UI +- [x] T09 - Add the Canvas fallback UI - Render the latest valid snapshot with one **Close** button. - Make non-control points draggable without forwarding target input. - Retain old pixels until a new frame is fully decoded. - Reject stale frame sequences and release renderer resources on teardown. - Reuse `common.close` without adding new copy. -- [ ] T10 - Complete lifecycle and privacy cleanup +- [x] T10 - Complete lifecycle and privacy cleanup - Hide on host blur/hide/minimize and inactive route/session. - Remove on terminal run/session, target change, host close, and shutdown. - Ensure another host/session can never see retained pixels. - Ensure no disk, settings, telemetry, crash metadata, or generic MCP event stores frame bytes. -- [ ] T11 - Add main-process regression coverage +- [x] T11 - Add main-process regression coverage - Test shared coordinator startup/shutdown, toolbar switching, arbitration, and dismissal. - Test Browser activation, close, capture refresh, and panel handoff after extraction. - Test CUA recognition, permission paths, run identity, target changes, and stale results. - Test valid, malformed, unsupported, oversized, failed, and aborted image results. - Test native/fallback routing and latest-wins scheduling. + - Coordinator, Browser extraction, trust/permission/run propagation, native/fallback, + PNG/JPEG, malformed/unsupported/oversized/dimension validation, failed/aborted calls, target + epochs, dismissal, and latest-wins are covered. -- [ ] T12 - Add renderer regression coverage +- [x] T12 - Add renderer regression coverage - Test native headless mode and absence of frame subscriptions. - Test Canvas first frame, frame retention, sequence rejection, drag, and close. - Test focus, minimize, session switch, terminal state, and unmount cleanup. - Verify Computer Use exposes no expand/open-panel behavior. - -- [ ] T13 - Verify performance budgets - - Confirm zero additional operating-system captures and zero idle polling. + - Native headless mode, first Canvas frame, frame retention, target epoch/sequence rejection, + close-only controls, focus suspend/resume, terminal cleanup, drag, session switch, unmount, and + Browser supersession are covered. + +- [x] T13 - Add PiP-only post-click refresh + - After an eligible successful exact CUA `click`, asynchronously invoke one private + `get_window_state({ pid, window_id })`. + - Gate the call on trusted plugin ownership, valid session/run/target identity, active PiP, and + non-dismissed presenter state. + - Route the private result only through the preview observer; never publish, cache, or return it + to the Agent. + - Preserve click response, latency, permission, abort, and failure behavior. + - Cover successful isolation plus failed, untrusted, invalid-target, and inactive-preview paths. + +- [ ] T14 - Verify performance budgets + - Confirm at most one private capture per eligible successful `click` and zero idle polling. - Confirm one transform in flight with no unbounded queue. - Measure p95 valid-result-to-visible latency against the 150 ms budget. - Keep NativeKit slow-push warning behavior at 25 ms. - Confirm native mode sends zero image bytes to the renderer. -- [ ] T14 - Run packaged platform QA +- [ ] T15 - Run packaged platform QA - Verify one NativeKit native-overlay runtime. - Verify one renderer Canvas-fallback runtime. - Exercise host focus/minimize/restore and target-app foreground behavior. - Exercise Browser/Computer source transitions and toolbar profile changes. - Exercise close, later run, route/session switch, and app shutdown. -- [ ] T15 - Run repository verification +- [ ] T16 - Run repository verification - Run `pnpm run format`. - Run `pnpm run i18n`. - Run `pnpm run lint`. - Run `pnpm run typecheck`. - Run focused main and renderer tests, then the complete affected suites. + - Partial: format, i18n, lint, typecheck, focused suites, and all 1,625 renderer tests pass. The + main suite passes 5,383 tests and fails only + `ModelConfigHelper > Configuration Priority > uses provider metadata until a user config + overrides it`; the unchanged provider test also fails in isolation. -- [ ] T16 - Close implementation documentation +- [ ] T17 - Close implementation documentation - Update the proposal status and record deviations. - Update the Browser NativeKit architecture with the implemented shared-owner contract. - Mark only verified tasks complete. - Include BEFORE/AFTER ASCII layouts and platform QA status in the PR. + - Partial: implementation status, deviations, linked Browser architecture, verified tasks, ASCII + layouts, and open QA are recorded. PR handoff remains open. ## Implementation Order @@ -118,12 +142,13 @@ Proposal only. All implementation tasks remain open. 2. T04-T06 connect trusted CUA results to a bounded main-process snapshot pipeline. 3. T07-T09 add typed renderer coordination and fallback UI. 4. T10-T12 harden lifecycle, privacy, and regressions. -5. T13-T16 measure, validate, and close documentation. +5. T13 adds event-driven post-click refresh. +6. T14-T17 measure, validate, and close documentation. ## Done Definition -- A current successful CUA `get_window_state` image can show one latest-snapshot PiP without any - extra capture or polling. +- A current successful CUA `get_window_state` image can show one latest-snapshot PiP, and an + eligible successful `click` can schedule one private PiP-only refresh without idle polling. - Browser and Computer Use safely share one NativeKit owner and retain source-specific controls. - Computer Use PiP has only **Close**, is read-only, and never affects the Agent or target application. diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index 5b1cd8210..03e3e0d8c 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -57,6 +57,8 @@ import { TrayPresenter } from '../desktop/tray' import { OAuthService } from '../provider/auth' import { FloatingButtonPresenter } from '../desktop/floatingButton' import { YoBrowserPresenter } from '../desktop/browser/YoBrowserPresenter' +import { ComputerUsePreviewPresenter } from '@/desktop/computerUse/ComputerUsePreviewPresenter' +import { AgentPreviewCoordinator } from '@/desktop/preview/AgentPreviewCoordinator' import { KnowledgeService } from '../knowledge' import { WorkspaceService } from '../workspace' import { FileWatcherService } from '../platform/fileWatcher' @@ -262,6 +264,8 @@ export async function createMainProcessControl(dependencies: { let toolService: ToolServicePort let deepChatAgentHarness: DeepChatAgentHarness let yoBrowserPresenter: IYoBrowserPresenter + let computerUsePreviewPresenter: ComputerUsePreviewPresenter + let agentPreviewCoordinator: AgentPreviewCoordinator let dialogService: DialogServicePort let skillService: SkillServicePort let skillSyncService: SkillSyncServicePort @@ -610,7 +614,16 @@ export async function createMainProcessControl(dependencies: { notificationService = new NotificationService(desktopSettings, publishDeepchatEvent) trayPresenter = new TrayPresenter(desktopSettings, windowPresenter) dialogService = new DialogService(publishDeepchatEvent) - yoBrowserPresenter = new YoBrowserPresenter(windowPresenter, publishDeepchatEvent) + agentPreviewCoordinator = new AgentPreviewCoordinator() + yoBrowserPresenter = new YoBrowserPresenter( + windowPresenter, + publishDeepchatEvent, + agentPreviewCoordinator + ) + computerUsePreviewPresenter = new ComputerUsePreviewPresenter( + windowPresenter, + agentPreviewCoordinator + ) // Define the storage root for built-in knowledge databases. const dbDir = path.join(app.getPath('userData'), 'app_db') @@ -651,7 +664,8 @@ export async function createMainProcessControl(dependencies: { providerRuntime, () => deepChatAgentHarness.refreshToolRegistry(), publishDeepchatEvent, - (data) => deviceService.cacheImage(data) + (data) => deviceService.cacheImage(data), + computerUsePreviewPresenter ) const deeplinkActions = createDeeplinkActions({ window: windowPresenter, @@ -1631,7 +1645,13 @@ export async function createMainProcessControl(dependencies: { } await runDestroyStep('pluginService.shutdown', () => pluginService.shutdown()) await runDestroyStep('mcpService.shutdown', () => mcpService.shutdown()) + await runDestroyStep('computerUsePreviewPresenter.shutdown', () => + computerUsePreviewPresenter.shutdown() + ) await runDestroyStep('yoBrowserPresenter.shutdown', () => yoBrowserPresenter.shutdown()) + await runDestroyStep('agentPreviewCoordinator.shutdown', () => + agentPreviewCoordinator.shutdown() + ) await runDestroyStep('floatingButtonPresenter.destroy', () => floatingButtonPresenter.destroy()) await runDestroyStep('windowPresenter.destroyFloatingChatWindow', () => windowPresenter.destroyFloatingChatWindow() @@ -1743,6 +1763,8 @@ export async function createMainProcessControl(dependencies: { windowPresenter, shortcutPresenter, browserPresenter: yoBrowserPresenter, + computerUsePreviewPresenter, + desktopSessionBinding, tabPresenter, dialogService, settings: desktopSettings, diff --git a/src/main/desktop/browser/YoBrowserPresenter.ts b/src/main/desktop/browser/YoBrowserPresenter.ts index e81eb3de2..27b3c31ba 100644 --- a/src/main/desktop/browser/YoBrowserPresenter.ts +++ b/src/main/desktop/browser/YoBrowserPresenter.ts @@ -23,10 +23,10 @@ import { import type { DownloadInfo } from '@shared/types/browser' import type { IWindowPresenter, IYoBrowserPresenter } from '@shared/types/desktop' import { - AgentBrowserNativeOverlay, - type AgentBrowserNativeOverlayAction, - type AgentBrowserNativeOverlayTarget -} from './AgentBrowserNativeOverlay' + AgentPreviewCoordinator, + type AgentPreviewAction, + type AgentPreviewTarget +} from '@/desktop/preview/AgentPreviewCoordinator' import { BrowserTab as BrowserPage } from './BrowserTab' import { BrowserProfileImportService } from './BrowserProfileImportService' import { CDPManager } from './CDPManager' @@ -69,6 +69,7 @@ type SessionBrowserState = { previewTimer: ReturnType | null previewCapture: Promise | null previewEpoch: number + previewClaimSequence: number previewSequence: number previewBurstUntil: number } @@ -101,22 +102,25 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { ) private browserDataMutationActive = false private readonly windowPresenter: IWindowPresenter - private readonly nativeOverlay: AgentBrowserNativeOverlay + private readonly previewCoordinator: AgentPreviewCoordinator + private readonly unregisterPreviewHandler: () => void readonly toolHandler: YoBrowserToolHandler constructor( windowPresenter: IWindowPresenter, - private readonly publishEvent: DeepchatEventPublisher + private readonly publishEvent: DeepchatEventPublisher, + previewCoordinator: AgentPreviewCoordinator ) { this.windowPresenter = windowPresenter - this.nativeOverlay = new AgentBrowserNativeOverlay((action, target) => { + this.previewCoordinator = previewCoordinator + this.unregisterPreviewHandler = previewCoordinator.register('browser', (action, target) => { this.handleNativePreviewAction(action, target) }) this.toolHandler = new YoBrowserToolHandler(this) } async initialize(): Promise { - await this.nativeOverlay.initialize() + await this.previewCoordinator.initialize() } async getBrowserStatus(sessionId: string): Promise { @@ -192,7 +196,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return false } - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) this.detachOtherSessionBrowsers(hostWindowId, sessionId) @@ -284,9 +288,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { if (runId != null && state.agentRunId != null && runId !== state.agentRunId) { return { updated: false, surface: 'none' } } - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.previewSurface = 'none' return { updated: true, surface: 'none' } } @@ -299,7 +303,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return { updated: false, surface: 'none' } } - await this.nativeOverlay.initialize() + await this.previewCoordinator.initialize() const targetWindow = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) if (hostWindowId != null && (!targetWindow || targetWindow.isDestroyed())) { @@ -307,7 +311,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } if (mode === 'rendering') { - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) } await this.stopPreviewCapture(state) if (mode === 'capturing' && (!targetWindow || state.visible)) { @@ -322,34 +326,49 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewEpoch += 1 if (mode === 'rendering') { - this.nativeOverlay.hide(this.nativeTargetRef(state)) - state.previewSurface = this.nativeOverlay.isAvailable() ? 'native-overlay' : 'renderer-canvas' + this.previewCoordinator.hide(this.nativeTargetRef(state)) + state.previewSurface = this.previewCoordinator.isAvailable() + ? 'native-overlay' + : 'renderer-canvas' return { updated: true, surface: state.previewSurface } } if (!targetWindow) { return { updated: false, surface: 'none' } } - const nativeTarget = this.nativeTarget(state) - state.previewSurface = - nativeTarget && this.nativeOverlay.prepare(nativeTarget, targetWindow) - ? 'native-overlay' - : 'renderer-canvas' - - this.schedulePreviewCapture(state, 0, state.previewEpoch) + this.resumeClaimedPreview(state) return { updated: true, surface: state.previewSurface } } + dismissPreview(sessionId: string, runId: string): boolean { + const state = this.sessionBrowsers.get(sessionId) + if (!state || state.agentRunId !== runId) { + return false + } + + const dismissed = this.previewCoordinator.dismiss({ + source: 'browser', + sessionId, + runId + }) + if (dismissed) { + state.previewSurface = 'none' + void this.stopPreviewCapture(state) + this.emitPreviewSurface(state) + } + return dismissed + } + async destroySessionBrowser(sessionId: string): Promise { const state = this.sessionBrowsers.get(sessionId) if (!state) { return } - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) await this.releasePreviewHost(state) await this.detachSessionBrowser(sessionId) state.page.destroy() @@ -517,7 +536,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { for (const sessionId of Array.from(this.sessionBrowsers.keys())) { await this.destroySessionBrowser(sessionId) } - this.nativeOverlay.shutdown() + this.unregisterPreviewHandler() } private ensureSessionBrowserState(sessionId: string): SessionBrowserState { @@ -557,6 +576,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { previewTimer: null, previewCapture: null, previewEpoch: 0, + previewClaimSequence: 0, previewSequence: 0, previewBurstUntil: 0 } @@ -675,7 +695,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) this.disposePreviewHost(state) state.page.destroy() state.overlay.destroy() @@ -964,13 +984,31 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { agentRunId?: string ): void { if (activitySource === 'agent') { + const previousRunId = state.agentRunId + const nextRunId = agentRunId?.trim() || previousRunId || nanoid(12) + if (previousRunId && previousRunId !== nextRunId) { + this.previewCoordinator.removeTarget({ + source: 'browser', + sessionId: state.sessionId, + runId: previousRunId + }) + state.previewEpoch += 1 + } state.owner = 'agent' - state.agentRunId = agentRunId?.trim() || state.agentRunId || nanoid(12) + state.agentRunId = nextRunId + state.previewClaimSequence = this.previewCoordinator.claim({ + source: 'browser', + sessionId: state.sessionId, + runId: nextRunId + }) + this.resumeClaimedPreview(state) return } + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.owner = 'user' state.agentRunId = undefined + state.previewClaimSequence = 0 } private emitWindowUpdated(sessionId: string): void { @@ -1320,7 +1358,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewHost = host host.once('closed', () => { if (state.previewHost === host) { - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.previewHost = null state.previewMode = 'stopped' state.previewSurface = 'none' @@ -1398,6 +1436,44 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { host.destroy() } + private resumeClaimedPreview(state: SessionBrowserState): void { + if (state.previewMode !== 'capturing' || state.previewTargetWindowId == null) { + return + } + const target = this.nativeTarget(state) + const host = BrowserWindow.fromId(state.previewTargetWindowId) + if (!target || !host || host.isDestroyed() || state.visible) { + return + } + + state.previewSurface = this.previewCoordinator.prepare(target, host) + this.emitPreviewSurface(state) + if ( + state.previewSurface !== 'none' && + state.previewTimer == null && + state.previewCapture == null + ) { + this.schedulePreviewCapture(state, 0, state.previewEpoch) + } + } + + private emitPreviewSurface(state: SessionBrowserState): void { + if (state.previewTargetWindowId == null || !state.agentRunId) { + return + } + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.surface.changed', { + windowId: state.previewTargetWindowId, + sessionId: state.sessionId, + runId: state.agentRunId, + surface: state.previewSurface, + version: Date.now() + }) + ) + } + private schedulePreviewCapture(state: SessionBrowserState, delayMs: number, epoch: number): void { if (state.previewMode !== 'capturing' || state.previewEpoch !== epoch) { return @@ -1409,6 +1485,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { void capture.finally(() => { if (state.previewCapture === capture) { state.previewCapture = null + if (state.previewTimer == null && state.previewMode === 'capturing') { + this.resumeClaimedPreview(state) + } } }) }, delayMs) @@ -1416,11 +1495,14 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } private async capturePreviewFrame(state: SessionBrowserState, epoch: number): Promise { + const initialTarget = this.nativeTarget(state) if ( state.previewMode !== 'capturing' || state.previewEpoch !== epoch || state.previewTargetWindowId == null || !state.agentRunId || + !initialTarget || + !this.previewCoordinator.isCurrent(initialTarget) || state.page.contents.isDestroyed() ) { return @@ -1438,24 +1520,15 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { if (data.byteLength <= PREVIEW_MAX_BYTES) { if (state.previewSurface === 'native-overlay') { const target = this.nativeTarget(state) - if (target && target.captureEpoch === epoch) { - this.nativeOverlay.present(target, data) + if (target && target.epoch === epoch && !this.previewCoordinator.present(target, data)) { + const host = BrowserWindow.fromId(target.windowId) + state.previewSurface = + host && !host.isDestroyed() ? this.previewCoordinator.prepare(target, host) : 'none' + this.emitPreviewSurface(state) } - } else if (state.previewSurface === 'renderer-canvas') { - state.previewSequence += 1 - this.windowPresenter.sendToWindow( - state.previewTargetWindowId, - DEEPCHAT_EVENT_CHANNEL, - createDeepchatEventEnvelope('browser.preview.frame', { - sessionId: state.sessionId, - runId: state.agentRunId, - sequence: state.previewSequence, - ...PREVIEW_FRAME, - mimeType: 'image/jpeg', - data, - timestamp: Date.now() - }) - ) + } + if (state.previewSurface === 'renderer-canvas') { + this.sendRendererPreviewFrame(state, data) } } } catch (error) { @@ -1466,6 +1539,10 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } const active = state.page.contents.isLoading() || Date.now() < state.previewBurstUntil + const nextTarget = this.nativeTarget(state) + if (!nextTarget || !this.previewCoordinator.isCurrent(nextTarget)) { + return + } this.schedulePreviewCapture( state, active ? PREVIEW_ACTIVE_INTERVAL_MS : PREVIEW_IDLE_INTERVAL_MS, @@ -1473,45 +1550,78 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { ) } - private nativeTarget(state: SessionBrowserState): AgentBrowserNativeOverlayTarget | null { - if (state.previewTargetWindowId == null || !state.agentRunId) { + private nativeTarget(state: SessionBrowserState): AgentPreviewTarget | null { + if ( + state.previewTargetWindowId == null || + !state.agentRunId || + state.previewClaimSequence <= 0 + ) { return null } return { + source: 'browser', windowId: state.previewTargetWindowId, sessionId: state.sessionId, runId: state.agentRunId, - captureEpoch: state.previewEpoch + epoch: state.previewEpoch, + claimSequence: state.previewClaimSequence } } + private sendRendererPreviewFrame(state: SessionBrowserState, data: Buffer): void { + if (state.previewTargetWindowId == null || !state.agentRunId) { + return + } + state.previewSequence += 1 + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.frame', { + sessionId: state.sessionId, + runId: state.agentRunId, + sequence: state.previewSequence, + ...PREVIEW_FRAME, + mimeType: 'image/jpeg', + data, + timestamp: Date.now() + }) + ) + } + private nativeTargetRef(state: SessionBrowserState): { + source: 'browser' sessionId: string runId?: string } { return { + source: 'browser', sessionId: state.sessionId, ...(state.agentRunId ? { runId: state.agentRunId } : {}) } } - private handleNativePreviewAction( - action: AgentBrowserNativeOverlayAction, - target: AgentBrowserNativeOverlayTarget - ): void { + private handleNativePreviewAction(action: AgentPreviewAction, target: AgentPreviewTarget): void { const state = this.sessionBrowsers.get(target.sessionId) if ( !state || - state.previewSurface !== 'native-overlay' || + target.source !== 'browser' || state.previewTargetWindowId !== target.windowId || state.agentRunId !== target.runId || - state.previewEpoch !== target.captureEpoch + state.previewEpoch !== target.epoch || + state.previewClaimSequence !== target.claimSequence ) { return } - state.previewMode = 'rendering' + state.previewSurface = 'none' void this.stopPreviewCapture(state) + this.emitPreviewSurface(state) + + if (action === 'superseded') { + return + } + + state.previewMode = 'rendering' if (action === 'activate') { this.windowPresenter.show(target.windowId, true) diff --git a/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts new file mode 100644 index 000000000..65caf480a --- /dev/null +++ b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts @@ -0,0 +1,590 @@ +import { BrowserWindow } from 'electron' +import { performance } from 'node:perf_hooks' +import sharp from 'sharp' +import { DEEPCHAT_EVENT_CHANNEL } from '@shared/contracts/channels' +import { createDeepchatEventEnvelope } from '@shared/contracts/events' +import logger from '@shared/logger' +import type { + ComputerUsePreviewMode, + ComputerUsePreviewModeResult, + ComputerUsePreviewSurface +} from '@shared/types/computerUse' +import type { IComputerUsePreviewPresenter, IWindowPresenter } from '@shared/types/desktop' +import type { MCPImageContent, MCPToolResponse } from '@shared/types/mcp' +import type { ComputerUsePreviewCall, ComputerUsePreviewObserver } from '@/mcp/toolManager' +import { + AgentPreviewCoordinator, + type AgentPreviewAction, + type AgentPreviewTarget +} from '@/desktop/preview/AgentPreviewCoordinator' + +type ComputerUseFrame = { + data: Buffer + width: number + height: number + sequence: number + timestamp: number +} + +type PendingSnapshot = { + call: ComputerUsePreviewCall + image: MCPImageContent + epoch: number + claimSequence: number +} + +type ComputerUsePreviewState = { + sessionId: string + mode: ComputerUsePreviewMode + hostWindowId: number | null + runId: string | null + toolCallId: string | null + toolCallEpoch: number + pid: number | null + targetWindowId: number | null + epoch: number + claimSequence: number + dismissedRunId: string | null + surface: ComputerUsePreviewSurface + frame: ComputerUseFrame | null + nextFrameSequence: number + transformActive: boolean + pendingSnapshot: PendingSnapshot | null +} + +const FRAME_MAX_WIDTH = 480 +const FRAME_MAX_HEIGHT = 300 +const FRAME_MAX_BYTES = 512 * 1024 +const INPUT_MAX_BYTES = 16 * 1024 * 1024 +const INPUT_MAX_DIMENSION = 8192 +const INPUT_MAX_PIXELS = INPUT_MAX_DIMENSION * INPUT_MAX_DIMENSION +const JPEG_QUALITY = 72 +const SLOW_TRANSFORM_WARNING_MS = 150 +const WARNING_INTERVAL_MS = 60_000 +const SUPPORTED_MIME_TYPES = new Set(['image/jpeg', 'image/png']) + +export class ComputerUsePreviewPresenter + implements IComputerUsePreviewPresenter, ComputerUsePreviewObserver +{ + private readonly states = new Map() + private readonly unregisterPreviewHandler: () => void + private lastValidationWarningAt = 0 + private lastSlowTransformWarningAt = 0 + + constructor( + private readonly windowPresenter: IWindowPresenter, + private readonly previewCoordinator: AgentPreviewCoordinator + ) { + this.unregisterPreviewHandler = previewCoordinator.register('computer-use', (action, target) => + this.handlePreviewAction(action, target) + ) + } + + shouldCaptureAfterClick(call: ComputerUsePreviewCall): boolean { + const state = this.states.get(call.conversationId) + const pid = this.readPositiveInteger(call.args.pid) + const targetWindowId = this.readPositiveInteger(call.args.window_id) + return Boolean( + call.toolName === 'click' && + state && + pid != null && + targetWindowId != null && + state.mode === 'eligible' && + state.hostWindowId != null && + state.runId === call.runId && + state.pid === pid && + state.targetWindowId === targetWindowId && + state.dismissedRunId !== call.runId + ) + } + + started(call: ComputerUsePreviewCall): void { + if (call.toolName !== 'get_window_state') { + return + } + const pid = this.readPositiveInteger(call.args.pid) + const targetWindowId = this.readPositiveInteger(call.args.window_id) + if (pid == null || targetWindowId == null) { + return + } + + const state = this.states.get(call.conversationId) ?? this.createState(call.conversationId) + const targetChanged = + state.runId !== call.runId || state.pid !== pid || state.targetWindowId !== targetWindowId + if (targetChanged) { + if (state.runId) { + this.previewCoordinator.removeTarget({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + state.surface = 'none' + if (state.runId !== call.runId) { + state.dismissedRunId = null + } + } + + state.runId = call.runId + state.toolCallId = call.toolCallId + state.toolCallEpoch = state.epoch + state.pid = pid + state.targetWindowId = targetWindowId + state.claimSequence = this.previewCoordinator.claim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: call.runId + }) + + if (state.mode === 'eligible') { + if (targetChanged || state.surface === 'none') { + this.presentCurrent(state) + } else { + this.prepareCurrent(state) + } + } + } + + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void { + const state = this.states.get(call.conversationId) + if ( + !state || + result.isError === true || + state.dismissedRunId === call.runId || + state.runId !== call.runId || + state.toolCallId !== call.toolCallId || + state.toolCallEpoch !== state.epoch + ) { + return + } + + const image = this.readInlineImage(result) + if (!image) { + return + } + this.enqueueSnapshot(state, { + call, + image, + epoch: state.epoch, + claimSequence: state.claimSequence + }) + } + + failed(_call: ComputerUsePreviewCall, _error: unknown): void { + // Keep the last valid frame for the same run and target. + } + + async setPreviewMode( + sessionId: string, + mode: ComputerUsePreviewMode, + hostWindowId?: number + ): Promise { + const normalizedSessionId = sessionId.trim() + if (!normalizedSessionId) { + return { updated: false, surface: 'none' } + } + + if (mode === 'stopped') { + const state = this.states.get(normalizedSessionId) + if ( + !state || + (hostWindowId != null && state.hostWindowId != null && state.hostWindowId !== hostWindowId) + ) { + return { updated: false, surface: 'none' } + } + state.epoch += 1 + state.mode = 'stopped' + if (state.runId) { + this.previewCoordinator.releaseClaim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + this.states.delete(normalizedSessionId) + return { updated: true, surface: 'none' } + } + + const host = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) + if (!host || host.isDestroyed()) { + return { updated: false, surface: 'none' } + } + + this.stopOtherSessionsForHost(normalizedSessionId, host.id) + await this.previewCoordinator.initialize() + const state = this.states.get(normalizedSessionId) ?? this.createState(normalizedSessionId) + state.hostWindowId = host.id + state.mode = mode + + if (mode === 'suspended') { + this.previewCoordinator.hide({ + source: 'computer-use', + sessionId: normalizedSessionId, + ...(state.runId ? { runId: state.runId } : {}) + }) + this.setSurface(state, 'none') + return { updated: true, surface: 'none' } + } + + const surface = this.presentCurrent(state, true) + return { updated: true, surface } + } + + dismissPreview(sessionId: string, runId: string): boolean { + const state = this.states.get(sessionId) + if (!state || state.runId !== runId) { + return false + } + + const dismissed = this.previewCoordinator.dismiss({ + source: 'computer-use', + sessionId, + runId + }) + if (dismissed) { + state.dismissedRunId = runId + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + this.setSurface(state, 'none') + } + return dismissed + } + + shutdown(): void { + for (const state of this.states.values()) { + if (state.runId) { + this.previewCoordinator.releaseClaim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + } + this.states.clear() + this.unregisterPreviewHandler() + } + + private createState(sessionId: string): ComputerUsePreviewState { + const state: ComputerUsePreviewState = { + sessionId, + mode: 'stopped', + hostWindowId: null, + runId: null, + toolCallId: null, + toolCallEpoch: 0, + pid: null, + targetWindowId: null, + epoch: 0, + claimSequence: 0, + dismissedRunId: null, + surface: 'none', + frame: null, + nextFrameSequence: 0, + transformActive: false, + pendingSnapshot: null + } + this.states.set(sessionId, state) + return state + } + + private readPositiveInteger(value: unknown): number | null { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null + } + + private readInlineImage(result: MCPToolResponse): MCPImageContent | null { + if (!Array.isArray(result.content)) { + return null + } + for (const item of result.content) { + if ( + item.type === 'image' && + typeof item.data === 'string' && + typeof item.mimeType === 'string' && + SUPPORTED_MIME_TYPES.has(item.mimeType.toLowerCase()) + ) { + return item + } + } + return null + } + + private enqueueSnapshot(state: ComputerUsePreviewState, snapshot: PendingSnapshot): void { + if (state.transformActive) { + state.pendingSnapshot = snapshot + return + } + state.transformActive = true + void this.transformSnapshot(state, snapshot).finally(() => { + const current = this.states.get(snapshot.call.conversationId) + if (current !== state) { + return + } + state.transformActive = false + const pending = state.pendingSnapshot + state.pendingSnapshot = null + if (pending) { + this.enqueueSnapshot(state, pending) + } + }) + } + + private async transformSnapshot( + owner: ComputerUsePreviewState, + snapshot: PendingSnapshot + ): Promise { + const startedAt = performance.now() + try { + const source = this.decodeBase64(snapshot.image.data) + const image = sharp(source, { + failOn: 'warning', + limitInputPixels: INPUT_MAX_PIXELS + }) + const metadata = await image.metadata() + const mimeType = snapshot.image.mimeType.toLowerCase() + const expectedFormat = mimeType === 'image/png' ? 'png' : 'jpeg' + if ( + metadata.format !== expectedFormat || + !metadata.width || + !metadata.height || + metadata.width > INPUT_MAX_DIMENSION || + metadata.height > INPUT_MAX_DIMENSION + ) { + throw new Error('unsupported image metadata') + } + + const transformed = await image + .resize({ + width: FRAME_MAX_WIDTH, + height: FRAME_MAX_HEIGHT, + fit: 'inside', + withoutEnlargement: true + }) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer({ resolveWithObject: true }) + if ( + transformed.data.byteLength === 0 || + transformed.data.byteLength > FRAME_MAX_BYTES || + transformed.info.width <= 0 || + transformed.info.height <= 0 + ) { + throw new Error('invalid transformed image') + } + + const state = this.states.get(snapshot.call.conversationId) + if ( + state !== owner || + state.runId !== snapshot.call.runId || + state.toolCallId !== snapshot.call.toolCallId || + state.epoch !== snapshot.epoch || + state.claimSequence !== snapshot.claimSequence || + state.dismissedRunId === snapshot.call.runId + ) { + return + } + + state.nextFrameSequence += 1 + state.frame = { + data: transformed.data, + width: transformed.info.width, + height: transformed.info.height, + sequence: state.nextFrameSequence, + timestamp: Date.now() + } + if (state.mode === 'eligible') { + this.presentCurrent(state) + } + } catch (error) { + this.warnValidation(snapshot.call, error) + } finally { + const durationMs = performance.now() - startedAt + if ( + durationMs > SLOW_TRANSFORM_WARNING_MS && + Date.now() - this.lastSlowTransformWarningAt >= WARNING_INTERVAL_MS + ) { + this.lastSlowTransformWarningAt = Date.now() + logger.warn('[ComputerUsePreview] Slow snapshot transform', { + durationMs: Math.round(durationMs) + }) + } + } + } + + private decodeBase64(data: string): Buffer { + const normalized = data.trim() + const maxBase64Length = Math.ceil((INPUT_MAX_BYTES * 4) / 3) + 4 + if ( + !normalized || + normalized.length > maxBase64Length || + normalized.length % 4 === 1 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(normalized) + ) { + throw new Error('invalid base64 image') + } + const buffer = Buffer.from(normalized, 'base64') + if (buffer.byteLength === 0 || buffer.byteLength > INPUT_MAX_BYTES) { + throw new Error('image input exceeds limit') + } + return buffer + } + + private prepareCurrent( + state: ComputerUsePreviewState, + announceSurface = false + ): ComputerUsePreviewSurface { + const target = this.previewTarget(state) + const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) + if (!target || !host || host.isDestroyed() || state.mode !== 'eligible') { + return this.setSurface(state, 'none', announceSurface) + } + return this.setSurface(state, this.previewCoordinator.prepare(target, host), announceSurface) + } + + private presentCurrent( + state: ComputerUsePreviewState, + announceSurface = false + ): ComputerUsePreviewSurface { + const surface = this.prepareCurrent(state, announceSurface) + const frame = state.frame + const target = this.previewTarget(state) + if (surface === 'none' || !frame || !target || state.hostWindowId == null) { + return surface + } + + if (surface === 'native-overlay') { + if (this.previewCoordinator.present(target, frame.data)) { + return surface + } + const host = BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + return this.setSurface(state, 'none') + } + const fallbackSurface = this.previewCoordinator.prepare(target, host) + this.setSurface(state, fallbackSurface) + if (fallbackSurface !== 'renderer-canvas') { + return fallbackSurface + } + } + + this.windowPresenter.sendToWindow( + state.hostWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('computerUse.preview.frame', { + sessionId: state.sessionId, + runId: target.runId, + epoch: state.epoch, + sequence: frame.sequence, + width: frame.width, + height: frame.height, + mimeType: 'image/jpeg', + data: frame.data, + timestamp: frame.timestamp + }) + ) + return state.surface + } + + private previewTarget(state: ComputerUsePreviewState): AgentPreviewTarget | null { + if ( + state.hostWindowId == null || + !state.runId || + state.pid == null || + state.targetWindowId == null || + state.claimSequence <= 0 || + state.dismissedRunId === state.runId + ) { + return null + } + return { + source: 'computer-use', + windowId: state.hostWindowId, + sessionId: state.sessionId, + runId: state.runId, + epoch: state.epoch, + claimSequence: state.claimSequence + } + } + + private setSurface( + state: ComputerUsePreviewState, + surface: ComputerUsePreviewSurface, + forceEvent = false + ): ComputerUsePreviewSurface { + if (state.surface === surface && !forceEvent) { + return surface + } + state.surface = surface + if (state.hostWindowId != null && state.runId) { + this.windowPresenter.sendToWindow( + state.hostWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('computerUse.preview.surface.changed', { + windowId: state.hostWindowId, + sessionId: state.sessionId, + runId: state.runId, + epoch: state.epoch, + surface, + version: Date.now() + }) + ) + } + return surface + } + + private stopOtherSessionsForHost(sessionId: string, hostWindowId: number): void { + for (const [otherSessionId, state] of this.states) { + if (otherSessionId === sessionId || state.hostWindowId !== hostWindowId) { + continue + } + state.epoch += 1 + state.mode = 'stopped' + state.frame = null + state.pendingSnapshot = null + if (state.runId) { + this.previewCoordinator.releaseClaim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + this.states.delete(otherSessionId) + } + } + + private handlePreviewAction(action: AgentPreviewAction, target: AgentPreviewTarget): void { + if (target.source !== 'computer-use') { + return + } + const state = this.states.get(target.sessionId) + if ( + !state || + state.runId !== target.runId || + state.epoch !== target.epoch || + state.claimSequence !== target.claimSequence + ) { + return + } + + if (action === 'dismiss') { + state.dismissedRunId = target.runId + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + } + this.setSurface(state, 'none') + } + + private warnValidation(call: ComputerUsePreviewCall, error: unknown): void { + if (Date.now() - this.lastValidationWarningAt < WARNING_INTERVAL_MS) { + return + } + this.lastValidationWarningAt = Date.now() + logger.warn('[ComputerUsePreview] Snapshot rejected', { + toolCallId: call.toolCallId, + reason: error instanceof Error ? error.message : String(error) + }) + } +} diff --git a/src/main/desktop/browser/AgentBrowserNativeOverlay.ts b/src/main/desktop/preview/AgentPreviewCoordinator.ts similarity index 62% rename from src/main/desktop/browser/AgentBrowserNativeOverlay.ts rename to src/main/desktop/preview/AgentPreviewCoordinator.ts index 1e85ad1aa..ab42c8824 100644 --- a/src/main/desktop/browser/AgentBrowserNativeOverlay.ts +++ b/src/main/desktop/preview/AgentPreviewCoordinator.ts @@ -4,25 +4,35 @@ import logger from '@shared/logger' type NativeKitOverlay = (typeof import('@zerob13/nativekit'))['overlay'] -export type AgentBrowserNativeOverlayTarget = { +export type AgentPreviewSource = 'browser' | 'computer-use' + +export type AgentPreviewTarget = { + source: AgentPreviewSource windowId: number sessionId: string runId: string - captureEpoch: number + epoch: number + claimSequence: number } -export type AgentBrowserNativeOverlayAction = 'activate' | 'dismiss' +export type AgentPreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' -type NativeOverlayActionHandler = ( - action: AgentBrowserNativeOverlayAction, - target: AgentBrowserNativeOverlayTarget -) => void +export type AgentPreviewAction = 'activate' | 'dismiss' | 'superseded' -type NativeOverlayTargetRef = { +type AgentPreviewTargetRef = { + source?: AgentPreviewSource sessionId: string runId?: string } +type AgentPreviewActionHandler = (action: AgentPreviewAction, target: AgentPreviewTarget) => void + +type AgentPreviewClaim = { + source: AgentPreviewSource + runId: string + sequence: number +} + type HostListeners = { focus: () => void blur: () => void @@ -48,25 +58,35 @@ const OPEN_PANEL_ICON_DATA_URL = const CLOSE_ICON_DATA_URL = 'data:image/png;base64,' + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAdklEQVQ4jbWTSwrAIAxE31G71F111d6z9DoWQSSKn1DbB64yM5gY4SM24AaMQmuSNnoyFxDS8fTxQhc9GSsKvRBpDslTsFeCU9RcVTtK6zhEbe5dddaaOkRtpnHteib/teAaAxu9TsFo2tMQu7pIy6u8/Jle8wCGAVmb4KsRPAAAAABJRU5ErkJggg==' -const TOOLBAR_OPTIONS: Parameters[0] = { + +const toolbarOptions = (source: AgentPreviewSource): Parameters[0] => ({ toolbar: { style: 'dark', - buttons: [ - { - id: OPEN_PANEL_CONTROL_ID, - imageData: OPEN_PANEL_ICON_DATA_URL, - tooltip: 'Open in side panel' - }, - { - id: CLOSE_CONTROL_ID, - imageData: CLOSE_ICON_DATA_URL, - tooltip: 'Close' - } - ] - } -} - -export class AgentBrowserNativeOverlay { + buttons: + source === 'browser' + ? [ + { + id: OPEN_PANEL_CONTROL_ID, + imageData: OPEN_PANEL_ICON_DATA_URL, + tooltip: 'Open in side panel' + }, + { + id: CLOSE_CONTROL_ID, + imageData: CLOSE_ICON_DATA_URL, + tooltip: 'Close' + } + ] + : [ + { + id: CLOSE_CONTROL_ID, + imageData: CLOSE_ICON_DATA_URL, + tooltip: 'Close' + } + ] + } +}) + +export class AgentPreviewCoordinator { private overlay: NativeKitOverlay | null = null private available = false private unavailable = false @@ -74,23 +94,30 @@ export class AgentBrowserNativeOverlay { private host: BrowserWindow | null = null private hostListeners: HostListeners | null = null private hostSyncTimer: ReturnType | null = null - private target: AgentBrowserNativeOverlayTarget | null = null + private target: AgentPreviewTarget | null = null private desiredVisible = false private overlayVisible: boolean | null = null private activeSessionId: string | null = null + private toolbarSource: AgentPreviewSource | null = null + private nextClaimSequence = 0 + private readonly claims = new Map() + private readonly dismissedRuns = new Map() + private readonly handlers = new Map() private lastSlowPushWarningAt = 0 private lastPushFailureWarningAt = 0 private listeningForDisplays = false private readonly handleActivate = () => { - this.emitAction('activate') + if (this.target?.source === 'browser') { + this.emitAction('activate') + } } private readonly handleControl = (controlId: string) => { - if (controlId === OPEN_PANEL_CONTROL_ID) { + if (controlId === OPEN_PANEL_CONTROL_ID && this.target?.source === 'browser') { this.emitAction('activate') } else if (controlId === CLOSE_CONTROL_ID) { - this.emitAction('dismiss') + this.dismissCurrent() } } @@ -98,7 +125,65 @@ export class AgentBrowserNativeOverlay { this.scheduleHostSync() } - constructor(private readonly onAction: NativeOverlayActionHandler) {} + register(source: AgentPreviewSource, handler: AgentPreviewActionHandler): () => void { + this.handlers.set(source, handler) + return () => { + if (this.handlers.get(source) === handler) { + this.handlers.delete(source) + } + } + } + + claim(input: { source: AgentPreviewSource; sessionId: string; runId: string }): number { + const sessionId = input.sessionId.trim() + const runId = input.runId.trim() + if (!sessionId || !runId) { + return 0 + } + + if (this.dismissedRuns.get(sessionId) !== runId) { + this.dismissedRuns.delete(sessionId) + } + + const sequence = ++this.nextClaimSequence + this.claims.set(sessionId, { + source: input.source, + runId, + sequence + }) + return sequence + } + + releaseClaim(target: AgentPreviewTargetRef): void { + const claim = this.claims.get(target.sessionId) + if ( + claim && + (target.source == null || claim.source === target.source) && + (target.runId == null || claim.runId === target.runId) + ) { + this.claims.delete(target.sessionId) + } + this.removeTarget(target) + } + + dismiss(target: AgentPreviewTargetRef): boolean { + const current = this.target + const claim = this.claims.get(target.sessionId) + const runId = target.runId ?? current?.runId ?? claim?.runId + if (!runId) { + return false + } + const currentOwnsRun = + current?.sessionId === target.sessionId && (target.runId == null || current.runId === runId) + const owner = currentOwnsRun ? current.source : claim?.source + if (target.source && owner !== target.source) { + return false + } + + this.dismissedRuns.set(target.sessionId, runId) + this.removeTarget({ sessionId: target.sessionId, runId }) + return true + } async initialize(): Promise { if (this.available) { @@ -119,32 +204,51 @@ export class AgentBrowserNativeOverlay { return this.available } - prepare(target: AgentBrowserNativeOverlayTarget, host: BrowserWindow): boolean { - if (!this.available || !this.overlay || host.isDestroyed()) { - return false + isCurrent(target: AgentPreviewTarget): boolean { + return this.matchesTarget(target) && this.matchesClaim(target) && !this.isDismissed(target) + } + + prepare(target: AgentPreviewTarget, host: BrowserWindow): AgentPreviewSurface { + if ( + host.isDestroyed() || + !this.matchesClaim(target) || + this.isDismissed(target) || + target.claimSequence <= 0 + ) { + return 'none' } + const previous = this.target const presentationChanged = - this.target != null && this.presentationId(this.target) !== this.presentationId(target) + previous != null && this.presentationId(previous) !== this.presentationId(target) if (presentationChanged) { this.desiredVisible = false this.setOverlayVisible(false) this.removeCurrentPresentation() - } - - if (!this.attachHost(host)) { - this.disable('host_attach_failed') - return false + this.target = null + this.handlers.get(previous.source)?.('superseded', { ...previous }) } this.target = { ...target } this.desiredVisible = true - return true + + if (!this.available || !this.overlay) { + return 'renderer-canvas' + } + if (!this.configureToolbar(target.source)) { + this.disableNative('toolbar_update_failed') + return 'renderer-canvas' + } + if (!this.attachHost(host)) { + this.disableNative('host_attach_failed') + return 'renderer-canvas' + } + return 'native-overlay' } - present(target: AgentBrowserNativeOverlayTarget, jpeg: Buffer): boolean { + present(target: AgentPreviewTarget, jpeg: Buffer): boolean { const overlay = this.overlay - if (!this.available || !overlay || !this.matchesTarget(target) || !this.host) { + if (!this.available || !overlay || !this.isCurrent(target) || !this.host) { return false } @@ -153,16 +257,17 @@ export class AgentBrowserNativeOverlay { const pushed = overlay.pushImage({ hostId: this.hostId(target.windowId), presentationId: this.presentationId(target), - sessionId: this.nativeSessionId(target.sessionId), + sessionId: this.nativeSessionId(target), imageData: `data:image/jpeg;base64,${jpeg.toString('base64')}` }) this.recordPushDuration(performance.now() - startedAt) if (!pushed) { this.warnFramePush(new Error('pushImage returned false')) + this.disableNative('frame_push_failed') return false } - const nativeSessionId = this.nativeSessionId(target.sessionId) + const nativeSessionId = this.nativeSessionId(target) if (this.activeSessionId !== nativeSessionId) { if (overlay.setActiveSession(nativeSessionId)) { this.activeSessionId = nativeSessionId @@ -178,20 +283,21 @@ export class AgentBrowserNativeOverlay { } catch (error) { this.recordPushDuration(performance.now() - startedAt) this.warnFramePush(error) + this.disableNative('frame_push_failed') return false } } - hide(target?: NativeOverlayTargetRef): void { - if (target && !this.matchesRun(target)) { + hide(target?: AgentPreviewTargetRef): void { + if (target && !this.matchesRef(target)) { return } this.desiredVisible = false this.setOverlayVisible(false) } - removeTarget(target?: NativeOverlayTargetRef): void { - if (target && !this.matchesRun(target)) { + removeTarget(target?: AgentPreviewTargetRef): void { + if (target && !this.matchesRef(target)) { return } this.desiredVisible = false @@ -202,27 +308,11 @@ export class AgentBrowserNativeOverlay { shutdown(): void { this.removeTarget() - this.unbindHost() - this.stopDisplayListeners() - - const overlay = this.overlay - this.overlay = null - this.available = false + this.claims.clear() + this.dismissedRuns.clear() + this.handlers.clear() this.unavailable = true - this.overlayVisible = null - this.activeSessionId = null - this.initialization = null - if (!overlay) { - return - } - - overlay.removeListener('activate', this.handleActivate) - overlay.removeListener('control', this.handleControl) - try { - overlay.stop() - } catch (error) { - this.warn('shutdown_failed', error) - } + this.stopNative() } private async start(): Promise { @@ -236,7 +326,7 @@ export class AgentBrowserNativeOverlay { const nativekit = await import('@zerob13/nativekit') const overlay = nativekit.overlay if ( - !overlay.start(TOOLBAR_OPTIONS) || + !overlay.start(toolbarOptions('browser')) || !overlay.setMaxSize(PREVIEW_MAX_EDGE) || !overlay.setVisible(false) ) { @@ -253,22 +343,43 @@ export class AgentBrowserNativeOverlay { overlay.on('activate', this.handleActivate) overlay.on('control', this.handleControl) this.overlay = overlay + this.toolbarSource = 'browser' this.overlayVisible = false this.available = true this.startDisplayListeners() return true } catch (error) { - this.shutdown() + this.unavailable = true + this.stopNative() this.logUnavailable('load_failed', error) return false } } - private attachHost(host: BrowserWindow): boolean { + private configureToolbar(source: AgentPreviewSource): boolean { + if (!this.overlay || this.toolbarSource === source) { + return Boolean(this.overlay) + } + try { + if (!this.overlay.start(toolbarOptions(source))) { + return false + } + this.toolbarSource = source + return true + } catch (error) { + this.warn('toolbar_update_failed', error) + return false + } + } + + private attachHost(host: BrowserWindow, force = false): boolean { const overlay = this.overlay if (!overlay || host.isDestroyed()) { return false } + if (!force && this.host?.id === host.id && this.hostListeners) { + return true + } if (this.host && this.host.id !== host.id) { this.unbindHost() @@ -367,7 +478,7 @@ export class AgentBrowserNativeOverlay { this.hostSyncTimer = null const host = this.host if (host && !host.isDestroyed()) { - this.attachHost(host) + this.attachHost(host, true) } }, HOST_SYNC_DELAY_MS) this.hostSyncTimer.unref?.() @@ -400,14 +511,27 @@ export class AgentBrowserNativeOverlay { screen.removeListener('display-metrics-changed', this.handleDisplayChange) } - private emitAction(action: AgentBrowserNativeOverlayAction): void { + private emitAction(action: 'activate' | 'dismiss'): void { + const target = this.target + if (!target) { + return + } + this.desiredVisible = false + this.setOverlayVisible(false) + this.handlers.get(target.source)?.(action, { ...target }) + } + + private dismissCurrent(): void { const target = this.target if (!target) { return } + this.dismissedRuns.set(target.sessionId, target.runId) this.desiredVisible = false this.setOverlayVisible(false) - this.onAction(action, { ...target }) + this.removeCurrentPresentation() + this.target = null + this.handlers.get(target.source)?.('dismiss', { ...target }) } private setOverlayVisible(visible: boolean): void { @@ -437,22 +561,38 @@ export class AgentBrowserNativeOverlay { } } - private matchesTarget(target: AgentBrowserNativeOverlayTarget): boolean { + private matchesTarget(target: AgentPreviewTarget): boolean { return ( - this.target?.windowId === target.windowId && + this.target?.source === target.source && + this.target.windowId === target.windowId && this.target.sessionId === target.sessionId && this.target.runId === target.runId && - this.target.captureEpoch === target.captureEpoch + this.target.epoch === target.epoch && + this.target.claimSequence === target.claimSequence ) } - private matchesRun(target: NativeOverlayTargetRef): boolean { + private matchesRef(target: AgentPreviewTargetRef): boolean { return ( this.target?.sessionId === target.sessionId && + (target.source == null || this.target.source === target.source) && (target.runId == null || this.target.runId === target.runId) ) } + private matchesClaim(target: AgentPreviewTarget): boolean { + const claim = this.claims.get(target.sessionId) + return ( + claim?.source === target.source && + claim.runId === target.runId && + claim.sequence === target.claimSequence + ) + } + + private isDismissed(target: Pick): boolean { + return this.dismissedRuns.get(target.sessionId) === target.runId + } + private isHostEligible(host: BrowserWindow): boolean { return !host.isDestroyed() && host.isVisible() && host.isFocused() && !host.isMinimized() } @@ -472,12 +612,12 @@ export class AgentBrowserNativeOverlay { return `chat-window:${windowId}` } - private presentationId(target: AgentBrowserNativeOverlayTarget): string { - return `agent-browser:${target.windowId}:${target.sessionId}` + private presentationId(target: AgentPreviewTarget): string { + return `agent-preview:${target.source}:${target.windowId}:${target.sessionId}` } - private nativeSessionId(sessionId: string): string { - return `agent-browser:${sessionId}` + private nativeSessionId(target: AgentPreviewTarget): string { + return `agent-preview:${target.source}:${target.sessionId}` } private normalizeBounds(bounds: Rectangle): Rectangle { @@ -489,10 +629,34 @@ export class AgentBrowserNativeOverlay { } } - private disable(reason: string): void { + private disableNative(reason: string): void { this.unavailable = true this.logUnavailable(reason) - this.shutdown() + this.stopNative() + } + + private stopNative(): void { + this.unbindHost() + this.stopDisplayListeners() + + const overlay = this.overlay + this.overlay = null + this.available = false + this.overlayVisible = null + this.activeSessionId = null + this.toolbarSource = null + this.initialization = null + if (!overlay) { + return + } + + overlay.removeListener('activate', this.handleActivate) + overlay.removeListener('control', this.handleControl) + try { + overlay.stop() + } catch (error) { + this.warn('shutdown_failed', error) + } } private recordPushDuration(durationMs: number): void { @@ -503,7 +667,7 @@ export class AgentBrowserNativeOverlay { return } this.lastSlowPushWarningAt = Date.now() - logger.warn('[AgentBrowserNativeOverlay] Slow frame presentation', { + logger.warn('[AgentPreviewCoordinator] Slow frame presentation', { durationMs: Math.round(durationMs) }) } @@ -517,7 +681,7 @@ export class AgentBrowserNativeOverlay { } private logUnavailable(reason: string, error?: unknown): void { - logger.info('[AgentBrowserNativeOverlay] Native overlay unavailable', { + logger.info('[AgentPreviewCoordinator] Native overlay unavailable', { platform: process.platform, arch: process.arch, reason, @@ -526,7 +690,7 @@ export class AgentBrowserNativeOverlay { } private warn(reason: string, error: unknown): void { - logger.warn('[AgentBrowserNativeOverlay] Native overlay operation failed', { + logger.warn('[AgentPreviewCoordinator] Native overlay operation failed', { reason, error: error instanceof Error ? error.message : String(error) }) diff --git a/src/main/desktop/routes.ts b/src/main/desktop/routes.ts index e53f3c924..461f55c1c 100644 --- a/src/main/desktop/routes.ts +++ b/src/main/desktop/routes.ts @@ -3,6 +3,7 @@ import type { IShortcutPresenter, ITabPresenter, IWindowPresenter, + IComputerUsePreviewPresenter, IYoBrowserPresenter } from '@shared/types/desktop' import type { DialogServicePort } from '@shared/types/dialog' @@ -11,6 +12,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -22,6 +24,8 @@ import { browserScanImportSourcesRoute, browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute, + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute, configGetFloatingButtonRoute, configGetLanguageRoute, configGetShortcutKeysRoute, @@ -56,18 +60,28 @@ import { } from '@shared/contracts/routes' import { DEV_EVENTS } from '../events' import { createRouteMap, type DeepchatRouteMap, type RouteContext } from '@/routes/routeRegistry' +import type { DesktopSessionBinding } from '@/desktop/sessionBinding' export function createDesktopRoutes(deps: { windowPresenter: IWindowPresenter shortcutPresenter: IShortcutPresenter browserPresenter: IYoBrowserPresenter + computerUsePreviewPresenter: IComputerUsePreviewPresenter + desktopSessionBinding: Pick tabPresenter: ITabPresenter dialogService: DialogServicePort settings: DesktopSettings setFloatingButtonEnabled(enabled: boolean): void recordActivity(input: SettingsActivityInput): void }): DeepchatRouteMap { - const { windowPresenter, shortcutPresenter, browserPresenter, tabPresenter, dialogService } = deps + const { + windowPresenter, + shortcutPresenter, + browserPresenter, + computerUsePreviewPresenter, + tabPresenter, + dialogService + } = deps const readWindowState = (context: RouteContext) => { const window = context.windowId == null ? null : BrowserWindow.fromId(context.windowId) const exists = Boolean(window && !window.isDestroyed()) @@ -402,6 +416,50 @@ export function createDesktopRoutes(deps: { ) } ], + [ + browserDismissPreviewRoute.name, + async (rawInput) => { + const input = browserDismissPreviewRoute.input.parse(rawInput) + return browserDismissPreviewRoute.output.parse({ + dismissed: browserPresenter.dismissPreview(input.sessionId, input.runId) + }) + } + ], + [ + computerUseSetPreviewModeRoute.name, + async (rawInput, context) => { + const input = computerUseSetPreviewModeRoute.input.parse(rawInput) + if ( + context.windowId == null || + (input.mode !== 'stopped' && + deps.desktopSessionBinding.getActiveId(context.webContentsId) !== input.sessionId) + ) { + return computerUseSetPreviewModeRoute.output.parse({ + updated: false, + surface: 'none' + }) + } + return computerUseSetPreviewModeRoute.output.parse( + await computerUsePreviewPresenter.setPreviewMode( + input.sessionId, + input.mode, + context.windowId + ) + ) + } + ], + [ + computerUseDismissPreviewRoute.name, + async (rawInput, context) => { + const input = computerUseDismissPreviewRoute.input.parse(rawInput) + const active = + deps.desktopSessionBinding.getActiveId(context.webContentsId) === input.sessionId + return computerUseDismissPreviewRoute.output.parse({ + dismissed: + active && computerUsePreviewPresenter.dismissPreview(input.sessionId, input.runId) + }) + } + ], [ browserDestroyRoute.name, async (rawInput) => { diff --git a/src/main/mcp/index.ts b/src/main/mcp/index.ts index fae6e66e5..287a7d65d 100644 --- a/src/main/mcp/index.ts +++ b/src/main/mcp/index.ts @@ -20,7 +20,7 @@ import { import type { ProviderRuntimePort } from '@shared/types/provider' import { ServerManager } from './serverManager' import type { McpClient as RuntimeMcpClient } from './mcpClient' -import { ToolManager } from './toolManager' +import { ToolManager, type ComputerUsePreviewObserver } from './toolManager' import { McpRouterManager } from './mcprouterManager' import { McpOAuthManager } from './mcpOAuthManager' import { getErrorMessageLabels } from '@shared/i18n' @@ -134,7 +134,8 @@ export class McpService implements McpServicePort { providerRuntime: Pick, onRegistryChanged: () => void, private readonly publishEvent: DeepchatEventPublisher, - cacheImage?: (data: string) => Promise + cacheImage?: (data: string) => Promise, + computerUsePreviewObserver?: ComputerUsePreviewObserver ) { logger.info('Initializing MCP service') @@ -167,7 +168,8 @@ export class McpService implements McpServicePort { this.locale, this.mcpSettings, this.serverManager, - this.publishEvent + this.publishEvent, + computerUsePreviewObserver ) // init mcprouter manager try { @@ -822,6 +824,7 @@ export class McpService implements McpServicePort { signal?: AbortSignal agentId?: string enabledServerIds?: string[] + runId?: string } ): Promise<{ content: string; rawData: MCPToolResponse }> { const toolCallResult = await this.toolManager.callTool(request, options) diff --git a/src/main/mcp/toolManager.ts b/src/main/mcp/toolManager.ts index a519fde80..027391110 100644 --- a/src/main/mcp/toolManager.ts +++ b/src/main/mcp/toolManager.ts @@ -32,6 +32,26 @@ type McpToolAccessContext = { conversationId?: string } +export type ComputerUsePreviewCall = { + conversationId: string + runId: string + toolCallId: string + toolName: string + args: Record + source: { + serverName: string + ownerPluginId?: string + sourceId?: string + } +} + +export type ComputerUsePreviewObserver = { + shouldCaptureAfterClick?(call: ComputerUsePreviewCall): boolean + started(call: ComputerUsePreviewCall): void + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void + failed(call: ComputerUsePreviewCall, error: unknown): void +} + type ActiveToolDefinitionsRefresh = { completion: Promise settle: () => void @@ -76,7 +96,8 @@ export class ToolManager { locale: Pick, mcpSettings: McpSettings, serverManager: ServerManager, - private readonly publishEvent: DeepchatEventPublisher + private readonly publishEvent: DeepchatEventPublisher, + private readonly computerUsePreviewObserver?: ComputerUsePreviewObserver ) { this.agentSettings = agentSettings this.locale = locale @@ -104,11 +125,13 @@ export class ToolManager { private isCuaComputerUseServer(client: McpClient, serverConfig?: MCPServerConfig): boolean { const clientConfig = client.serverConfig as { ownerPluginId?: unknown + source?: unknown sourceId?: unknown } const ownerPluginId = serverConfig?.ownerPluginId ?? clientConfig.ownerPluginId + const source = serverConfig?.source ?? clientConfig.source const sourceId = serverConfig?.sourceId ?? clientConfig.sourceId - return ownerPluginId === CUA_PLUGIN_ID || sourceId === CUA_PLUGIN_ID + return ownerPluginId === CUA_PLUGIN_ID || (source === 'plugin' && sourceId === CUA_PLUGIN_ID) } public async getRunningClients(): Promise { @@ -582,8 +605,11 @@ export class ToolManager { toolCall: MCPToolCall, access?: Pick & { signal?: AbortSignal + runId?: string } ): Promise { + let previewCall: ComputerUsePreviewCall | null = null + let previewTerminalNotified = false try { access?.signal?.throwIfAborted() const finalName = toolCall.function.name @@ -772,37 +798,32 @@ export class ToolManager { } } + previewCall = + originalName === 'get_window_state' + ? this.createComputerUsePreviewCall({ + client: targetClient, + serverConfig, + toolCall, + toolName: originalName, + args: preparedArgs.args, + runId: access?.runId + }) + : null + if (previewCall) { + this.notifyComputerUsePreview('started', previewCall) + } + // Call the tool on the target client using the ORIGINAL name const result = access?.signal ? await targetClient.callTool(originalName, preparedArgs.args, { signal: access.signal }) : await targetClient.callTool(originalName, preparedArgs.args) access?.signal?.throwIfAborted() - // Format response - let formattedContent: string | MCPContentItem[] = '' - if (typeof result.content === 'string') { - formattedContent = result.content - } else if (Array.isArray(result.content)) { - formattedContent = result.content.map((item): MCPContentItem => { - if (typeof item === 'string') { - return { type: 'text', text: item } as MCPTextContent - } - if (item.type === 'text' || item.type === 'image' || item.type === 'resource') { - return item as MCPContentItem - } - if (item.type && item.text) { - return { type: 'text', text: item.text } as MCPTextContent - } - return { type: 'text', text: JSON.stringify(item) } as MCPTextContent - }) - } else if (result.content) { - formattedContent = JSON.stringify(result.content) - } + const response = this.formatToolResponse(toolCall.id, result) - const response: MCPToolResponse = { - toolCallId: toolCall.id, - content: formattedContent, - isError: result.isError + if (previewCall) { + this.notifyComputerUsePreview('completed', previewCall, response) + previewTerminalNotified = true } this.publishEvent('mcp.toolCall.result', { @@ -811,8 +832,22 @@ export class ToolManager { version: Date.now() }) + this.scheduleComputerUsePreviewAfterClick({ + client: targetClient, + serverConfig, + toolCall, + toolName: originalName, + args: preparedArgs.args, + runId: access?.runId, + response, + signal: access?.signal + }) + return response } catch (error: unknown) { + if (previewCall && !previewTerminalNotified) { + this.notifyComputerUsePreview('failed', previewCall, error) + } if (access?.signal?.aborted || isAbortError(error)) { throw error } @@ -827,6 +862,188 @@ export class ToolManager { } } + private createComputerUsePreviewCall(input: { + client: McpClient + serverConfig: MCPServerConfig + toolCall: MCPToolCall + toolName: string + args: Record + runId?: string + }): ComputerUsePreviewCall | null { + const conversationId = input.toolCall.conversationId?.trim() + const runId = input.runId?.trim() + if ( + !conversationId || + !runId || + !this.isCuaComputerUseServer(input.client, input.serverConfig) + ) { + return null + } + + const clientConfig = input.client.serverConfig as { + ownerPluginId?: unknown + sourceId?: unknown + } + const ownerPluginId = input.serverConfig.ownerPluginId ?? clientConfig.ownerPluginId + const sourceId = input.serverConfig.sourceId ?? clientConfig.sourceId + return { + conversationId, + runId, + toolCallId: input.toolCall.id, + toolName: input.toolName, + args: { ...input.args }, + source: { + serverName: input.client.serverName, + ...(typeof ownerPluginId === 'string' ? { ownerPluginId } : {}), + ...(typeof sourceId === 'string' ? { sourceId } : {}) + } + } + } + + private scheduleComputerUsePreviewAfterClick(input: { + client: McpClient + serverConfig: MCPServerConfig + toolCall: MCPToolCall + toolName: string + args: Record + runId?: string + response: MCPToolResponse + signal?: AbortSignal + }): void { + if ( + input.toolName !== 'click' || + input.response.isError === true || + !this.computerUsePreviewObserver?.shouldCaptureAfterClick + ) { + return + } + + const clickCall = this.createComputerUsePreviewCall(input) + const pid = this.readPositiveIntegerArg(input.args.pid) + const windowId = this.readPositiveIntegerArg(input.args.window_id) + if (!clickCall || pid == null || windowId == null) { + return + } + + let shouldCapture = false + try { + shouldCapture = this.computerUsePreviewObserver.shouldCaptureAfterClick(clickCall) + } catch (error) { + logger.warn('[ToolManager] Computer Use preview eligibility check failed', { + toolCallId: clickCall.toolCallId, + error: error instanceof Error ? error.message : String(error) + }) + } + if (!shouldCapture) { + return + } + + const snapshotCall: ComputerUsePreviewCall = { + ...clickCall, + toolCallId: `${clickCall.toolCallId}:pip-snapshot`, + toolName: 'get_window_state', + args: { + pid, + window_id: windowId + } + } + void this.captureComputerUsePreviewSnapshot(input.client, snapshotCall, input.signal) + } + + private async captureComputerUsePreviewSnapshot( + client: McpClient, + call: ComputerUsePreviewCall, + signal?: AbortSignal + ): Promise { + let started = false + try { + signal?.throwIfAborted() + this.notifyComputerUsePreview('started', call) + started = true + const result = signal + ? await client.callTool('get_window_state', call.args, { signal }) + : await client.callTool('get_window_state', call.args) + signal?.throwIfAborted() + this.notifyComputerUsePreview( + 'completed', + call, + this.formatToolResponse(call.toolCallId, result) + ) + } catch (error) { + if (started) { + this.notifyComputerUsePreview('failed', call, error) + } + } + } + + private formatToolResponse( + toolCallId: string, + result: { content?: unknown; isError?: boolean } + ): MCPToolResponse { + let formattedContent: string | MCPContentItem[] = '' + if (typeof result.content === 'string') { + formattedContent = result.content + } else if (Array.isArray(result.content)) { + formattedContent = result.content.map((item): MCPContentItem => { + if (typeof item === 'string') { + return { type: 'text', text: item } as MCPTextContent + } + if (item && typeof item === 'object' && ('type' in item || 'text' in item)) { + const contentItem = item as { type?: unknown; text?: unknown } + if ( + contentItem.type === 'text' || + contentItem.type === 'image' || + contentItem.type === 'resource' + ) { + return item as MCPContentItem + } + if (contentItem.type && contentItem.text) { + return { type: 'text', text: String(contentItem.text) } as MCPTextContent + } + } + return { type: 'text', text: JSON.stringify(item) } as MCPTextContent + }) + } else if (result.content) { + formattedContent = JSON.stringify(result.content) + } + + return { + toolCallId, + content: formattedContent, + isError: result.isError + } + } + + private readPositiveIntegerArg(value: unknown): number | null { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null + } + + private notifyComputerUsePreview( + phase: 'started' | 'completed' | 'failed', + call: ComputerUsePreviewCall, + value?: MCPToolResponse | unknown + ): void { + const observer = this.computerUsePreviewObserver + if (!observer) { + return + } + try { + if (phase === 'started') { + observer.started(call) + } else if (phase === 'completed') { + observer.completed(call, value as MCPToolResponse) + } else { + observer.failed(call, value) + } + } catch (error) { + logger.warn('[ToolManager] Computer Use preview observer failed', { + phase, + toolCallId: call.toolCallId, + error: error instanceof Error ? error.message : String(error) + }) + } + } + private async prepareToolArguments( client: McpClient, serverConfig: MCPServerConfig, diff --git a/src/main/tool/index.ts b/src/main/tool/index.ts index 9d124a0fc..fdbaad76c 100644 --- a/src/main/tool/index.ts +++ b/src/main/tool/index.ts @@ -355,6 +355,7 @@ export class ToolService implements ToolServicePort { return await this.options.mcpService.callTool(request, { agentId: options?.agentId ?? storedAccess?.agentId, enabledServerIds: options?.enabledMcpServerIds ?? storedAccess?.enabledMcpServerIds, + runId: options?.runId, signal: options?.signal }) } diff --git a/src/renderer/api/BrowserClient.ts b/src/renderer/api/BrowserClient.ts index 5358e8bd5..e48b2fcf0 100644 --- a/src/renderer/api/BrowserClient.ts +++ b/src/renderer/api/BrowserClient.ts @@ -4,6 +4,7 @@ import { browserOpenRequestedEvent, browserPreviewActionEvent, browserPreviewFrameEvent, + browserPreviewSurfaceChangedEvent, browserStatusChangedEvent, type DeepchatEventPayload } from '@shared/contracts/events' @@ -11,6 +12,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -88,6 +90,11 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return await bridge.invoke(browserSetPreviewModeRoute.name, { sessionId, mode, runId }) } + async function dismissPreview(sessionId: string, runId: string) { + const result = await bridge.invoke(browserDismissPreviewRoute.name, { sessionId, runId }) + return result.dismissed + } + async function destroy(sessionId: string) { const result = await bridge.invoke(browserDestroyRoute.name, { sessionId }) return result.destroyed @@ -211,6 +218,12 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return bridge.on(browserPreviewActionEvent.name, listener) } + function onPreviewSurfaceChanged( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(browserPreviewSurfaceChangedEvent.name, listener) + } + return { getStatus, loadUrl, @@ -218,6 +231,7 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() updateCurrentWindowBounds, detach, setPreviewMode, + dismissPreview, destroy, goBack, goForward, @@ -232,7 +246,8 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() onStatusChanged, onActivityChanged, onPreviewFrame, - onPreviewAction + onPreviewAction, + onPreviewSurfaceChanged } } diff --git a/src/renderer/api/ComputerUseClient.ts b/src/renderer/api/ComputerUseClient.ts new file mode 100644 index 000000000..4516b64fc --- /dev/null +++ b/src/renderer/api/ComputerUseClient.ts @@ -0,0 +1,46 @@ +import type { DeepchatBridge } from '@shared/contracts/bridge' +import { + computerUsePreviewFrameEvent, + computerUsePreviewSurfaceChangedEvent, + type DeepchatEventPayload +} from '@shared/contracts/events' +import { + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute +} from '@shared/contracts/routes' +import type { ComputerUsePreviewMode } from '@shared/types/computerUse' +import { getDeepchatBridge } from './core' + +export function createComputerUseClient(bridge: DeepchatBridge = getDeepchatBridge()) { + async function setPreviewMode(sessionId: string, mode: ComputerUsePreviewMode) { + return await bridge.invoke(computerUseSetPreviewModeRoute.name, { sessionId, mode }) + } + + async function dismissPreview(sessionId: string, runId: string) { + const result = await bridge.invoke(computerUseDismissPreviewRoute.name, { sessionId, runId }) + return result.dismissed + } + + function onPreviewFrame( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(computerUsePreviewFrameEvent.name, listener) + } + + function onPreviewSurfaceChanged( + listener: ( + payload: DeepchatEventPayload + ) => void + ) { + return bridge.on(computerUsePreviewSurfaceChangedEvent.name, listener) + } + + return { + setPreviewMode, + dismissPreview, + onPreviewFrame, + onPreviewSurfaceChanged + } +} + +export type ComputerUseClient = ReturnType diff --git a/src/renderer/src/apps/chat-main/ChatTabView.vue b/src/renderer/src/apps/chat-main/ChatTabView.vue index db48817d5..65373fd87 100644 --- a/src/renderer/src/apps/chat-main/ChatTabView.vue +++ b/src/renderer/src/apps/chat-main/ChatTabView.vue @@ -32,6 +32,9 @@ + void) | null = null let stopActivityChanged: (() => void) | null = null let stopPreviewFrame: (() => void) | null = null let stopPreviewAction: (() => void) | null = null +let stopPreviewSurfaceChanged: (() => void) | null = null let stopWindowStateChanged: (() => void) | null = null let dragState: | { @@ -274,9 +275,13 @@ const loadStatus = async () => { } } -const dismiss = () => { - dismissedRunId.value = currentRunId.value +const dismiss = async () => { + const sessionId = currentSessionId.value + const runId = currentRunId.value + if (!sessionId || !runId) return + dismissedRunId.value = runId toolbarVisible.value = false + await browserClient.dismissPreview(sessionId, runId) } const openInPanel = async () => { @@ -506,11 +511,26 @@ onMounted(() => { return } if (payload.action === 'dismiss') { - dismiss() + void dismiss() return } void openInPanel() }) + stopPreviewSurfaceChanged = browserClient.onPreviewSurfaceChanged((payload) => { + if ( + payload.windowId !== currentWindowId.value || + payload.sessionId !== currentSessionId.value || + payload.runId !== currentRunId.value + ) { + return + } + if (payload.surface === 'none') { + frameDecodeVersion += 1 + latestFrameSequence = -1 + hasFrame.value = false + } + previewSurface.value = payload.surface + }) stopWindowStateChanged = windowClient.onCurrentStateChanged((payload) => { windowStateVersion += 1 currentWindowId.value = payload.windowId @@ -533,6 +553,7 @@ onBeforeUnmount(() => { stopActivityChanged?.() stopPreviewFrame?.() stopPreviewAction?.() + stopPreviewSurfaceChanged?.() stopWindowStateChanged?.() if (currentSessionId.value) { pendingPreviewRequest = { diff --git a/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue new file mode 100644 index 000000000..f78963912 --- /dev/null +++ b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue @@ -0,0 +1,403 @@ + + + diff --git a/src/shared/contracts/events.ts b/src/shared/contracts/events.ts index be01e4dcd..697122f5f 100644 --- a/src/shared/contracts/events.ts +++ b/src/shared/contracts/events.ts @@ -21,8 +21,13 @@ import { browserOpenRequestedEvent, browserPreviewActionEvent, browserPreviewFrameEvent, + browserPreviewSurfaceChangedEvent, browserStatusChangedEvent } from './events/browser.events' +import { + computerUsePreviewFrameEvent, + computerUsePreviewSurfaceChangedEvent +} from './events/computerUse.events' import { chatPlanUpdatedEvent, chatStreamCompletedEvent, @@ -129,6 +134,7 @@ import { } from './events/workspace.events' export * from './events/browser.events' +export * from './events/computerUse.events' export * from './events/acp-terminal.events' export * from './events/app-runtime.events' export * from './events/chat.events' @@ -161,6 +167,9 @@ export const DEEPCHAT_EVENT_CATALOG = { [browserActivityChangedEvent.name]: browserActivityChangedEvent, [browserPreviewActionEvent.name]: browserPreviewActionEvent, [browserPreviewFrameEvent.name]: browserPreviewFrameEvent, + [browserPreviewSurfaceChangedEvent.name]: browserPreviewSurfaceChangedEvent, + [computerUsePreviewFrameEvent.name]: computerUsePreviewFrameEvent, + [computerUsePreviewSurfaceChangedEvent.name]: computerUsePreviewSurfaceChangedEvent, [browserOpenRequestedEvent.name]: browserOpenRequestedEvent, [browserStatusChangedEvent.name]: browserStatusChangedEvent, [settingsChangedEvent.name]: settingsChangedEvent, diff --git a/src/shared/contracts/events/browser.events.ts b/src/shared/contracts/events/browser.events.ts index 3a79c25de..c3113bd28 100644 --- a/src/shared/contracts/events/browser.events.ts +++ b/src/shared/contracts/events/browser.events.ts @@ -98,3 +98,14 @@ export const browserPreviewActionEvent = defineEventContract({ runId: z.string().min(1) }) }) + +export const browserPreviewSurfaceChangedEvent = defineEventContract({ + name: 'browser.preview.surface.changed', + payload: z.object({ + windowId: z.number().int().positive(), + sessionId: z.string().min(1), + runId: z.string().min(1), + surface: z.enum(['native-overlay', 'renderer-canvas', 'none']), + version: TimestampMsSchema + }) +}) diff --git a/src/shared/contracts/events/computerUse.events.ts b/src/shared/contracts/events/computerUse.events.ts new file mode 100644 index 000000000..4b8a29c51 --- /dev/null +++ b/src/shared/contracts/events/computerUse.events.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { TimestampMsSchema, defineEventContract } from '../common' + +const PreviewSurfaceSchema = z.enum(['native-overlay', 'renderer-canvas', 'none']) + +export const computerUsePreviewFrameEvent = defineEventContract({ + name: 'computerUse.preview.frame', + payload: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1), + epoch: z.number().int().nonnegative(), + sequence: z.number().int().nonnegative(), + width: z.number().int().positive().max(480), + height: z.number().int().positive().max(300), + mimeType: z.literal('image/jpeg'), + data: z.custom( + (value) => value instanceof Uint8Array && value.byteLength <= 512 * 1024 + ), + timestamp: TimestampMsSchema + }) +}) + +export const computerUsePreviewSurfaceChangedEvent = defineEventContract({ + name: 'computerUse.preview.surface.changed', + payload: z.object({ + windowId: z.number().int().positive(), + sessionId: z.string().min(1), + runId: z.string().min(1), + epoch: z.number().int().nonnegative(), + surface: PreviewSurfaceSchema, + version: TimestampMsSchema + }) +}) diff --git a/src/shared/contracts/routes.ts b/src/shared/contracts/routes.ts index c28f80404..658407605 100644 --- a/src/shared/contracts/routes.ts +++ b/src/shared/contracts/routes.ts @@ -5,6 +5,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -17,6 +18,10 @@ import { browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute } from './routes/browser.routes' +import { + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute +} from './routes/computerUse.routes' import { chatCancelSubmissionRoute, chatRespondToolInteractionRoute, @@ -508,6 +513,7 @@ import { } from './routes/workspace.routes' export * from './routes/browser.routes' +export * from './routes/computerUse.routes' export * from './routes/acp-terminal.routes' export * from './routes/chat.routes' export * from './routes/config.routes' @@ -693,8 +699,11 @@ const DEEPCHAT_ROUTE_CATALOG_PART_2 = { [browserClearSandboxDataRoute.name]: browserClearSandboxDataRoute, [browserScanImportSourcesRoute.name]: browserScanImportSourcesRoute, [browserSetPreviewModeRoute.name]: browserSetPreviewModeRoute, + [browserDismissPreviewRoute.name]: browserDismissPreviewRoute, [browserPreviewImportRoute.name]: browserPreviewImportRoute, [browserApplyImportRoute.name]: browserApplyImportRoute, + [computerUseSetPreviewModeRoute.name]: computerUseSetPreviewModeRoute, + [computerUseDismissPreviewRoute.name]: computerUseDismissPreviewRoute, [tabCaptureCurrentAreaRoute.name]: tabCaptureCurrentAreaRoute, [tabStitchImagesWithWatermarkRoute.name]: tabStitchImagesWithWatermarkRoute } satisfies Record diff --git a/src/shared/contracts/routes/browser.routes.ts b/src/shared/contracts/routes/browser.routes.ts index ce93a49ea..ba5bc6e20 100644 --- a/src/shared/contracts/routes/browser.routes.ts +++ b/src/shared/contracts/routes/browser.routes.ts @@ -69,6 +69,17 @@ export const browserSetPreviewModeRoute = defineRouteContract({ }) }) +export const browserDismissPreviewRoute = defineRouteContract({ + name: 'browser.dismissPreview', + input: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1) + }), + output: z.object({ + dismissed: z.boolean() + }) +}) + export const browserDestroyRoute = defineRouteContract({ name: 'browser.destroy', input: z.object({ diff --git a/src/shared/contracts/routes/computerUse.routes.ts b/src/shared/contracts/routes/computerUse.routes.ts new file mode 100644 index 000000000..5697bc9bc --- /dev/null +++ b/src/shared/contracts/routes/computerUse.routes.ts @@ -0,0 +1,27 @@ +import { z } from 'zod' +import { defineRouteContract } from '../common' + +const ComputerUsePreviewSurfaceSchema = z.enum(['native-overlay', 'renderer-canvas', 'none']) + +export const computerUseSetPreviewModeRoute = defineRouteContract({ + name: 'computerUse.setPreviewMode', + input: z.object({ + sessionId: z.string().min(1), + mode: z.enum(['eligible', 'suspended', 'stopped']) + }), + output: z.object({ + updated: z.boolean(), + surface: ComputerUsePreviewSurfaceSchema + }) +}) + +export const computerUseDismissPreviewRoute = defineRouteContract({ + name: 'computerUse.dismissPreview', + input: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1) + }), + output: z.object({ + dismissed: z.boolean() + }) +}) diff --git a/src/shared/types/computerUse.ts b/src/shared/types/computerUse.ts new file mode 100644 index 000000000..469ac65ca --- /dev/null +++ b/src/shared/types/computerUse.ts @@ -0,0 +1,8 @@ +export type ComputerUsePreviewMode = 'eligible' | 'suspended' | 'stopped' + +export type ComputerUsePreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' + +export interface ComputerUsePreviewModeResult { + updated: boolean + surface: ComputerUsePreviewSurface +} diff --git a/src/shared/types/desktop.ts b/src/shared/types/desktop.ts index 70b9eb7e2..e4fbb3cdd 100644 --- a/src/shared/types/desktop.ts +++ b/src/shared/types/desktop.ts @@ -11,6 +11,7 @@ import type { YoBrowserStatus } from './browser' import type { MCPToolDefinition } from './mcp' +import type { ComputerUsePreviewMode, ComputerUsePreviewModeResult } from './computerUse' import type { ProviderInstallPreview } from '@shared/providerDeeplink' import type { SettingsNavigationPayload } from '@shared/settingsNavigation' @@ -129,6 +130,7 @@ export interface IYoBrowserPresenter { hostWindowId?: number, runId?: string ): Promise + dismissPreview(sessionId: string, runId: string): boolean destroySessionBrowser(sessionId: string): Promise goBack(sessionId: string): Promise goForward(sessionId: string): Promise @@ -156,6 +158,16 @@ export interface IYoBrowserPresenter { } } +export interface IComputerUsePreviewPresenter { + setPreviewMode( + sessionId: string, + mode: ComputerUsePreviewMode, + hostWindowId?: number + ): Promise + dismissPreview(sessionId: string, runId: string): boolean + shutdown(): void +} + export interface ITabPresenter { createTab(windowId: number, url: string, options?: TabCreateOptions): Promise closeTab(tabId: number): Promise diff --git a/src/shared/types/mcp.ts b/src/shared/types/mcp.ts index 54e2348d1..cd417b57b 100644 --- a/src/shared/types/mcp.ts +++ b/src/shared/types/mcp.ts @@ -234,6 +234,7 @@ export interface McpServicePort { signal?: AbortSignal agentId?: string enabledServerIds?: string[] + runId?: string } ): Promise<{ content: string; rawData: MCPToolResponse }> preCheckToolPermission( diff --git a/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts b/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts deleted file mode 100644 index b7be12837..000000000 --- a/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { EventEmitter } from 'node:events' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -class MockBrowserWindow extends EventEmitter { - readonly id = 7 - destroyed = false - visible = true - focused = true - minimized = false - - isDestroyed() { - return this.destroyed - } - - isVisible() { - return this.visible - } - - isFocused() { - return this.focused - } - - isMinimized() { - return this.minimized - } - - getTitle() { - return 'DeepChat' - } - - getContentBounds() { - return { x: 10, y: 20, width: 800, height: 600 } - } - - getNativeWindowHandle() { - return Buffer.alloc(process.platform === 'linux' ? 4 : 8) - } -} - -describe('AgentBrowserNativeOverlay', () => { - beforeEach(() => { - vi.resetModules() - vi.useFakeTimers() - }) - - afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() - }) - - const setup = async (startResult = true) => { - const screen = new EventEmitter() - const overlay = Object.assign(new EventEmitter(), { - start: vi.fn(() => startResult), - stop: vi.fn(() => true), - attachHost: vi.fn(() => true), - detachHost: vi.fn(() => true), - setVisible: vi.fn(() => true), - setMaxSize: vi.fn(() => true), - pushImage: vi.fn(() => true), - setActiveSession: vi.fn(() => true), - removeImage: vi.fn(() => true) - }) - - vi.doMock('electron', () => ({ - app: { - getName: () => 'DeepChat' - }, - screen - })) - vi.doMock('@zerob13/nativekit', () => ({ overlay })) - vi.doMock('@shared/logger', () => ({ - default: { - info: vi.fn(), - warn: vi.fn() - } - })) - - const { AgentBrowserNativeOverlay } = - await import('@/desktop/browser/AgentBrowserNativeOverlay') - const onAction = vi.fn() - const adapter = new AgentBrowserNativeOverlay(onAction) - - return { - adapter, - onAction, - overlay, - screen, - host: new MockBrowserWindow() - } - } - - it('pushes a current JPEG before showing the native panel', async () => { - const { adapter, overlay, host } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 3 - } - - await expect(adapter.initialize()).resolves.toBe(true) - expect(overlay.start).toHaveBeenCalledWith({ - toolbar: { - style: 'dark', - buttons: [ - { - id: 'open-panel', - imageData: expect.stringMatching(/^data:image\/png;base64,/), - tooltip: 'Open in side panel' - }, - { - id: 'close', - imageData: expect.stringMatching(/^data:image\/png;base64,/), - tooltip: 'Close' - } - ] - } - }) - expect(overlay.setMaxSize).toHaveBeenCalledWith(360) - expect(overlay.setVisible).toHaveBeenCalledWith(false) - expect(adapter.prepare(target, host as never)).toBe(true) - expect(overlay.setVisible).toHaveBeenLastCalledWith(false) - - expect(adapter.present(target, Buffer.from('frame'))).toBe(true) - - expect(overlay.pushImage).toHaveBeenCalledWith({ - hostId: 'chat-window:7', - presentationId: 'agent-browser:7:session-1', - sessionId: 'agent-browser:session-1', - imageData: `data:image/jpeg;base64,${Buffer.from('frame').toString('base64')}` - }) - expect(overlay.setActiveSession).toHaveBeenCalledWith('agent-browser:session-1') - expect(overlay.setVisible).toHaveBeenLastCalledWith(true) - expect(overlay.pushImage.mock.invocationCallOrder[0]).toBeLessThan( - overlay.setActiveSession.mock.invocationCallOrder[0] - ) - expect(overlay.setActiveSession.mock.invocationCallOrder[0]).toBeLessThan( - overlay.setVisible.mock.invocationCallOrder.at(-1)! - ) - - adapter.shutdown() - }) - - it('refreshes the same presentation without resetting native placement state', async () => { - const { adapter, overlay, host } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 3 - } - - await adapter.initialize() - adapter.prepare(target, host as never) - adapter.present(target, Buffer.from('frame-1')) - - overlay.attachHost.mockClear() - overlay.detachHost.mockClear() - overlay.removeImage.mockClear() - overlay.setActiveSession.mockClear() - overlay.setVisible.mockClear() - - expect(adapter.present(target, Buffer.from('frame-2'))).toBe(true) - - expect(overlay.pushImage).toHaveBeenCalledTimes(2) - expect(overlay.pushImage).toHaveBeenLastCalledWith( - expect.objectContaining({ - presentationId: 'agent-browser:7:session-1', - sessionId: 'agent-browser:session-1' - }) - ) - expect(overlay.attachHost).not.toHaveBeenCalled() - expect(overlay.detachHost).not.toHaveBeenCalled() - expect(overlay.removeImage).not.toHaveBeenCalled() - expect(overlay.setActiveSession).not.toHaveBeenCalled() - expect(overlay.setVisible).not.toHaveBeenCalled() - - adapter.shutdown() - }) - - it('keeps host movement native and maps unscoped controls to the current target', async () => { - const { adapter, onAction, overlay, host, screen } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 4 - } - await adapter.initialize() - adapter.prepare(target, host as never) - adapter.present(target, Buffer.from('frame')) - - host.focused = false - host.emit('blur') - expect(overlay.setVisible).toHaveBeenLastCalledWith(false) - - host.emit('move') - screen.emit('display-metrics-changed') - await vi.advanceTimersByTimeAsync(50) - expect(overlay.attachHost).toHaveBeenCalledTimes(2) - - overlay.emit('activate') - overlay.emit('control', 'open-panel') - overlay.emit('control', 'unknown') - overlay.emit('control', 'close') - expect(onAction).toHaveBeenNthCalledWith(1, 'activate', target) - expect(onAction).toHaveBeenNthCalledWith(2, 'activate', target) - expect(onAction).toHaveBeenNthCalledWith(3, 'dismiss', target) - expect(onAction).toHaveBeenCalledTimes(3) - - adapter.shutdown() - }) - - it('reports unavailable without attaching when native startup fails', async () => { - const { adapter, overlay, host } = await setup(false) - - await expect(adapter.initialize()).resolves.toBe(false) - expect( - adapter.prepare( - { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 1 - }, - host as never - ) - ).toBe(false) - expect(overlay.attachHost).not.toHaveBeenCalled() - }) - - it('falls back permanently when the first real host cannot attach', async () => { - const { adapter, overlay, host } = await setup() - overlay.attachHost.mockReturnValue(false) - await adapter.initialize() - - expect( - adapter.prepare( - { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 1 - }, - host as never - ) - ).toBe(false) - expect(adapter.isAvailable()).toBe(false) - await expect(adapter.initialize()).resolves.toBe(false) - expect(overlay.stop).toHaveBeenCalledTimes(1) - }) -}) diff --git a/test/main/desktop/browser/YoBrowserPresenter.test.ts b/test/main/desktop/browser/YoBrowserPresenter.test.ts index edca8992e..805371d9f 100644 --- a/test/main/desktop/browser/YoBrowserPresenter.test.ts +++ b/test/main/desktop/browser/YoBrowserPresenter.test.ts @@ -9,13 +9,16 @@ const overlayHideMock = vi.fn() const overlayDestroyMock = vi.fn() const nativeInitializeMock = vi.fn(async () => false) const nativeIsAvailableMock = vi.fn(() => false) -const nativePrepareMock = vi.fn(() => false) +const nativePrepareMock = vi.fn(() => 'renderer-canvas') const nativePresentMock = vi.fn(() => true) const nativeHideMock = vi.fn() const nativeRemoveTargetMock = vi.fn() -const nativeShutdownMock = vi.fn() +const nativeReleaseClaimMock = vi.fn() +const nativeClaimMock = vi.fn(() => 1) +const nativeDismissMock = vi.fn(() => true) +const nativeIsCurrentMock = vi.fn(() => true) let nativeActionHandler: - | ((action: 'activate' | 'dismiss', target: Record) => void) + | ((action: 'activate' | 'dismiss' | 'superseded', target: Record) => void) | null = null class MockWebContents extends EventEmitter { @@ -167,8 +170,11 @@ describe('YoBrowserPresenter', () => { overlayDestroyMock.mockClear() nativeInitializeMock.mockResolvedValue(false) nativeIsAvailableMock.mockReturnValue(false) - nativePrepareMock.mockReturnValue(false) + nativePrepareMock.mockReturnValue('renderer-canvas') nativePresentMock.mockReturnValue(true) + nativeClaimMock.mockReturnValue(1) + nativeDismissMock.mockReturnValue(true) + nativeIsCurrentMock.mockReturnValue(true) nativeActionHandler = null vi.useFakeTimers() }) @@ -243,24 +249,6 @@ describe('YoBrowserPresenter', () => { } })) - vi.doMock('@/desktop/browser/AgentBrowserNativeOverlay', () => ({ - AgentBrowserNativeOverlay: class { - constructor( - onAction: (action: 'activate' | 'dismiss', target: Record) => void - ) { - nativeActionHandler = onAction - } - - initialize = nativeInitializeMock - isAvailable = nativeIsAvailableMock - prepare = nativePrepareMock - present = nativePresentMock - hide = nativeHideMock - removeTarget = nativeRemoveTargetMock - shutdown = nativeShutdownMock - } - })) - const { YoBrowserPresenter } = await import('@/desktop/browser/YoBrowserPresenter') const windowPresenter = { show: vi.fn((windowId: number) => { @@ -285,9 +273,37 @@ describe('YoBrowserPresenter', () => { }) } - const presenter = new YoBrowserPresenter(windowPresenter as any, (name, payload) => { - sendToAllWindowsMock('deepchat:event', createDeepchatEventEnvelope(name, payload)) - }) + const previewCoordinator = { + register: vi.fn( + ( + _source: 'browser', + handler: ( + action: 'activate' | 'dismiss' | 'superseded', + target: Record + ) => void + ) => { + nativeActionHandler = handler + return vi.fn() + } + ), + initialize: nativeInitializeMock, + isAvailable: nativeIsAvailableMock, + claim: nativeClaimMock, + releaseClaim: nativeReleaseClaimMock, + dismiss: nativeDismissMock, + prepare: nativePrepareMock, + present: nativePresentMock, + hide: nativeHideMock, + removeTarget: nativeRemoveTargetMock, + isCurrent: nativeIsCurrentMock + } + const presenter = new YoBrowserPresenter( + windowPresenter as any, + (name, payload) => { + sendToAllWindowsMock('deepchat:event', createDeepchatEventEnvelope(name, payload)) + }, + previewCoordinator as never + ) const getSessionWebContents = (sessionId: string) => { return ((presenter as any).sessionBrowsers.get(sessionId)?.view?.webContents ?? @@ -580,7 +596,7 @@ describe('YoBrowserPresenter', () => { windows.set(1, new MockBrowserWindow(1)) nativeInitializeMock.mockResolvedValue(true) nativeIsAvailableMock.mockReturnValue(true) - nativePrepareMock.mockReturnValue(true) + nativePrepareMock.mockReturnValue('native-overlay') await presenter.initialize() const loadPromise = presenter.loadUrl( diff --git a/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts new file mode 100644 index 000000000..c63b35d94 --- /dev/null +++ b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts @@ -0,0 +1,474 @@ +import { EventEmitter } from 'node:events' +import sharp from 'sharp' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +class MockBrowserWindow extends EventEmitter { + destroyed = false + + constructor(readonly id: number) { + super() + } + + isDestroyed() { + return this.destroyed + } +} + +describe('ComputerUsePreviewPresenter', () => { + beforeEach(() => { + vi.resetModules() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const setup = async (nativeSurface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas') => { + const windows = new Map([[7, new MockBrowserWindow(7)]]) + vi.doMock('electron', () => ({ + BrowserWindow: { + fromId: (id: number) => windows.get(id) ?? null + } + })) + vi.doMock('@shared/logger', () => ({ + default: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn() + } + })) + + const { ComputerUsePreviewPresenter } = + await import('@/desktop/computerUse/ComputerUsePreviewPresenter') + let actionHandler: + | ((action: 'activate' | 'dismiss' | 'superseded', target: Record) => void) + | null = null + let nextClaimSequence = 0 + const coordinator = { + register: vi.fn( + ( + _source: 'computer-use', + handler: ( + action: 'activate' | 'dismiss' | 'superseded', + target: Record + ) => void + ) => { + actionHandler = handler + return vi.fn() + } + ), + initialize: vi.fn(async () => nativeSurface === 'native-overlay'), + claim: vi.fn(() => ++nextClaimSequence), + releaseClaim: vi.fn(), + dismiss: vi.fn(() => true), + prepare: vi.fn(() => nativeSurface), + present: vi.fn(() => true), + hide: vi.fn(), + removeTarget: vi.fn() + } + const windowPresenter = { + sendToWindow: vi.fn(() => true) + } + const presenter = new ComputerUsePreviewPresenter( + windowPresenter as never, + coordinator as never + ) + + return { + presenter, + coordinator, + windowPresenter, + windows, + getActionHandler: () => actionHandler + } + } + + const call = ( + toolCallId: string, + options: { + runId?: string + pid?: number + windowId?: number + toolName?: string + } = {} + ) => ({ + conversationId: 'session-1', + runId: options.runId ?? 'run-1', + toolCallId, + toolName: options.toolName ?? 'get_window_state', + args: { + pid: options.pid ?? 12, + window_id: options.windowId ?? 34 + }, + source: { + serverName: 'cua-driver', + ownerPluginId: 'com.deepchat.plugins.cua' + } + }) + + const createPng = async (width = 960, height = 600, red = 20) => + await sharp({ + create: { + width, + height, + channels: 3, + background: { r: red, g: 40, b: 60 } + } + }) + .png() + .toBuffer() + + const completeWithImage = async ( + presenter: { + completed: (call: ReturnType, result: never) => void + }, + currentCall: ReturnType, + image: Buffer, + mimeType = 'image/png' + ) => { + presenter.completed(currentCall, { + toolCallId: currentCall.toolCallId, + content: [ + { + type: 'image', + mimeType, + data: image.toString('base64') + } + ], + isError: false + } as never) + } + + it('allows post-click capture only for the eligible current non-dismissed target', async () => { + const { presenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + presenter.started(call('snapshot')) + const click = call('click', { toolName: 'click' }) + + expect(presenter.shouldCaptureAfterClick(click)).toBe(true) + expect( + presenter.shouldCaptureAfterClick(call('right-click', { toolName: 'right_click' })) + ).toBe(false) + expect( + presenter.shouldCaptureAfterClick(call('stale-target', { toolName: 'click', windowId: 35 })) + ).toBe(false) + + await presenter.setPreviewMode('session-1', 'suspended', 7) + expect(presenter.shouldCaptureAfterClick(click)).toBe(false) + + await presenter.setPreviewMode('session-1', 'eligible', 7) + expect(presenter.dismissPreview('session-1', 'run-1')).toBe(true) + expect(presenter.shouldCaptureAfterClick(click)).toBe(false) + }) + + it('publishes a bounded Canvas frame only after a valid current snapshot', async () => { + const { presenter, coordinator, windowPresenter } = await setup() + await expect(presenter.setPreviewMode('session-1', 'eligible', 7)).resolves.toEqual({ + updated: true, + surface: 'none' + }) + const currentCall = call('tool-1') + + presenter.started(currentCall) + + expect(coordinator.prepare).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1', + epoch: 1 + }), + expect.objectContaining({ id: 7 }) + ) + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + + await completeWithImage(presenter, currentCall, await createPng()) + + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(true) + ) + const frameEnvelope = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: Record }) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frameEnvelope?.payload).toEqual( + expect.objectContaining({ + sessionId: 'session-1', + runId: 'run-1', + epoch: 1, + sequence: 1, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: expect.any(Uint8Array) + }) + ) + expect((frameEnvelope?.payload.data as Uint8Array).byteLength).toBeLessThanOrEqual(512 * 1024) + expect(coordinator.present).not.toHaveBeenCalled() + }) + + it('clears the old target epoch and rejects an out-of-order result', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-1') + presenter.started(first) + await completeWithImage(presenter, first, await createPng()) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const stale = call('tool-2', { windowId: 35 }) + const current = call('tool-3', { windowId: 35 }) + presenter.started(stale) + presenter.started(current) + await completeWithImage(presenter, stale, await createPng(320, 200, 80)) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + expect( + windowPresenter.sendToWindow.mock.calls.some(([, , envelope]) => { + const event = envelope as { name?: string; payload?: { epoch?: number } } + return event.name === 'computerUse.preview.surface.changed' && event.payload?.epoch === 2 + }) + ).toBe(true) + + await completeWithImage(presenter, current, await createPng(320, 200, 100)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + ) + const frames = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: { epoch?: number } }) + .filter((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frames[1].payload.epoch).toBe(2) + }) + + it('keeps native snapshot bytes out of renderer events', async () => { + const { presenter, coordinator, windowPresenter } = await setup('native-overlay') + await presenter.setPreviewMode('session-1', 'eligible', 7) + const currentCall = call('tool-native') + presenter.started(currentCall) + + await completeWithImage(presenter, currentCall, await createPng(320, 200)) + await vi.waitFor(() => expect(coordinator.present).toHaveBeenCalledOnce()) + + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + }) + + it('rejects malformed images and keeps the last valid same-target frame', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const validCall = call('tool-valid') + presenter.started(validCall) + await completeWithImage(presenter, validCall, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const invalidCall = call('tool-invalid') + presenter.started(invalidCall) + presenter.completed(invalidCall, { + toolCallId: invalidCall.toolCallId, + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'not-base64!' + } + ], + isError: false + }) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + }) + + it('enforces image bounds without upscaling the last valid frame', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const validCall = call('tool-small') + presenter.started(validCall) + await completeWithImage(presenter, validCall, await createPng(120, 80)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const frame = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: Record }) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frame?.payload).toEqual(expect.objectContaining({ width: 120, height: 80 })) + + const jpegCall = call('tool-jpeg') + presenter.started(jpegCall) + const jpeg = await sharp(await createPng(160, 90)) + .jpeg() + .toBuffer() + await completeWithImage(presenter, jpegCall, jpeg, 'image/jpeg') + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + ) + + const unsupportedCall = call('tool-unsupported') + presenter.started(unsupportedCall) + await completeWithImage(presenter, unsupportedCall, await createPng(120, 80), 'image/webp') + + const oversizedCall = call('tool-oversized') + presenter.started(oversizedCall) + presenter.completed(oversizedCall, { + toolCallId: oversizedCall.toolCallId, + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'A'.repeat(Math.ceil(((16 * 1024 * 1024 + 1) * 4) / 3)) + } + ], + isError: false + }) + await new Promise((resolve) => setImmediate(resolve)) + + const overDimensionCall = call('tool-dimension') + presenter.started(overDimensionCall) + await completeWithImage(presenter, overDimensionCall, await createPng(8193, 1)) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + }) + + it('drops intermediate transforms and publishes only the latest overlapping snapshot', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-first') + const intermediate = call('tool-intermediate') + const latest = call('tool-latest') + + presenter.started(first) + await completeWithImage(presenter, first, await createPng(480, 300, 20)) + presenter.started(intermediate) + await completeWithImage(presenter, intermediate, await createPng(480, 300, 100)) + presenter.started(latest) + await completeWithImage(presenter, latest, await createPng(480, 300, 220)) + + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + const frame = windowPresenter.sendToWindow.mock.calls + .map( + ([, , envelope]) => + envelope as { name: string; payload: { data: Uint8Array; sequence: number } } + ) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frame).toBeDefined() + const stats = await sharp(frame!.payload.data).stats() + + expect(frame!.payload.sequence).toBe(1) + expect(stats.channels[0].mean).toBeGreaterThan(180) + }) + + it('retains the last valid frame after failed and aborted snapshot calls', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const valid = call('tool-valid') + presenter.started(valid) + await completeWithImage(presenter, valid, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const failed = call('tool-failed') + presenter.started(failed) + presenter.failed(failed, new Error('snapshot failed')) + const aborted = call('tool-aborted') + presenter.started(aborted) + presenter.failed(aborted, new DOMException('Aborted', 'AbortError')) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + }) + + it('dismisses only the current run and permits a later run', async () => { + const { presenter, coordinator, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-1') + presenter.started(first) + + expect(presenter.dismissPreview('session-1', 'run-1')).toBe(true) + await completeWithImage(presenter, first, await createPng()) + await new Promise((resolve) => setImmediate(resolve)) + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + + const second = call('tool-2', { runId: 'run-2' }) + presenter.started(second) + await completeWithImage(presenter, second, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(true) + ) + expect(coordinator.dismiss).toHaveBeenCalledWith({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + }) +}) diff --git a/test/main/desktop/preview/AgentPreviewCoordinator.test.ts b/test/main/desktop/preview/AgentPreviewCoordinator.test.ts new file mode 100644 index 000000000..e282d22e3 --- /dev/null +++ b/test/main/desktop/preview/AgentPreviewCoordinator.test.ts @@ -0,0 +1,344 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +class MockBrowserWindow extends EventEmitter { + readonly id = 7 + destroyed = false + visible = true + focused = true + minimized = false + + isDestroyed() { + return this.destroyed + } + + isVisible() { + return this.visible + } + + isFocused() { + return this.focused + } + + isMinimized() { + return this.minimized + } + + getTitle() { + return 'DeepChat' + } + + getContentBounds() { + return { x: 10, y: 20, width: 800, height: 600 } + } + + getNativeWindowHandle() { + return Buffer.alloc(process.platform === 'linux' ? 4 : 8) + } +} + +describe('AgentPreviewCoordinator', () => { + beforeEach(() => { + vi.resetModules() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + const setup = async (startResult = true) => { + const screen = new EventEmitter() + const overlay = Object.assign(new EventEmitter(), { + start: vi.fn(() => startResult), + stop: vi.fn(() => true), + attachHost: vi.fn(() => true), + detachHost: vi.fn(() => true), + setVisible: vi.fn(() => true), + setMaxSize: vi.fn(() => true), + pushImage: vi.fn(() => true), + setActiveSession: vi.fn(() => true), + removeImage: vi.fn(() => true) + }) + + vi.doMock('electron', () => ({ + app: { + getName: () => 'DeepChat' + }, + screen + })) + vi.doMock('@zerob13/nativekit', () => ({ overlay })) + vi.doMock('@shared/logger', () => ({ + default: { + info: vi.fn(), + warn: vi.fn() + } + })) + + const { AgentPreviewCoordinator } = await import('@/desktop/preview/AgentPreviewCoordinator') + const browserAction = vi.fn() + const computerAction = vi.fn() + const coordinator = new AgentPreviewCoordinator() + coordinator.register('browser', browserAction) + coordinator.register('computer-use', computerAction) + + const browserTarget = (epoch = 3) => { + const claimSequence = coordinator.claim({ + source: 'browser', + sessionId: 'session-1', + runId: 'run-1' + }) + return { + source: 'browser' as const, + windowId: 7, + sessionId: 'session-1', + runId: 'run-1', + epoch, + claimSequence + } + } + + return { + coordinator, + browserAction, + computerAction, + browserTarget, + overlay, + screen, + host: new MockBrowserWindow() + } + } + + it('pushes a current JPEG before showing the native panel', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + const target = browserTarget() + + await expect(coordinator.initialize()).resolves.toBe(true) + expect(overlay.start).toHaveBeenCalledWith({ + toolbar: { + style: 'dark', + buttons: [ + { + id: 'open-panel', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Open in side panel' + }, + { + id: 'close', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Close' + } + ] + } + }) + expect(overlay.setMaxSize).toHaveBeenCalledWith(360) + expect(overlay.setVisible).toHaveBeenCalledWith(false) + expect(coordinator.prepare(target, host as never)).toBe('native-overlay') + expect(overlay.setVisible).toHaveBeenLastCalledWith(false) + + expect(coordinator.present(target, Buffer.from('frame'))).toBe(true) + + expect(overlay.pushImage).toHaveBeenCalledWith({ + hostId: 'chat-window:7', + presentationId: 'agent-preview:browser:7:session-1', + sessionId: 'agent-preview:browser:session-1', + imageData: `data:image/jpeg;base64,${Buffer.from('frame').toString('base64')}` + }) + expect(overlay.setActiveSession).toHaveBeenCalledWith('agent-preview:browser:session-1') + expect(overlay.setVisible).toHaveBeenLastCalledWith(true) + expect(overlay.pushImage.mock.invocationCallOrder[0]).toBeLessThan( + overlay.setActiveSession.mock.invocationCallOrder[0] + ) + expect(overlay.setActiveSession.mock.invocationCallOrder[0]).toBeLessThan( + overlay.setVisible.mock.invocationCallOrder.at(-1)! + ) + + coordinator.shutdown() + }) + + it('refreshes the same presentation without resetting native placement state', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + const target = browserTarget() + + await coordinator.initialize() + coordinator.prepare(target, host as never) + coordinator.present(target, Buffer.from('frame-1')) + coordinator.prepare(target, host as never) + expect(overlay.attachHost).toHaveBeenCalledOnce() + + overlay.attachHost.mockClear() + overlay.detachHost.mockClear() + overlay.removeImage.mockClear() + overlay.setActiveSession.mockClear() + overlay.setVisible.mockClear() + + expect(coordinator.present(target, Buffer.from('frame-2'))).toBe(true) + + expect(overlay.pushImage).toHaveBeenCalledTimes(2) + expect(overlay.pushImage).toHaveBeenLastCalledWith( + expect.objectContaining({ + presentationId: 'agent-preview:browser:7:session-1', + sessionId: 'agent-preview:browser:session-1' + }) + ) + expect(overlay.attachHost).not.toHaveBeenCalled() + expect(overlay.detachHost).not.toHaveBeenCalled() + expect(overlay.removeImage).not.toHaveBeenCalled() + expect(overlay.setActiveSession).not.toHaveBeenCalled() + expect(overlay.setVisible).not.toHaveBeenCalled() + + coordinator.shutdown() + }) + + it('switches to the close-only Computer Use toolbar and ignores activation', async () => { + const { coordinator, browserAction, browserTarget, computerAction, overlay, host } = + await setup() + const browser = browserTarget(4) + await coordinator.initialize() + coordinator.prepare(browser, host as never) + coordinator.present(browser, Buffer.from('browser')) + + const claimSequence = coordinator.claim({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + const computer = { + source: 'computer-use' as const, + windowId: 7, + sessionId: 'session-1', + runId: 'run-1', + epoch: 5, + claimSequence + } + + expect(coordinator.prepare(computer, host as never)).toBe('native-overlay') + expect(browserAction).toHaveBeenCalledWith('superseded', browser) + expect(overlay.start).toHaveBeenLastCalledWith({ + toolbar: { + style: 'dark', + buttons: [ + { + id: 'close', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Close' + } + ] + } + }) + + overlay.emit('activate') + overlay.emit('control', 'open-panel') + expect(computerAction).not.toHaveBeenCalled() + + overlay.emit('control', 'close') + expect(computerAction).toHaveBeenCalledWith('dismiss', computer) + + coordinator.shutdown() + }) + + it('keeps host movement native and maps Browser controls to the current target', async () => { + const { coordinator, browserAction, browserTarget, overlay, host, screen } = await setup() + const target = browserTarget(4) + await coordinator.initialize() + coordinator.prepare(target, host as never) + coordinator.present(target, Buffer.from('frame')) + + host.focused = false + host.emit('blur') + expect(overlay.setVisible).toHaveBeenLastCalledWith(false) + + host.emit('move') + screen.emit('display-metrics-changed') + await vi.advanceTimersByTimeAsync(50) + expect(overlay.attachHost).toHaveBeenCalledTimes(2) + + overlay.emit('activate') + overlay.emit('control', 'open-panel') + overlay.emit('control', 'unknown') + overlay.emit('control', 'close') + expect(browserAction).toHaveBeenNthCalledWith(1, 'activate', target) + expect(browserAction).toHaveBeenNthCalledWith(2, 'activate', target) + expect(browserAction).toHaveBeenNthCalledWith(3, 'dismiss', target) + expect(browserAction).toHaveBeenCalledTimes(3) + + coordinator.shutdown() + }) + + it('returns Canvas fallback when native startup fails', async () => { + const { coordinator, browserTarget, overlay, host } = await setup(false) + const target = browserTarget(1) + + await expect(coordinator.initialize()).resolves.toBe(false) + expect(coordinator.prepare(target, host as never)).toBe('renderer-canvas') + expect(overlay.attachHost).not.toHaveBeenCalled() + }) + + it('falls back permanently when the first real host cannot attach', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + overlay.attachHost.mockReturnValue(false) + await coordinator.initialize() + + expect(coordinator.prepare(browserTarget(1), host as never)).toBe('renderer-canvas') + expect(coordinator.isAvailable()).toBe(false) + await expect(coordinator.initialize()).resolves.toBe(false) + expect(overlay.stop).toHaveBeenCalledTimes(1) + }) + + it('keeps run dismissal scoped to the current source owner', async () => { + const { coordinator, browserTarget, host } = await setup(false) + const browser = browserTarget() + await coordinator.initialize() + coordinator.prepare(browser, host as never) + + expect( + coordinator.dismiss({ + source: 'computer-use', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(false) + expect(coordinator.isCurrent(browser)).toBe(true) + expect( + coordinator.dismiss({ + source: 'browser', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(true) + expect(coordinator.isCurrent(browser)).toBe(false) + }) + + it('uses the matching session claim when another session owns the visible surface', async () => { + const { coordinator, browserTarget, host } = await setup(false) + const browser = browserTarget() + await coordinator.initialize() + coordinator.prepare(browser, host as never) + + const computerClaimSequence = coordinator.claim({ + source: 'computer-use', + sessionId: 'session-2', + runId: 'run-2' + }) + coordinator.prepare( + { + source: 'computer-use', + windowId: 7, + sessionId: 'session-2', + runId: 'run-2', + epoch: 1, + claimSequence: computerClaimSequence + }, + host as never + ) + + expect( + coordinator.dismiss({ + source: 'browser', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(true) + }) +}) diff --git a/test/main/mcp/toolManager.test.ts b/test/main/mcp/toolManager.test.ts index fd746e2c8..bf5a34cfe 100644 --- a/test/main/mcp/toolManager.test.ts +++ b/test/main/mcp/toolManager.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { ToolManager } from '@/mcp/toolManager' +import { ToolManager, type ComputerUsePreviewObserver } from '@/mcp/toolManager' function deferred() { let resolve!: (value: T) => void @@ -77,13 +77,18 @@ describe('ToolManager', () => { } } - function createToolManager(providerSettings: unknown, serverManager: unknown) { + function createToolManager( + providerSettings: unknown, + serverManager: unknown, + computerUsePreviewObserver?: ComputerUsePreviewObserver + ) { return new ToolManager( providerSettings as never, { getLanguage: vi.fn().mockReturnValue('en-US') }, providerSettings as never, serverManager as never, - vi.fn() + vi.fn(), + computerUsePreviewObserver ) } @@ -688,6 +693,520 @@ describe('ToolManager', () => { expect(providerSettings.getAgentMcpSelections).not.toHaveBeenCalled() }) + it('observes trusted CUA snapshots with run metadata without changing tool arguments', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + client.callTool.mockResolvedValue({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ], + isError: false + }) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([client]), + observer + ) + const toolCall = { + id: 'cua-state-1', + type: 'function' as const, + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + } + + const result = await manager.callTool(toolCall, { runId: 'run-1' }) + + expect(client.callTool).toHaveBeenCalledWith('get_window_state', { + pid: 12, + window_id: 34 + }) + expect(observer.started).toHaveBeenCalledWith({ + conversationId: 'session-1', + runId: 'run-1', + toolCallId: 'cua-state-1', + toolName: 'get_window_state', + args: { + pid: 12, + window_id: 34 + }, + source: { + serverName: 'cua-driver', + ownerPluginId: 'com.deepchat.plugins.cua' + } + }) + expect(observer.completed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-state-1' }), + result + ) + expect(observer.failed).not.toHaveBeenCalled() + expect(JSON.stringify(client.callTool.mock.calls)).not.toContain('run-1') + }) + + it('refreshes PiP after a trusted click without exposing or awaiting the private snapshot', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + }, + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const privateSnapshots = [ + deferred<{ + content: Array<{ type: string; mimeType: string; data: string }> + isError: boolean + }>(), + deferred<{ + content: Array<{ type: string; mimeType: string; data: string }> + isError: boolean + }>() + ] + let privateSnapshotIndex = 0 + client.callTool.mockImplementation((toolName: string) => { + if (toolName === 'click') { + return Promise.resolve({ + content: 'clicked', + isError: false + }) + } + return privateSnapshots[privateSnapshotIndex++].promise + }) + const publishEvent = vi.fn() + const observer = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const providerSettings = createProviderSettings('cua-driver') + const manager = new ToolManager( + providerSettings as never, + { getLanguage: vi.fn().mockReturnValue('en-US') }, + providerSettings as never, + createServerManager([client]) as never, + publishEvent, + observer + ) + + const result = await manager.callTool( + { + id: 'cua-click-1', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result).toEqual({ + toolCallId: 'cua-click-1', + content: 'clicked', + isError: false + }) + expect(client.callTool).toHaveBeenNthCalledWith(1, 'click', { + pid: 12, + window_id: 34, + x: 10, + y: 20 + }) + expect(client.callTool).toHaveBeenNthCalledWith(2, 'get_window_state', { + pid: 12, + window_id: 34 + }) + expect(observer.shouldCaptureAfterClick).toHaveBeenCalledWith( + expect.objectContaining({ + toolCallId: 'cua-click-1', + toolName: 'click', + conversationId: 'session-1', + runId: 'run-1' + }) + ) + expect(observer.started).toHaveBeenCalledWith( + expect.objectContaining({ + toolCallId: 'cua-click-1:pip-snapshot', + toolName: 'get_window_state', + args: { + pid: 12, + window_id: 34 + } + }) + ) + expect(observer.completed).not.toHaveBeenCalled() + expect(publishEvent).toHaveBeenCalledOnce() + expect(publishEvent).toHaveBeenCalledWith( + 'mcp.toolCall.result', + expect.objectContaining({ + functionName: 'click', + content: 'clicked' + }) + ) + + privateSnapshots[0].resolve({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ], + isError: false + }) + await vi.waitFor(() => expect(observer.completed).toHaveBeenCalledOnce()) + + expect(observer.completed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-click-1:pip-snapshot' }), + expect.objectContaining({ + toolCallId: 'cua-click-1:pip-snapshot', + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ] + }) + ) + expect(publishEvent).toHaveBeenCalledOnce() + expect(observer.failed).not.toHaveBeenCalled() + + const laterResult = await manager.callTool( + { + id: 'cua-click-2', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":30,"y":40}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + expect(laterResult.content).toBe('clicked') + expect(publishEvent).toHaveBeenCalledTimes(2) + + const privateFailure = new Error('private snapshot failed') + privateSnapshots[1].reject(privateFailure) + await vi.waitFor(() => expect(observer.failed).toHaveBeenCalledOnce()) + + expect(observer.failed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-click-2:pip-snapshot' }), + privateFailure + ) + expect(publishEvent).toHaveBeenCalledTimes(2) + }) + + it('does not privately snapshot failed, invalid-target, or untrusted clicks', async () => { + const trustedClient = createClient( + 'cua-driver', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + trustedClient.callTool + .mockResolvedValueOnce({ content: 'click failed', isError: true }) + .mockResolvedValueOnce({ content: 'clicked', isError: false }) + const trustedObserver = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const trustedManager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([trustedClient]), + trustedObserver + ) + + await trustedManager.callTool( + { + id: 'failed-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + await trustedManager.callTool( + { + id: 'missing-window-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(trustedClient.callTool).toHaveBeenCalledTimes(2) + expect(trustedObserver.shouldCaptureAfterClick).not.toHaveBeenCalled() + expect(trustedObserver.started).not.toHaveBeenCalled() + + const untrustedClient = createClient( + 'manual-cua', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'manual', + sourceId: 'com.deepchat.plugins.cua' + } + ) + const untrustedObserver = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const untrustedManager = createToolManager( + createProviderSettings('manual-cua'), + createServerManager([untrustedClient]), + untrustedObserver + ) + + await untrustedManager.callTool( + { + id: 'untrusted-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(untrustedClient.callTool).toHaveBeenCalledOnce() + expect(untrustedObserver.shouldCaptureAfterClick).not.toHaveBeenCalled() + expect(untrustedObserver.started).not.toHaveBeenCalled() + }) + + it('does not observe a CUA permission response before actual invocation', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const providerSettings = createProviderSettings('cua-driver') + providerSettings.getMcpServers.mockResolvedValue({ + 'cua-driver': { + autoApprove: [] + } + }) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager(providerSettings, createServerManager([client]), observer) + + const result = await manager.callTool( + { + id: 'cua-state-permission', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result.requiresPermission).toBe(true) + expect(client.callTool).not.toHaveBeenCalled() + expect(observer.started).not.toHaveBeenCalled() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).not.toHaveBeenCalled() + }) + + it('does not trust a non-plugin server that spoofs the CUA source id', async () => { + const client = createClient( + 'spoofed-cua', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'manual', + sourceId: 'com.deepchat.plugins.cua' + } + ) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('spoofed-cua'), + createServerManager([client]), + observer + ) + + await manager.callTool( + { + id: 'spoofed-cua-state', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(client.callTool).toHaveBeenCalledOnce() + expect(observer.started).not.toHaveBeenCalled() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).not.toHaveBeenCalled() + }) + + it('reports a failed trusted CUA invocation without changing the tool error response', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const failure = new Error('driver failed') + client.callTool.mockRejectedValue(failure) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([client]), + observer + ) + + const result = await manager.callTool( + { + id: 'cua-state-failed', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result).toEqual({ + toolCallId: 'cua-state-failed', + content: "Error: Failed to execute tool 'get_window_state': driver failed", + isError: true + }) + expect(observer.started).toHaveBeenCalledOnce() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-state-failed' }), + failure + ) + }) + it('normalizes CUA Windows launch bundle paths before dispatch', async () => { const client = createClient('cua-driver', [], { source: 'plugin', diff --git a/test/main/routes/contracts.test.ts b/test/main/routes/contracts.test.ts index ad9f5d2e9..5a5a96285 100644 --- a/test/main/routes/contracts.test.ts +++ b/test/main/routes/contracts.test.ts @@ -74,7 +74,10 @@ describe('main kernel contracts', () => { 'acpTerminal.kill', 'browser.attachCurrentWindow', 'browser.clearSandboxData', + 'browser.dismissPreview', 'browser.setPreviewMode', + 'computerUse.dismissPreview', + 'computerUse.setPreviewMode', 'databaseSecurity.repairSchema', 'debug.createMockChatSession', 'config.addManualAcpAgent', @@ -1788,7 +1791,10 @@ describe('main kernel contracts', () => { 'browser.open.requested', 'browser.preview.action', 'browser.preview.frame', + 'browser.preview.surface.changed', 'browser.status.changed', + 'computerUse.preview.frame', + 'computerUse.preview.surface.changed', 'chat.plan.updated', 'chat.stream.completed', 'chat.stream.failed', @@ -1927,6 +1933,39 @@ describe('main kernel contracts', () => { ).toBe(false) }) + it('validates bounded Computer Use preview frames with a target epoch', () => { + const payload = { + sessionId: 'session-1', + runId: 'run-1', + epoch: 2, + sequence: 3, + width: 480, + height: 300, + mimeType: 'image/jpeg', + timestamp: Date.now() + } as const + + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint8Array([1, 2, 3]) + }).success + ).toBe(true) + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + epoch: -1, + data: new Uint8Array([1, 2, 3]) + }).success + ).toBe(false) + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint8Array(512 * 1024 + 1) + }).success + ).toBe(false) + }) + it('validates browser preview actions against an exact target', () => { expect( DEEPCHAT_EVENT_CATALOG['browser.preview.action'].payload.safeParse({ diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index fdd49f1b3..270d54715 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -446,7 +446,7 @@ function createRuntime() { activate: vi.fn().mockResolvedValue(undefined), deactivate: vi.fn().mockResolvedValue(undefined), getActive: vi.fn().mockResolvedValue(null), - getActiveId: vi.fn(() => null) + getActiveId: vi.fn((): string | null => null) } const sessionTurnPort = { sendMessage: vi.fn().mockResolvedValue({ @@ -1260,12 +1260,22 @@ function createRuntime() { attachSessionBrowser: vi.fn().mockResolvedValue(true), updateSessionBrowserBounds: vi.fn().mockResolvedValue(undefined), detachSessionBrowser: vi.fn().mockResolvedValue(undefined), + setPreviewMode: vi.fn().mockResolvedValue({ updated: true, surface: 'renderer-canvas' }), + dismissPreview: vi.fn(() => true), destroySessionBrowser: vi.fn().mockResolvedValue(undefined), goBack: vi.fn().mockResolvedValue(undefined), goForward: vi.fn().mockResolvedValue(undefined), reload: vi.fn().mockResolvedValue(undefined), clearSandboxData: vi.fn().mockResolvedValue(undefined) } as unknown as IYoBrowserPresenter + const computerUsePreviewPresenter = { + setPreviewMode: vi.fn(async (_sessionId: string, mode: string) => ({ + updated: true, + surface: mode === 'stopped' ? 'none' : 'renderer-canvas' + })), + dismissPreview: vi.fn(() => true), + shutdown: vi.fn() + } const tabPresenter = { captureTabArea: vi.fn().mockResolvedValue('data:image/png;base64,capture'), @@ -1552,6 +1562,8 @@ function createRuntime() { windowPresenter, shortcutPresenter, browserPresenter: yoBrowserPresenter, + computerUsePreviewPresenter, + desktopSessionBinding, tabPresenter, dialogService: dialogService as unknown as DialogServicePort, settings: desktopSettings as never, @@ -1799,6 +1811,7 @@ function createRuntime() { knowledgeService, workspaceService, yoBrowserPresenter, + computerUsePreviewPresenter, tabPresenter, cronJobs, usageStatsService, @@ -5902,6 +5915,66 @@ describe('dispatchDeepchatRoute', () => { expect(clearSandboxResult).toEqual({ cleared: true }) }) + it('scopes Computer Use preview routes to the active sender session', async () => { + const { runtime, computerUsePreviewPresenter, desktopSessionBinding, yoBrowserPresenter } = + createRuntime() + const context = { webContentsId: 88, windowId: 3 } + desktopSessionBinding.getActiveId.mockReturnValue('session-1') + + const eligible = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'eligible' }, + context + ) + const browserDismissed = await dispatchDeepchatRoute( + runtime, + 'browser.dismissPreview', + { sessionId: 'session-1', runId: 'run-1' }, + context + ) + const dismissed = await dispatchDeepchatRoute( + runtime, + 'computerUse.dismissPreview', + { sessionId: 'session-1', runId: 'run-1' }, + context + ) + + desktopSessionBinding.getActiveId.mockReturnValue('session-2') + const rejected = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'eligible' }, + context + ) + const cleanup = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'stopped' }, + context + ) + + expect(eligible).toEqual({ updated: true, surface: 'renderer-canvas' }) + expect(computerUsePreviewPresenter.setPreviewMode).toHaveBeenNthCalledWith( + 1, + 'session-1', + 'eligible', + 3 + ) + expect(browserDismissed).toEqual({ dismissed: true }) + expect(yoBrowserPresenter.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(dismissed).toEqual({ dismissed: true }) + expect(computerUsePreviewPresenter.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(rejected).toEqual({ updated: false, surface: 'none' }) + expect(cleanup).toEqual({ updated: true, surface: 'none' }) + expect(computerUsePreviewPresenter.setPreviewMode).toHaveBeenNthCalledWith( + 2, + 'session-1', + 'stopped', + 3 + ) + }) + it('dispatches phase3 tab routes through the renderer tab adapter', async () => { const { runtime, tabPresenter } = createRuntime() diff --git a/test/main/tool/toolService.test.ts b/test/main/tool/toolService.test.ts index 02302d0ef..5bb681abc 100644 --- a/test/main/tool/toolService.test.ts +++ b/test/main/tool/toolService.test.ts @@ -609,16 +609,23 @@ describe('ToolService', () => { expect((toolService as any).agentToolManager).toBe(runtimeManager) expect(runtimeManager.agentWorkspacePath).toBe('C:\\runtime-workspace') - await toolService.callTool({ - id: 'tool-1', - type: 'function', - function: { name: 'mcp_only', arguments: '{}' }, - conversationId: 'conv-1' - }) + await toolService.callTool( + { + id: 'tool-1', + type: 'function', + function: { name: 'mcp_only', arguments: '{}' }, + conversationId: 'conv-1' + }, + { runId: 'run-1' } + ) expect(mcpService.callTool).toHaveBeenCalledWith( expect.objectContaining({ function: expect.objectContaining({ name: 'mcp_only' }) }), - expect.objectContaining({ agentId: 'agent-1', enabledServerIds: ['mcp-server'] }) + expect.objectContaining({ + agentId: 'agent-1', + enabledServerIds: ['mcp-server'], + runId: 'run-1' + }) ) }) diff --git a/test/renderer/api/clients.test.ts b/test/renderer/api/clients.test.ts index a4c879a4e..921499108 100644 --- a/test/renderer/api/clients.test.ts +++ b/test/renderer/api/clients.test.ts @@ -4,6 +4,7 @@ import type { HooksNotificationsSettings } from '@shared/hooksNotifications' import { createAcpTerminalClient } from '../../../src/renderer/api/AcpTerminalClient' import { createAppRuntimeClient } from '../../../src/renderer/api/AppRuntimeClient' import { createBrowserClient } from '../../../src/renderer/api/BrowserClient' +import { createComputerUseClient } from '../../../src/renderer/api/ComputerUseClient' import { createChatClient } from '../../../src/renderer/api/ChatClient' import { createConfigClient } from '../../../src/renderer/api/ConfigClient' import { createContextMenuClient } from '../../../src/renderer/api/ContextMenuClient' @@ -877,6 +878,12 @@ describe('renderer api clients', () => { return { updated: true } case 'browser.setPreviewMode': return { updated: true, surface: 'renderer-canvas' } + case 'browser.dismissPreview': + return { dismissed: true } + case 'computerUse.setPreviewMode': + return { updated: true, surface: 'renderer-canvas' } + case 'computerUse.dismissPreview': + return { dismissed: true } case 'browser.clearSandboxData': return { cleared: true } case 'browser.import.scan': @@ -2236,6 +2243,39 @@ describe('renderer api clients', () => { }) }) + it('routes Browser and Computer Use preview dismissal through typed registry names', async () => { + const bridge = createBridge() + const browserClient = createBrowserClient(bridge) + const computerUseClient = createComputerUseClient(bridge) + + await expect(browserClient.dismissPreview('session-1', 'run-1')).resolves.toBe(true) + await expect(computerUseClient.dismissPreview('session-1', 'run-1')).resolves.toBe(true) + + expect(bridge.invoke).toHaveBeenNthCalledWith(1, 'browser.dismissPreview', { + sessionId: 'session-1', + runId: 'run-1' + }) + expect(bridge.invoke).toHaveBeenNthCalledWith(2, 'computerUse.dismissPreview', { + sessionId: 'session-1', + runId: 'run-1' + }) + }) + + it('routes Computer Use preview mode through the shared registry name', async () => { + const bridge = createBridge() + const client = createComputerUseClient(bridge) + + await expect(client.setPreviewMode('session-1', 'eligible')).resolves.toEqual({ + updated: true, + surface: 'renderer-canvas' + }) + + expect(bridge.invoke).toHaveBeenCalledWith('computerUse.setPreviewMode', { + sessionId: 'session-1', + mode: 'eligible' + }) + }) + it('routes browser website-data import through typed registry names', async () => { const bridge = createBridge() const browserClient = createBrowserClient(bridge) @@ -2912,4 +2952,21 @@ describe('renderer api clients', () => { expect(bridge.on).toHaveBeenCalledWith('browser.preview.action', listener) }) + + it('subscribes to Computer Use preview frames and surface changes', () => { + const bridge = createBridge() + const client = createComputerUseClient(bridge) + const frameListener = vi.fn() + const surfaceListener = vi.fn() + + client.onPreviewFrame(frameListener) + client.onPreviewSurfaceChanged(surfaceListener) + + expect(bridge.on).toHaveBeenNthCalledWith(1, 'computerUse.preview.frame', frameListener) + expect(bridge.on).toHaveBeenNthCalledWith( + 2, + 'computerUse.preview.surface.changed', + surfaceListener + ) + }) }) diff --git a/test/renderer/components/AgentBrowserPiP.test.ts b/test/renderer/components/AgentBrowserPiP.test.ts index 7bb689614..4720f2ca1 100644 --- a/test/renderer/components/AgentBrowserPiP.test.ts +++ b/test/renderer/components/AgentBrowserPiP.test.ts @@ -66,6 +66,15 @@ const setup = async ( runId: string }) => void) | null = null + let previewSurfaceChangedHandler: + | ((payload: { + windowId: number + sessionId: string + runId: string + surface: 'native-overlay' | 'renderer-canvas' | 'none' + version: number + }) => void) + | null = null const status = createStatus() const browserClient = { getStatus: vi.fn(async () => status), @@ -73,6 +82,7 @@ const setup = async ( updated: true, surface: options.surface ?? 'renderer-canvas' })), + dismissPreview: vi.fn(async () => true), onOpenRequestedForCurrentWindow: vi.fn(() => vi.fn()), onStatusChanged: vi.fn( (handler: (payload: { sessionId: string; status: YoBrowserStatus | null }) => void) => { @@ -88,6 +98,10 @@ const setup = async ( onPreviewAction: vi.fn((handler: NonNullable) => { previewActionHandler = handler return vi.fn() + }), + onPreviewSurfaceChanged: vi.fn((handler: NonNullable) => { + previewSurfaceChangedHandler = handler + return vi.fn() }) } const windowClient = { @@ -142,6 +156,7 @@ const setup = async ( emitStatus: statusChangedHandler!, emitPreviewFrame: previewFrameHandler!, emitPreviewAction: previewActionHandler!, + emitPreviewSurfaceChanged: previewSurfaceChangedHandler!, emitWindowState: windowStateHandler! } } @@ -223,7 +238,7 @@ describe('AgentBrowserPiP', () => { await flushPromises() expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) - expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'rendering', 'run-1') + expect(browserClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') }) it('ignores native actions targeted at another window', async () => { @@ -307,6 +322,52 @@ describe('AgentBrowserPiP', () => { expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) }) + it('clears retained Canvas pixels when another preview source supersedes Browser', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitPreviewFrame, emitPreviewSurfaceChanged } = await setup({ wide: true }) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-1', + sequence: 1, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([1, 2, 3]), + timestamp: Date.now() + }) + await flushPromises() + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) + + emitPreviewSurfaceChanged({ + windowId: 1, + sessionId: 'session-1', + runId: 'run-1', + surface: 'none', + version: Date.now() + }) + await nextTick() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + + emitPreviewSurfaceChanged({ + windowId: 1, + sessionId: 'session-1', + runId: 'run-1', + surface: 'renderer-canvas', + version: Date.now() + }) + await nextTick() + + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(true) + }) + it('accepts a reset frame sequence when the Agent run changes', async () => { const drawImage = vi.fn() vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage } as never) diff --git a/test/renderer/components/AgentComputerUsePiP.test.ts b/test/renderer/components/AgentComputerUsePiP.test.ts new file mode 100644 index 000000000..417f5788f --- /dev/null +++ b/test/renderer/components/AgentComputerUsePiP.test.ts @@ -0,0 +1,385 @@ +import { defineComponent, nextTick, reactive } from 'vue' +import { flushPromises, mount } from '@vue/test-utils' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const mountedWrappers: Array<{ unmount: () => void }> = [] + +const setup = async (surface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas') => { + vi.resetModules() + vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + x: 0, + y: 0, + top: 0, + left: 0, + right: 800, + bottom: 600, + width: 800, + height: 600, + toJSON: () => ({}) + }) + + type SurfacePayload = { + windowId: number + sessionId: string + runId: string + epoch: number + surface: 'native-overlay' | 'renderer-canvas' | 'none' + version: number + } + type FramePayload = { + sessionId: string + runId: string + epoch: number + sequence: number + width: number + height: number + mimeType: 'image/jpeg' + data: Uint8Array + timestamp: number + } + let surfaceHandler: ((payload: SurfacePayload) => void) | null = null + let frameHandler: ((payload: FramePayload) => void) | null = null + let windowStateHandler: + | ((payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => void) + | null = null + const stopFrameSubscription = vi.fn() + const stopSurfaceSubscription = vi.fn() + const stopWindowStateSubscription = vi.fn() + const computerUseClient = { + setPreviewMode: vi.fn( + async (_sessionId: string, mode: 'eligible' | 'suspended' | 'stopped') => ({ + updated: true, + surface: mode === 'eligible' ? surface : ('none' as const) + }) + ), + dismissPreview: vi.fn(async () => true), + onPreviewSurfaceChanged: vi.fn((handler: (payload: SurfacePayload) => void) => { + surfaceHandler = handler + return stopSurfaceSubscription + }), + onPreviewFrame: vi.fn((handler: (payload: FramePayload) => void) => { + frameHandler = handler + return stopFrameSubscription + }) + } + const windowClient = { + getCurrentState: vi.fn(async () => ({ windowId: 1, exists: true, isFocused: true })), + onCurrentStateChanged: vi.fn( + ( + handler: (payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => void + ) => { + windowStateHandler = handler + return stopWindowStateSubscription + } + ) + } + const sessionStore = reactive({ + sessions: [{ id: 'session-1', status: 'working' }] + }) + + vi.doMock('vue-i18n', () => ({ useI18n: () => ({ t: (key: string) => key }) })) + vi.doMock('@iconify/vue', () => ({ + Icon: defineComponent({ name: 'Icon', template: '' }) + })) + vi.doMock('@api/ComputerUseClient', () => ({ + createComputerUseClient: () => computerUseClient + })) + vi.doMock('@api/WindowClient', () => ({ createWindowClient: () => windowClient })) + vi.doMock('@/stores/ui/session', () => ({ useSessionStore: () => sessionStore })) + + const AgentComputerUsePiP = (await import('@/components/computerUse/AgentComputerUsePiP.vue')) + .default + const wrapper = mount(AgentComputerUsePiP, { + props: { sessionId: 'session-1' }, + global: { + stubs: { + Button: defineComponent({ + name: 'Button', + emits: ['click'], + template: '' + }) + } + } + }) + mountedWrappers.push(wrapper) + await flushPromises() + + return { + wrapper, + computerUseClient, + sessionStore, + stopFrameSubscription, + stopSurfaceSubscription, + stopWindowStateSubscription, + emitSurface: (payload: SurfacePayload) => surfaceHandler?.(payload), + emitFrame: (payload: FramePayload) => frameHandler?.(payload), + emitWindowState: (payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => + windowStateHandler?.(payload) + } +} + +const surfacePayload = ( + overrides: Partial<{ + windowId: number + sessionId: string + runId: string + epoch: number + surface: 'native-overlay' | 'renderer-canvas' | 'none' + }> = {} +) => ({ + windowId: overrides.windowId ?? 1, + sessionId: overrides.sessionId ?? 'session-1', + runId: overrides.runId ?? 'run-1', + epoch: overrides.epoch ?? 1, + surface: overrides.surface ?? ('renderer-canvas' as const), + version: Date.now() +}) + +const framePayload = ( + overrides: Partial<{ + sessionId: string + runId: string + epoch: number + sequence: number + width: number + height: number + data: Uint8Array + }> = {} +) => ({ + sessionId: overrides.sessionId ?? 'session-1', + runId: overrides.runId ?? 'run-1', + epoch: overrides.epoch ?? 1, + sequence: overrides.sequence ?? 1, + width: overrides.width ?? 480, + height: overrides.height ?? 300, + mimeType: 'image/jpeg' as const, + data: overrides.data ?? new Uint8Array([1, 2, 3]), + timestamp: Date.now() +}) + +afterEach(async () => { + mountedWrappers.splice(0).forEach((wrapper) => wrapper.unmount()) + await flushPromises() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +describe('AgentComputerUsePiP', () => { + it('keeps the native path headless without subscribing to frame bytes', async () => { + const { wrapper, computerUseClient, emitSurface } = await setup('native-overlay') + + emitSurface(surfacePayload({ surface: 'native-overlay' })) + await nextTick() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.onPreviewFrame).not.toHaveBeenCalled() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'eligible') + }) + + it('shows the first decoded Canvas snapshot with Close as its only control', async () => { + const drawImage = vi.fn() + const bitmapClose = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: bitmapClose })) + ) + const { wrapper, computerUseClient, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + await nextTick() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload()) + await flushPromises() + + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 480, 300) + expect(bitmapClose).toHaveBeenCalledOnce() + expect(pip.attributes('style')).not.toContain('display: none') + expect(pip.findAll('button')).toHaveLength(1) + expect(pip.get('button').attributes('aria-label')).toBe('common.close') + expect(wrapper.find('[aria-label="common.open"]').exists()).toBe(false) + + await pip.get('button').trigger('click') + await flushPromises() + + expect(computerUseClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(pip.attributes('style')).toContain('display: none') + }) + + it('keeps the current Canvas visible until a newer snapshot finishes decoding', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + let resolveSecondBitmap: ((bitmap: { close: () => void }) => void) | null = null + vi.stubGlobal( + 'createImageBitmap', + vi + .fn() + .mockResolvedValueOnce({ close: vi.fn() }) + .mockImplementationOnce( + () => + new Promise<{ close: () => void }>((resolve) => { + resolveSecondBitmap = resolve + }) + ) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + emitFrame(framePayload({ sequence: 1 })) + await flushPromises() + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + expect(pip.attributes('style')).not.toContain('display: none') + + emitFrame(framePayload({ sequence: 2 })) + await nextTick() + expect(drawImage).toHaveBeenCalledTimes(1) + expect(pip.attributes('style')).not.toContain('display: none') + + resolveSecondBitmap?.({ close: vi.fn() }) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(2) + }) + + it('drops stale target epochs and waits for the first new-target frame', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + emitFrame(framePayload({ sequence: 5 })) + await flushPromises() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).not.toContain( + 'display: none' + ) + + emitSurface(surfacePayload({ epoch: 2 })) + await nextTick() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload({ epoch: 1, sequence: 6 })) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(1) + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload({ epoch: 2, sequence: 1 })) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(2) + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).not.toContain( + 'display: none' + ) + }) + + it('suspends on blur, releases frame subscriptions, and resumes on focus', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { + wrapper, + computerUseClient, + stopFrameSubscription, + emitSurface, + emitFrame, + emitWindowState + } = await setup() + emitSurface(surfacePayload()) + emitFrame(framePayload()) + await flushPromises() + + emitWindowState({ windowId: 1, exists: true, isFocused: false }) + await flushPromises() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'suspended') + expect(stopFrameSubscription).toHaveBeenCalledOnce() + + emitWindowState({ windowId: 1, exists: true, isFocused: true }) + await flushPromises() + + expect(computerUseClient.setPreviewMode).toHaveBeenLastCalledWith('session-1', 'eligible') + expect(computerUseClient.onPreviewFrame).toHaveBeenCalledTimes(2) + }) + + it('drags the fallback without forwarding input to the target application', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + emitSurface(surfacePayload()) + emitFrame(framePayload()) + await flushPromises() + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + const initialStyle = pip.attributes('style') + + await pip.trigger('pointerdown', { button: 0, pointerId: 1, clientX: 500, clientY: 400 }) + await pip.trigger('pointermove', { pointerId: 1, clientX: 400, clientY: 300 }) + await pip.trigger('pointerup', { pointerId: 1, clientX: 400, clientY: 300 }) + + expect(pip.attributes('style')).not.toBe(initialStyle) + }) + + it('stops the previous session and releases subscriptions on teardown', async () => { + const { + wrapper, + computerUseClient, + sessionStore, + stopFrameSubscription, + stopSurfaceSubscription, + stopWindowStateSubscription, + emitSurface + } = await setup() + sessionStore.sessions.push({ id: 'session-2', status: 'working' }) + emitSurface(surfacePayload()) + await nextTick() + + await wrapper.setProps({ sessionId: 'session-2' }) + await flushPromises() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'stopped') + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-2', 'eligible') + expect(stopFrameSubscription).toHaveBeenCalledOnce() + + wrapper.unmount() + mountedWrappers.splice(mountedWrappers.indexOf(wrapper), 1) + await flushPromises() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-2', 'stopped') + expect(stopSurfaceSubscription).toHaveBeenCalledOnce() + expect(stopWindowStateSubscription).toHaveBeenCalledOnce() + }) + + it('stops and releases the fallback when the Agent run terminates', async () => { + const { wrapper, computerUseClient, sessionStore, emitSurface } = await setup() + emitSurface(surfacePayload()) + await nextTick() + + sessionStore.sessions[0].status = 'completed' + await flushPromises() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.setPreviewMode).toHaveBeenLastCalledWith('session-1', 'stopped') + }) +}) diff --git a/test/renderer/components/ChatTabView.test.ts b/test/renderer/components/ChatTabView.test.ts index e06557ee4..8d1fc8228 100644 --- a/test/renderer/components/ChatTabView.test.ts +++ b/test/renderer/components/ChatTabView.test.ts @@ -176,6 +176,12 @@ const setup = async (options: SetupOptions = {}) => { template: '
' }) })) + vi.doMock('@/components/computerUse/AgentComputerUsePiP.vue', () => ({ + default: defineComponent({ + name: 'AgentComputerUsePiP', + template: '
' + }) + })) vi.doMock('@/pages/AgentWelcomePage.vue', () => ({ default: defineComponent({ name: 'AgentWelcomePage', diff --git a/test/setup.ts b/test/setup.ts index a5ec3d04d..b29cb3dea 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -24,6 +24,12 @@ function getDefaultDeepchatInvokeResult( return { updated: true } case 'browser.setPreviewMode': return { updated: true, surface: 'renderer-canvas' } + case 'browser.dismissPreview': + return { dismissed: true } + case 'computerUse.setPreviewMode': + return { updated: true, surface: 'renderer-canvas' } + case 'computerUse.dismissPreview': + return { dismissed: true } case 'browser.detach': return { detached: true } case 'browser.destroy': From 71195479f7a73486613b50f18eaaa5ebe4a11c11 Mon Sep 17 00:00:00 2001 From: zerob13 Date: Tue, 28 Jul 2026 17:04:03 +0800 Subject: [PATCH 3/3] fix(pip): address review feedback --- .../computer-use-snapshot-pip/plan.md | 5 +- .../computer-use-snapshot-pip/spec.md | 4 +- .../ComputerUsePreviewPresenter.ts | 108 ++++++++++++------ src/main/mcp/toolManager.ts | 1 + .../components/browser/AgentBrowserPiP.vue | 4 +- .../computerUse/AgentComputerUsePiP.vue | 10 +- .../ComputerUsePreviewPresenter.test.ts | 21 ++++ test/main/mcp/toolManager.test.ts | 24 ++++ .../components/AgentBrowserPiP.test.ts | 6 +- .../components/AgentComputerUsePiP.test.ts | 48 +++++++- 10 files changed, 182 insertions(+), 49 deletions(-) diff --git a/docs/features/computer-use-snapshot-pip/plan.md b/docs/features/computer-use-snapshot-pip/plan.md index 2047f1bc7..eeaa60768 100644 --- a/docs/features/computer-use-snapshot-pip/plan.md +++ b/docs/features/computer-use-snapshot-pip/plan.md @@ -314,8 +314,9 @@ pnpm run format pnpm run i18n pnpm run lint pnpm run typecheck -pnpm test -- test/main -pnpm test -- test/renderer +pnpm run test:main +pnpm run test:renderer +pnpm exec electron-vite build ``` Prefer narrower test file invocations during development. Run the complete affected main and diff --git a/docs/features/computer-use-snapshot-pip/spec.md b/docs/features/computer-use-snapshot-pip/spec.md index b0d224dab..08c3f0871 100644 --- a/docs/features/computer-use-snapshot-pip/spec.md +++ b/docs/features/computer-use-snapshot-pip/spec.md @@ -248,8 +248,8 @@ other CUA tools are not frame sources. session, run, target, and non-dismissed state. - The private call uses only `{ pid, window_id }`, starts asynchronously after click completion, and does not delay or change the click response. -- The private call bypasses Agent-facing permission and result publication because the triggering - click has already passed access checks and the plugin policy explicitly allows +- The private call bypasses Agent-facing permission and result publication only after confirming + that the triggering click passed access checks and the current plugin policy explicitly allows `get_window_state`. - The private result enters only the same bounded preview observer and image pipeline. Its text, accessibility tree, and other content are discarded. diff --git a/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts index 65caf480a..23a5d7cf7 100644 --- a/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts +++ b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts @@ -37,6 +37,7 @@ type ComputerUsePreviewState = { sessionId: string mode: ComputerUsePreviewMode hostWindowId: number | null + hostClosedListener: (() => void) | null runId: string | null toolCallId: string | null toolCallEpoch: number @@ -82,6 +83,13 @@ export class ComputerUsePreviewPresenter shouldCaptureAfterClick(call: ComputerUsePreviewCall): boolean { const state = this.states.get(call.conversationId) + if (state?.hostWindowId != null) { + const host = BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + this.stopState(state) + return false + } + } const pid = this.readPositiveInteger(call.args.pid) const targetWindowId = this.readPositiveInteger(call.args.window_id) return Boolean( @@ -195,28 +203,30 @@ export class ComputerUsePreviewPresenter ) { return { updated: false, surface: 'none' } } - state.epoch += 1 - state.mode = 'stopped' - if (state.runId) { - this.previewCoordinator.releaseClaim({ - source: 'computer-use', - sessionId: state.sessionId, - runId: state.runId - }) - } - this.states.delete(normalizedSessionId) + this.stopState(state) return { updated: true, surface: 'none' } } const host = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) if (!host || host.isDestroyed()) { + const state = this.states.get(normalizedSessionId) + if (state && (state.hostWindowId == null || state.hostWindowId === hostWindowId)) { + this.stopState(state) + } return { updated: false, surface: 'none' } } this.stopOtherSessionsForHost(normalizedSessionId, host.id) await this.previewCoordinator.initialize() + if (host.isDestroyed()) { + const state = this.states.get(normalizedSessionId) + if (state && (state.hostWindowId == null || state.hostWindowId === host.id)) { + this.stopState(state) + } + return { updated: false, surface: 'none' } + } const state = this.states.get(normalizedSessionId) ?? this.createState(normalizedSessionId) - state.hostWindowId = host.id + this.bindHost(state, host) state.mode = mode if (mode === 'suspended') { @@ -256,15 +266,8 @@ export class ComputerUsePreviewPresenter shutdown(): void { for (const state of this.states.values()) { - if (state.runId) { - this.previewCoordinator.releaseClaim({ - source: 'computer-use', - sessionId: state.sessionId, - runId: state.runId - }) - } + this.stopState(state) } - this.states.clear() this.unregisterPreviewHandler() } @@ -273,6 +276,7 @@ export class ComputerUsePreviewPresenter sessionId, mode: 'stopped', hostWindowId: null, + hostClosedListener: null, runId: null, toolCallId: null, toolCallEpoch: 0, @@ -291,6 +295,51 @@ export class ComputerUsePreviewPresenter return state } + private bindHost(state: ComputerUsePreviewState, host: BrowserWindow): void { + if (state.hostWindowId === host.id && state.hostClosedListener) { + return + } + this.unbindHost(state) + const closed = () => { + if (state.hostClosedListener !== closed) { + return + } + state.hostClosedListener = null + this.stopState(state) + } + state.hostWindowId = host.id + state.hostClosedListener = closed + host.once('closed', closed) + } + + private unbindHost(state: ComputerUsePreviewState): void { + const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) + if (host && state.hostClosedListener) { + host.removeListener('closed', state.hostClosedListener) + } + state.hostWindowId = null + state.hostClosedListener = null + } + + private stopState(state: ComputerUsePreviewState): void { + if (this.states.get(state.sessionId) !== state) { + return + } + state.epoch += 1 + state.mode = 'stopped' + state.frame = null + state.pendingSnapshot = null + if (state.runId) { + this.previewCoordinator.releaseClaim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + this.unbindHost(state) + this.states.delete(state.sessionId) + } + private readPositiveInteger(value: unknown): number | null { return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null } @@ -436,10 +485,14 @@ export class ComputerUsePreviewPresenter announceSurface = false ): ComputerUsePreviewSurface { const target = this.previewTarget(state) - const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) - if (!target || !host || host.isDestroyed() || state.mode !== 'eligible') { + if (!target || state.mode !== 'eligible') { return this.setSurface(state, 'none', announceSurface) } + const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + this.stopState(state) + return 'none' + } return this.setSurface(state, this.previewCoordinator.prepare(target, host), announceSurface) } @@ -539,18 +592,7 @@ export class ComputerUsePreviewPresenter if (otherSessionId === sessionId || state.hostWindowId !== hostWindowId) { continue } - state.epoch += 1 - state.mode = 'stopped' - state.frame = null - state.pendingSnapshot = null - if (state.runId) { - this.previewCoordinator.releaseClaim({ - source: 'computer-use', - sessionId: state.sessionId, - runId: state.runId - }) - } - this.states.delete(otherSessionId) + this.stopState(state) } } diff --git a/src/main/mcp/toolManager.ts b/src/main/mcp/toolManager.ts index 027391110..a5d37c973 100644 --- a/src/main/mcp/toolManager.ts +++ b/src/main/mcp/toolManager.ts @@ -913,6 +913,7 @@ export class ToolManager { if ( input.toolName !== 'click' || input.response.isError === true || + getPluginToolPolicy(input.client.serverName, 'get_window_state') !== 'allow' || !this.computerUsePreviewObserver?.shouldCaptureAfterClick ) { return diff --git a/src/renderer/src/components/browser/AgentBrowserPiP.vue b/src/renderer/src/components/browser/AgentBrowserPiP.vue index b3ddd7a4a..b333af623 100644 --- a/src/renderer/src/components/browser/AgentBrowserPiP.vue +++ b/src/renderer/src/components/browser/AgentBrowserPiP.vue @@ -275,13 +275,13 @@ const loadStatus = async () => { } } -const dismiss = async () => { +const dismiss = () => { const sessionId = currentSessionId.value const runId = currentRunId.value if (!sessionId || !runId) return dismissedRunId.value = runId toolbarVisible.value = false - await browserClient.dismissPreview(sessionId, runId) + void browserClient.dismissPreview(sessionId, runId).catch(() => undefined) } const openInPanel = async () => { diff --git a/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue index f78963912..d56ba7599 100644 --- a/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue +++ b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue @@ -169,12 +169,12 @@ const resetFrame = (runId = '', epoch = -1) => { } } -const dismiss = async () => { +const dismiss = () => { const sessionId = currentSessionId.value const runId = currentRunId.value if (!sessionId || !runId) return dismissedRunId.value = runId - await computerUseClient.dismissPreview(sessionId, runId) + void computerUseClient.dismissPreview(sessionId, runId).catch(() => undefined) } const isControl = (target: EventTarget | null) => @@ -397,7 +397,11 @@ onBeforeUnmount(() => { stopPreviewSurfaceChanged?.() stopWindowStateChanged?.() if (currentSessionId.value) { - void computerUseClient.setPreviewMode(currentSessionId.value, 'stopped') + pendingPreviewRequest = { + sessionId: currentSessionId.value, + mode: 'stopped' + } + drainPreviewRequests() } }) diff --git a/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts index c63b35d94..49f56eaf6 100644 --- a/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts +++ b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts @@ -161,6 +161,27 @@ describe('ComputerUsePreviewPresenter', () => { expect(presenter.shouldCaptureAfterClick(click)).toBe(false) }) + it('releases preview state when the host closes without renderer cleanup', async () => { + const { presenter, coordinator, windows } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + presenter.started(call('snapshot')) + + const host = windows.get(7)! + host.destroyed = true + host.emit('closed') + + expect(coordinator.releaseClaim).toHaveBeenCalledWith({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + expect(presenter.shouldCaptureAfterClick(call('click', { toolName: 'click' }))).toBe(false) + await expect(presenter.setPreviewMode('session-1', 'stopped', 7)).resolves.toEqual({ + updated: false, + surface: 'none' + }) + }) + it('publishes a bounded Canvas frame only after a valid current snapshot', async () => { const { presenter, coordinator, windowPresenter } = await setup() await expect(presenter.setPreviewMode('session-1', 'eligible', 7)).resolves.toEqual({ diff --git a/test/main/mcp/toolManager.test.ts b/test/main/mcp/toolManager.test.ts index bf5a34cfe..33dc3d36e 100644 --- a/test/main/mcp/toolManager.test.ts +++ b/test/main/mcp/toolManager.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ToolManager, type ComputerUsePreviewObserver } from '@/mcp/toolManager' +import * as toolPolicyStore from '@/plugin/toolPolicyStore' function deferred() { let resolve!: (value: T) => void @@ -770,6 +771,10 @@ describe('ToolManager', () => { }) it('refreshes PiP after a trusted click without exposing or awaiting the private snapshot', async () => { + let snapshotPolicy: 'allow' | 'ask' = 'allow' + vi.spyOn(toolPolicyStore, 'getPluginToolPolicy').mockImplementation((_serverId, toolName) => + toolName === 'get_window_state' ? snapshotPolicy : null + ) const client = createClient( 'cua-driver', [ @@ -940,6 +945,25 @@ describe('ToolManager', () => { privateFailure ) expect(publishEvent).toHaveBeenCalledTimes(2) + + snapshotPolicy = 'ask' + const guardedResult = await manager.callTool( + { + id: 'cua-click-3', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":50,"y":60}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(guardedResult.content).toBe('clicked') + expect(client.callTool).toHaveBeenCalledTimes(5) + expect(observer.shouldCaptureAfterClick).toHaveBeenCalledTimes(2) + expect(observer.started).toHaveBeenCalledTimes(2) }) it('does not privately snapshot failed, invalid-target, or untrusted clicks', async () => { diff --git a/test/renderer/components/AgentBrowserPiP.test.ts b/test/renderer/components/AgentBrowserPiP.test.ts index 4720f2ca1..99757ff5c 100644 --- a/test/renderer/components/AgentBrowserPiP.test.ts +++ b/test/renderer/components/AgentBrowserPiP.test.ts @@ -194,12 +194,14 @@ describe('AgentBrowserPiP', () => { }) it('stays dismissed for the current run', async () => { - const { wrapper } = await setup() + const { wrapper, browserClient } = await setup() + browserClient.dismissPreview.mockRejectedValueOnce(new Error('dismiss failed')) await wrapper.get('[aria-label="common.close"]').trigger('click') - await nextTick() + await flushPromises() expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + expect(browserClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') }) it('renders no Canvas surface and handles native panel actions', async () => { diff --git a/test/renderer/components/AgentComputerUsePiP.test.ts b/test/renderer/components/AgentComputerUsePiP.test.ts index 417f5788f..44b46a2c4 100644 --- a/test/renderer/components/AgentComputerUsePiP.test.ts +++ b/test/renderer/components/AgentComputerUsePiP.test.ts @@ -1,10 +1,20 @@ import { defineComponent, nextTick, reactive } from 'vue' import { flushPromises, mount } from '@vue/test-utils' import { afterEach, describe, expect, it, vi } from 'vitest' +import type { + ComputerUsePreviewMode, + ComputerUsePreviewModeResult +} from '@shared/types/computerUse' const mountedWrappers: Array<{ unmount: () => void }> = [] -const setup = async (surface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas') => { +const setup = async ( + surface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas', + setPreviewMode?: ( + sessionId: string, + mode: ComputerUsePreviewMode + ) => Promise +) => { vi.resetModules() vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ x: 0, @@ -47,10 +57,11 @@ const setup = async (surface: 'native-overlay' | 'renderer-canvas' = 'renderer-c const stopWindowStateSubscription = vi.fn() const computerUseClient = { setPreviewMode: vi.fn( - async (_sessionId: string, mode: 'eligible' | 'suspended' | 'stopped') => ({ - updated: true, - surface: mode === 'eligible' ? surface : ('none' as const) - }) + setPreviewMode ?? + (async (_sessionId: string, mode: 'eligible' | 'suspended' | 'stopped') => ({ + updated: true, + surface: mode === 'eligible' ? surface : ('none' as const) + })) ), dismissPreview: vi.fn(async () => true), onPreviewSurfaceChanged: vi.fn((handler: (payload: SurfacePayload) => void) => { @@ -205,6 +216,7 @@ describe('AgentComputerUsePiP', () => { expect(pip.get('button').attributes('aria-label')).toBe('common.close') expect(wrapper.find('[aria-label="common.open"]').exists()).toBe(false) + computerUseClient.dismissPreview.mockRejectedValueOnce(new Error('dismiss failed')) await pip.get('button').trigger('click') await flushPromises() @@ -371,6 +383,32 @@ describe('AgentComputerUsePiP', () => { expect(stopWindowStateSubscription).toHaveBeenCalledOnce() }) + it('queues teardown behind an in-flight preview mode update', async () => { + let resolveEligible: ((result: ComputerUsePreviewModeResult) => void) | undefined + const setPreviewMode = vi.fn(async (_sessionId: string, mode: ComputerUsePreviewMode) => { + if (mode === 'eligible') { + return await new Promise((resolve) => { + resolveEligible = resolve + }) + } + return { updated: true, surface: 'none' as const } + }) + const { wrapper } = await setup('renderer-canvas', setPreviewMode) + const callsBeforeUnmount = setPreviewMode.mock.calls.length + expect(setPreviewMode).toHaveBeenLastCalledWith('session-1', 'eligible') + + wrapper.unmount() + mountedWrappers.splice(mountedWrappers.indexOf(wrapper), 1) + await nextTick() + + expect(setPreviewMode).toHaveBeenCalledTimes(callsBeforeUnmount) + resolveEligible?.({ updated: true, surface: 'renderer-canvas' }) + await flushPromises() + + expect(setPreviewMode).toHaveBeenCalledTimes(callsBeforeUnmount + 1) + expect(setPreviewMode).toHaveBeenLastCalledWith('session-1', 'stopped') + }) + it('stops and releases the fallback when the Agent run terminates', async () => { const { wrapper, computerUseClient, sessionStore, emitSurface } = await setup() emitSurface(surfacePayload())