diff --git a/docs/architecture/nativekit-agent-browser-pip/spec.md b/docs/architecture/nativekit-agent-browser-pip/spec.md index 462c93ee7..3a497ef37 100644 --- a/docs/architecture/nativekit-agent-browser-pip/spec.md +++ b/docs/architecture/nativekit-agent-browser-pip/spec.md @@ -56,6 +56,13 @@ contracts remain in force. The current renderer Canvas remains the compatibility GitHub issue sync was not requested and was not performed. +On 2026-07-28, a +[Computer Use latest-snapshot PiP extension](../../features/computer-use-snapshot-pip/spec.md) was +implemented. `AgentPreviewCoordinator` now owns the process-global NativeKit lifecycle and +arbitrates one Browser or Computer Use presentation. Browser retains its page, capture, panel +handoff, and **Open in panel** plus **Close** behavior; Computer Use contributes only bounded latest +snapshots with **Close**. + ## Counterpoint The NativeKit path will make panel dragging fluid and allow the panel to leave the DeepChat window. @@ -247,21 +254,24 @@ focusless render-host BaseWindow -> Agent WebContentsView at 1280 x 800 - branches completed frames to exactly one surface; - stops capture and clears the surface on every existing lifecycle edge. -#### `AgentBrowserNativeOverlay` +#### `AgentPreviewCoordinator` -A focused main-process adapter owns only: +A focused process-global main-process coordinator owns only: - dynamic package loading and one-time capability detection; - `overlay.start()` / `stop()` lifecycle; - one active host and one active presentation; +- Browser and Computer Use toolbar profile switching; +- latest explicit source claim and shared run-scoped dismissal; - host move/resize/close synchronization; - JPEG `Buffer` to data-URL conversion; - prepaint-before-show ordering; -- mapping NativeKit `activate` and configured `control` IDs to the current logical - `{ sessionId, runId, windowId }`; +- mapping NativeKit activation and configured controls to the current source-specific handler; - timing counters around synchronous native calls. -It is not a general Presenter, registry, plugin system, or cross-native abstraction. +`YoBrowserPresenter` remains the Browser producer and `ComputerUsePreviewPresenter` remains the +Computer Use producer. The coordinator is not a public preview framework, plugin system, or +cross-native abstraction. #### Renderer diff --git a/docs/features/agent-browser-pip/spec.md b/docs/features/agent-browser-pip/spec.md index b5b471222..a24a4adc3 100644 --- a/docs/features/agent-browser-pip/spec.md +++ b/docs/features/agent-browser-pip/spec.md @@ -16,6 +16,12 @@ Canvas described here remains the compatibility fallback. This document's page o background viewport, read-only preview, panel handoff, and run-scoped dismissal contracts remain authoritative. +On 2026-07-28, a +[Computer Use latest-snapshot PiP extension](../computer-use-snapshot-pip/spec.md) was implemented. +It shares only process-global NativeKit presentation ownership; it does not generalize Browser +page, capture, panel-handoff, or public feature contracts. Browser keeps **Open in panel** plus +**Close**, continuous bounded capture, and its existing Canvas compatibility behavior. + The referenced screenshot was not available in the current task context, so the interaction and layout are specified here while exact visual styling remains provisional. diff --git a/docs/features/computer-use-snapshot-pip/plan.md b/docs/features/computer-use-snapshot-pip/plan.md new file mode 100644 index 000000000..eeaa60768 --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/plan.md @@ -0,0 +1,333 @@ +# Computer Use Snapshot Picture-in-Picture Implementation Plan + +## Status + +Implemented on branch `codex/computer-use-snapshot-pip` on 2026-07-28. + +Phases 1-5 are implemented with focused automated coverage, and the complete renderer suite passes. +The complete main suite retains one isolated, unrelated provider-metadata expectation failure. +Performance measurement and packaged native and fallback platform QA remain open; the exact +remaining work is recorded in `tasks.md`. + +CUA runtime, plugin, policy, skill, and asset updates are explicitly out of scope. The plan consumes +successful Agent-requested snapshots plus one bounded private `get_window_state` result after an +eligible successful `click`. + +## Delivery Strategy + +Establish one process-global PiP owner before adding the Computer Use presenter. NativeKit toolbar +configuration is global, so implementing a second standalone overlay adapter first would create a +known Browser/Computer race. + +Then carry existing Agent run identity through the MCP execution path and observe only resolved, +trusted CUA calls. Build the snapshot transform and renderer fallback after those identities are +available. Finish with arbitration, lifecycle hardening, and Browser regression verification. + +```text +shared NativeKit owner + | + +--> existing Browser producer + | + +--> Computer Use observer -> presenter -> snapshot producer + | + native or Canvas surface +``` + +## Phase 1: Shared PiP Ownership + +### 1. Extract the process-global overlay owner + +Refactor `AgentBrowserNativeOverlay` so NativeKit lifecycle and host synchronization are owned by +one coordinator instantiated in `src/main/app/composition.ts`. + +Keep the extraction narrow: + +- one dynamic NativeKit import and capability result; +- one `overlay.start()` and `overlay.stop()` lifecycle; +- one host attachment and host/display listener set; +- one visible presentation; +- toolbar profiles for Browser and Computer Use; +- source-specific activation/control callbacks; +- common visibility, removal, prepaint, and latency behavior. + +The Browser presenter remains authoritative for Browser page/run/capture state. Migrate its current +native adapter calls to the coordinator without changing Browser contracts or fallback behavior. + +### 2. Add explicit toolbar profiles + +Configure: + +- Browser: **Open in panel**, then **Close**; activation opens the Browser panel. +- Computer Use: **Close** only; activation is ignored. + +Switch profiles before presenting a source. Hide/remove the previous presentation during a profile +change so controls can never act on the wrong target. + +### 3. Add claim arbitration + +Represent the latest retained claim per source and host/session with run, target identity, and a +monotonic activity sequence. + +- Explicit Browser tool activity claims Browser. +- An actual resolved CUA `get_window_state` invocation claims Computer Use. +- Frame updates are refreshes and cannot claim. +- A background or inactive host/session claim cannot preempt the foreground active session. +- A producer may present only while its claim is the latest eligible sequence for the foreground + active host/session. +- Focus/session changes reevaluate retained sequences without manufacturing new activity. +- Dismissal applies to the current shared session/run. + +Add coordinator unit tests before wiring the Computer Use image path. + +## Phase 2: MCP Execution Identity + +### 4. Propagate `runId` + +Extend MCP call options through: + +1. `McpServicePort.callTool`. +2. `McpService.callTool`. +3. `ToolManager.callTool`. +4. Narrow test doubles and call sites that implement those contracts. + +Update `ToolService` to forward `ToolCallOptions.runId` on the MCP branch alongside access and abort +metadata. Do not add it to `MCPToolCall`, serialize it to the MCP server, or place it in tool +arguments. + +Add a focused regression test proving that an Agent run ID reaches `ToolManager` for MCP calls and +that the outgoing CUA arguments remain byte-for-byte equivalent. + +### 5. Introduce a narrow Computer Use observer + +Define an injected main-process observer port with `started`, `completed`, and `failed` callbacks. +Call it only after `ToolManager` has: + +- resolved the real MCP client and original tool name; +- loaded and validated the server config; +- completed access and permission checks; +- repaired and prepared tool arguments; +- identified the plugin source through `ownerPluginId` or `sourceId`. + +Emit `started` immediately before actual client invocation. Emit one terminal callback without +changing response mapping or error propagation. + +Calls lacking run/session identity and non-CUA calls should produce no preview callbacks. + +### 5a. Add a private post-click refresh + +After a successful exact `click` on the resolved official CUA client: + +- require valid current session/run plus positive integer `pid` and `window_id`; +- synchronously ask the presenter whether the same target is PiP-eligible and not dismissed; +- asynchronously call `get_window_state` on that already resolved client with only + `{ pid, window_id }`; +- send the private result only through the preview observer; +- never publish or return the private result to the Agent or conversation; +- retain the original click response and latency behavior if capture fails. + +Do not trigger on permission responses, failed clicks, stale targets, `right_click`, +`double_click`, inactive PiP, or arbitrary servers. Do not add timers, polling, or CUA runtime +changes. + +## Phase 3: Computer Use Snapshot Presenter + +### 6. Add presenter state and lifecycle + +Create `ComputerUsePreviewPresenter` in `src/main/desktop/computerUse/` and instantiate it once from +the application composition root. + +Track: + +- host window and active chat session; +- run and tool call; +- `pid` and `windowId`; +- epoch and active coordinator claim; +- renderer preview mode; +- dismissed run; +- current valid encoded frame; +- transform in-flight and latest pending result. + +On a new run or target, advance the epoch, hide/remove the old presentation, release the old frame, +and wait for the first valid new frame. + +### 7. Implement snapshot recognition + +On observer callbacks: + +- require the trusted Computer Use plugin identity; +- require original tool name `get_window_state`; +- parse finite positive integer `pid` and `window_id`; +- claim the Computer Use source at invocation start; +- accept only a successful result containing a supported inline image; +- retain the prior valid same-target frame on failure; +- discard results whose session, run, target, tool call, claim, or epoch is stale. + +Use shared utilities only where an existing utility already matches the raw inline MCP image +contract. Do not route through cached message-preview artifacts. + +### 8. Build the bounded image pipeline + +In main: + +1. Select the first supported PNG/JPEG inline image block. +2. Validate base64 and a 16 MiB encoded-input ceiling. +3. Decode and validate dimensions no larger than 8192 x 8192. +4. Resize proportionally within 480 x 300 without upscaling. +5. Encode JPEG quality 72. +6. Reject output larger than 512 KiB. +7. Revalidate epoch and claim. +8. Present to NativeKit or emit the Canvas fallback frame. + +Keep one transform in flight and one replaceable latest pending result. Add rate-limited, +metadata-only warnings for validation and latency failures. + +## Phase 4: Typed Desktop Contracts and Renderer Fallback + +### 9. Add routes and events + +Add Computer Use preview schemas beside the existing desktop/browser contract organization: + +- `computerUse.setPreviewMode`; +- `computerUse.dismissPreview`; +- `computerUse.preview.frame`. + +Expose them through the existing typed preload/client boundary. Validate route sender and session +binding in main. Emit frame events only for Canvas fallback. + +### 10. Add the renderer controller + +Mount `AgentComputerUsePiP.vue` beside `AgentBrowserPiP.vue` in `ChatTabView.vue`. + +Derive: + +- active chat session; +- working versus terminal session state; +- current route; +- host focus. + +Send `eligible`, `suspended`, or `stopped` only when the derived mode changes. On native surfaces, +render nothing and never subscribe to image payloads. + +### 11. Add the Canvas fallback + +For fallback only: + +- render one bounded Canvas card; +- expose one existing translated **Close** button; +- make non-control surface points draggable with pointer capture; +- decode a frame before replacing current Canvas pixels; +- reject stale session/run/sequence events; +- release image resources on replacement and unmount; +- forward no input to the target application. + +Do not add a generic Browser/Computer Vue component unless the completed implementation has +meaningful stable duplication. + +## Phase 5: Lifecycle, Privacy, and Failure Hardening + +### 12. Complete foreground lifecycle + +Use existing window/session binding and host listeners to: + +- hide on blur, hide, minimize, inactive route, or suspended mode; +- restore only when the same current claim remains eligible; +- remove on stopped session, terminal run, target change, host close, or shutdown; +- prevent one host/session from seeing another session's retained frame. + +When the target application comes to the foreground, DeepChat loses focus and the preview hides. + +### 13. Enforce run-scoped dismissal + +Validate exact session and run on close. Suppress all later frames and source changes for that run +without canceling tools. Clear suppression only for a later run or terminal cleanup. + +### 14. Enforce memory-only image handling + +Audit logs, events, caches, and teardown: + +- no base64 or pixel buffer logging; +- no disk writes or settings persistence; +- no generic MCP image broadcast; +- no renderer image payload on native path; +- prompt release of stale buffers, Canvas resources, and native presentations. + +## Phase 6: Verification and Rollout + +### 15. Add focused automated coverage + +Add the smallest tests for: + +- shared coordinator profile switching, claims, dismissal, and stale refresh rejection; +- Browser toolbar/activation/dismissal regression; +- MCP `runId` propagation and unchanged outgoing arguments; +- trusted source recognition and permission/non-invocation paths; +- successful, failed, aborted, and out-of-order observer callbacks; +- presenter run/target/epoch transitions; +- image validation, resize, output limit, and latest-wins behavior; +- native delivery versus Canvas-only frame publication; +- renderer focus/session modes, close, frame retention, and cleanup. + +### 16. Measure performance + +Instrument metadata-only durations for result receipt, transform completion, and NativeKit push. +Verify: + +- at most one added capture call per eligible successful `click` and none while PiP is ineligible; +- zero idle polling; +- one in-flight transform; +- p95 result-to-visible latency at or below 150 ms; +- no renderer image traffic on native surface; +- no unbounded buffer retention. + +### 17. Run platform QA + +Verify at least: + +- one packaged native-overlay runtime; +- one packaged Canvas-fallback runtime; +- host focus/minimize/restore; +- target application foreground transition; +- Browser-to-Computer and Computer-to-Browser claims; +- native toolbar profile switching; +- run close/restart and session switching; +- malformed and oversized image handling. + +Keep the feature behind an internal gate until both surface paths pass if the platform matrix cannot +complete in the implementation change. + +### 18. Close documentation + +After implementation: + +- update `spec.md` status and record material deviations; +- update the linked Browser NativeKit architecture with the final shared-owner contract; +- mark completed tasks in `tasks.md`; +- include the BEFORE/AFTER ASCII layout in the PR; +- record remaining packaged platform QA explicitly. + +## Verification Commands + +Run after implementation: + +```bash +pnpm run format +pnpm run i18n +pnpm run lint +pnpm run typecheck +pnpm run test:main +pnpm run test:renderer +pnpm exec electron-vite build +``` + +Prefer narrower test file invocations during development. Run the complete affected main and +renderer suites before handoff because the change crosses MCP execution, desktop presentation, and +renderer lifecycle boundaries. + +## Rollback + +The Computer Use observer and presenter must be independently disableable. Disabling them restores +the previous behavior without changing CUA tool execution. + +If the shared coordinator causes a Browser regression, revert the coordinator integration as one +unit and keep the run-propagation changes only if their focused tests pass and no public behavior +changes. Do not fall back to two independent NativeKit owners. diff --git a/docs/features/computer-use-snapshot-pip/spec.md b/docs/features/computer-use-snapshot-pip/spec.md new file mode 100644 index 000000000..08c3f0871 --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/spec.md @@ -0,0 +1,673 @@ +# Computer Use Snapshot Picture-in-Picture Spec + +## Status + +Implemented on branch `codex/computer-use-snapshot-pip` on 2026-07-28. + +Format, i18n, lint, typecheck, focused main/renderer coverage, and the complete renderer suite pass. +The complete main suite has one unrelated provider-metadata expectation failure in +`test/main/provider/modelConfig.test.ts`; the same test fails in isolation and neither it nor its +implementation is changed by this feature. Packaged native-overlay and Canvas-fallback QA, +cross-platform interaction checks, and the 150 ms p95 result-to-visible measurement remain open and +are tracked in `tasks.md`. + +This feature adds a read-only picture-in-picture surface for the current Computer Use target by +reusing successful `get_window_state` image results. In addition to Agent-requested snapshots, one +eligible successful `click` may schedule one private `get_window_state` refresh whose result is +consumed only by PiP. It does not add a capture loop, change the CUA runtime, enable the upstream +experimental PiP, or modify CUA plugin policies and skill instructions. + +The existing +[Agent Browser NativeKit architecture](../../architecture/nativekit-agent-browser-pip/spec.md) +now uses the implemented process-global, single-PiP coordinator shared with Computer Use while +preserving Browser-specific behavior. + +GitHub issue sync was not requested and was not performed. + +## Counterpoint + +Calling this surface "live Computer Use PiP" would promise behavior the available data source does +not provide. A Computer Use tool result contains a point-in-time image only when +`get_window_state` runs; no frame exists between those calls. + +Polling `get_window_state` merely to animate PiP is the wrong tradeoff. The call also performs +accessibility-state work and updates the driver's element cache, so extra polling would consume +driver, IPC, and model-adjacent resources while changing the semantics of the automation flow. + +An event-driven refresh after a successful `click` is a narrower tradeoff than polling: it adds at +most one capture at a user-visible state boundary, only while PiP is eligible, and does not put the +private result on the Agent path. `get_window_state` still refreshes the CUA accessibility cache, so +Agents must continue to obtain their own current state before selecting a later element index. + +The correct version is therefore an honest **latest snapshot** surface: + +- no idle or timer-driven captures; +- at most one private snapshot request after each eligible successful `click`; +- first appearance only after a valid snapshot exists; +- immediate replacement when the next valid snapshot arrives; +- retention of the last valid current-run image between snapshots; +- no video, freshness animation, or fake activity. + +## Feasibility Assessment + +The snapshot PiP is feasible without changing CUA. + +| Question | Finding | Consequence | +| --- | --- | --- | +| Is a usable image already available? | Successful `get_window_state` results contain an inline MCP image block. | Reuse Agent-requested results and issue only an event-driven private refresh after eligible clicks; add no capture API or polling. | +| Can the existing PiP surface be reused? | NativeKit accepts PNG/JPEG data URLs and supports repeated toolbar reconfiguration. | Share one native owner and switch explicit Browser/Computer profiles. | +| Can a result be tied to the correct Agent run? | The Agent runtime has a stable `runId`, but the MCP branch currently drops it. | Forward internal execution metadata through the MCP service path. | +| Can stale pixels be isolated? | ToolManager knows the resolved plugin source, original tool, prepared target arguments, and tool-call identity. | Observe at that boundary and validate run, target, claim, and epoch before display. | +| Does every platform have a native overlay? | No; the existing NativeKit matrix has Windows arm64, native Wayland, and load-failure gaps. | Preserve a small renderer Canvas fallback. | + +The main engineering risk is not screenshot performance. It is ownership: NativeKit is +process-global, while Browser and Computer Use have different controls and lifecycles. A shared +coordinator is therefore required for correctness. With that boundary in place, the Computer Use +path is a bounded image-result consumer rather than a new screen-capture subsystem. + +## Problem + +When an Agent operates a desktop application through Computer Use, the user can see tool calls in +the conversation but cannot see the target application's latest state without switching away from +DeepChat. That makes long-running automation difficult to supervise. + +The existing Agent Browser PiP solves a related problem, but it cannot be copied directly: + +- Browser owns a continuously capturable `WebContentsView`; Computer Use receives discrete MCP + image results from an external application. +- Browser PiP offers **Open in panel**; Computer Use has no DeepChat side panel to open. +- NativeKit exposes one process-global overlay manager whose toolbar is configured by + `overlay.start()`. Two independent adapters would race and replace each other's configuration. +- Browser emits frames continuously at a bounded rate; Computer Use uses discrete Agent-requested + and post-click snapshots and must not add background polling to simulate that behavior. + +## User Experience + +### Before + +```text +┌──────────────────────────────────────┐ +│ DeepChat │ +│ │ +│ Agent tool calls only; the target │ +│ application state is not visible. │ +└──────────────────────────────────────┘ +``` + +### After + +```text +┌──────────────────────────────────────┐ ┌────────────────── × ┐ +│ DeepChat │ │ │ +│ │ │ latest target app │ +│ Agent is working │ │ snapshot │ +└──────────────────────────────────────┘ └────────────────────┘ +``` + +The Computer Use PiP: + +- shows only the latest valid snapshot for the active target window; +- is draggable from any non-control point; +- has one **Close** button and no expand, open-panel, title, status, or action controls; +- is read-only and never forwards pointer, wheel, keyboard, or focus input to the target + application; +- closes only the preview for the current run and never closes the target application, cancels the + Agent, or interrupts a tool; +- disappears while DeepChat is not the foreground application, allowing the user to inspect the + real target application instead. + +Post-click freshness changes without changing the layout: + +```text +BEFORE click -> Agent receives click result -> PiP retains the prior snapshot +AFTER click -> Agent receives click result + `-> private get_window_state -> PiP replaces the snapshot +``` + +The existing `common.close` translation is sufficient. This feature adds no user-facing copy. + +## Goals + +- Let users supervise the current Computer Use target without leaving DeepChat. +- Reuse inline images returned by successful CUA `get_window_state` calls. +- Refresh PiP after an eligible successful `click` without exposing the private snapshot result to + the Agent, conversation, or generic MCP result event. +- Show the first PiP only after a valid current-run image has been decoded. +- Update the PiP after each later valid snapshot for the same active target. +- Keep the last valid image visible between snapshots without showing stale images from another + run, session, or target. +- Prefer the existing NativeKit overlay on supported runtimes and preserve the renderer Canvas + fallback elsewhere. +- Share one process-global native overlay safely between Browser and Computer Use. +- Preserve Browser PiP's existing **Open in panel** plus **Close** controls. +- Give Computer Use exactly one **Close** control and ignore native activation. +- Keep image bytes in the main process on the native path. +- Bound image input, transform work, output size, and queued work. +- Hide and remove the surface deterministically across focus, route, session, run, target, and + shutdown lifecycle edges. + +## Non-Goals + +- Updating the CUA runtime, driver, plugin manifest, policies, skill, or packaged assets. +- Enabling or integrating the upstream CUA experimental PiP. +- Adding a CUA driver screenshot hook, daemon socket, callback, or runtime patch. +- Polling `get_window_state` to produce a 2-4 FPS stream. +- Capturing external windows through Electron `desktopCapturer` or a new native capture module. +- Presenting video, shared textures, WebRTC, or a GPU surface. +- Interacting with the target application through the PiP. +- Opening a side panel or adding an expand button for Computer Use. +- Displaying more than one PiP at a time. +- Persisting PiP position, size, image, target identity, or dismissal across app restarts. +- Publishing raw Computer Use images through a generic MCP result event. +- Generalizing all tool previews into a public framework. + +## Terminology + +- **Computer Use source**: the trusted plugin-owned MCP server identified by + `ownerPluginId` or `sourceId`, not by a user-editable server name. +- **Snapshot call**: a CUA `get_window_state` tool invocation with valid `pid` and `window_id` + arguments. +- **Private post-click snapshot**: an asynchronous `get_window_state` invocation made through the + same resolved official CUA client after a successful `click`; its result is PiP-only. +- **Snapshot frame**: the first supported inline image block in a successful snapshot result. +- **Target**: the tuple `(pid, windowId)` within one chat session and Agent run. +- **Run**: the stable Agent execution identity already carried as `ToolCallOptions.runId`. +- **Claim**: explicit Browser tool activity or a Computer Use snapshot-call start that requests + ownership of the single PiP surface. +- **Refresh**: a new frame for the source that already owns the PiP. A refresh is not a claim. +- **Eligible**: the renderer session is active and working; the presenter may show a valid frame + when the host is foreground. +- **Suspended**: retain current target and frame but hide the surface temporarily. +- **Stopped**: remove the presentation and target state for the session. + +`sessionId` in the preview contract is the current chat `conversationId`. + +## Baseline Repository Contract + +- `YoBrowserPresenter` owns Browser capture, preview mode, and frame delivery. +- `AgentBrowserNativeOverlay` owns the current NativeKit integration and configures + **Open in panel** plus **Close** during `overlay.start()`. +- NativeKit provides one process-global overlay manager. Starting a second independent adapter + would replace global toolbar configuration. +- `AgentBrowserPiP.vue`, mounted by `ChatTabView.vue`, coordinates Browser preview eligibility and + renders the Canvas fallback. +- The Agent runtime already passes `io.requestId` as `ToolCallOptions.runId`. +- `ToolService` forwards `runId` to built-in Agent tools but currently drops it on the MCP path + before `McpService.callTool`. +- `ToolManager` resolves the actual MCP client, original tool name, plugin identity, repaired + arguments, permissions, and prepared arguments before invoking the tool. +- CUA recognition already uses plugin ownership metadata rather than the server display name. +- A successful MCP image block has `type: 'image'`, base64 `data`, and `mimeType`. +- The generic `mcp.toolCall.result` publication does not carry the complete run, source, and target + identity needed for safe preview routing and must not become a pixel transport. +- `toolCallImagePreviews` caches result images for conversation rendering after tool execution. PiP + must consume the inline result before that persistence path and must not create another cached + copy. + +## Implemented Deviations + +- Added source-specific `computerUse.preview.surface.changed` and + `browser.preview.surface.changed` events. They carry only bounded identity and surface metadata, + never image bytes, and let renderers react to native failure and cross-source arbitration without + polling. +- Added an `epoch` to Computer Use surface and frame events so a later target in the same run cannot + accept an in-flight frame from the prior target. +- The shared coordinator permanently falls back to Canvas after a NativeKit load, startup, toolbar, + host-attach, or frame-push failure for the current process. It does not repeatedly retry a failed + native path during Agent work. +- An eligible successful official CUA `click` schedules one non-blocking private + `get_window_state({ pid, window_id })` call. The private response is routed only to the preview + observer and is never published as an MCP result or returned to the Agent. +- No CUA runtime, driver, plugin manifest, policy, skill, or packaged asset was changed. + +## Product Contract + +### Frame Source + +Only an actual Agent-requested or private post-click invocation that meets every condition may +update the preview: + +1. The resolved MCP client belongs to the official Computer Use plugin. +2. The resolved original tool name is `get_window_state`. +3. The prepared arguments contain finite positive integer `pid` and `window_id` values. +4. The tool result is not an error. +5. The result contains a PNG or JPEG inline image block. +6. The call still matches the active session, run, target, tool call, and presenter epoch when + asynchronous transform work completes. + +Text, resources, cached preview references, permission responses, failed calls, and images from +other CUA tools are not frame sources. + +### Private Post-click Refresh + +- Only the exact resolved original tool `click` is eligible; `right_click`, `double_click`, and + other actions do not implicitly expand this contract. +- The click must succeed on the resolved official CUA client and carry finite positive integer + `pid` and `window_id` values. +- The presenter must confirm that PiP is eligible and that the click still matches its active + session, run, target, and non-dismissed state. +- The private call uses only `{ pid, window_id }`, starts asynchronously after click completion, and + does not delay or change the click response. +- The private call bypasses Agent-facing permission and result publication only after confirming + that the triggering click passed access checks and the current plugin policy explicitly allows + `get_window_state`. +- The private result enters only the same bounded preview observer and image pipeline. Its text, + accessibility tree, and other content are discarded. +- A private-call failure retains the last valid frame and never changes Agent execution. +- No private snapshot is scheduled while the preview is stopped, suspended, dismissed, stale, or + missing a valid active target. + +### First Frame and Refresh + +- A snapshot call start claims Computer Use ownership immediately before MCP invocation, after + permission checks and argument preparation. +- Claiming a new run or target hides and removes the prior presentation before any asynchronous + result can arrive. +- The PiP does not appear until the first valid frame for that claim is decoded and transformed. +- A later valid snapshot for the same run and target replaces the visible frame. +- A failed snapshot keeps the last valid frame for the same active run and target. +- A failed or invalid result never replaces a valid frame with an error image or placeholder. +- Results from a superseded call, run, target, session, or epoch are discarded. + +### Eligibility and Visibility + +The Computer Use PiP may be visible only when all of these are true: + +- its chat route and session are active in the host renderer; +- the session status is `working`; +- the host window exists, is shown, is not minimized, and is focused; +- Computer Use currently owns the shared PiP claim; +- a valid frame exists for the active run and target; +- the current run has not been dismissed. + +Side-panel visibility is irrelevant to Computer Use PiP. It remains relevant to Browser PiP under +the existing Browser contract. + +Temporary host blur, hide, minimize, route switch, or renderer suspension hides the surface without +discarding the valid current target frame. Session terminal state, teardown, new run, target change, +host destruction, or app shutdown removes the presentation. + +### Dismissal + +- **Close** records dismissal for the active session and run. +- Dismissal hides/removes the current shared presentation without changing the tool or target + application. +- Later frames from the dismissed run remain suppressed. +- A later Agent run clears the suppression and may show a new preview after its first valid frame. +- Dismissal is shared at the one-PiP coordinator level so a Browser/Computer source transition + within the same run cannot immediately reopen a surface the user just closed. + +## Target Architecture + +```text +Agent runtime + | + | ToolCallOptions.runId + v +ToolService -> McpService -> ToolManager + | + | resolved trusted CUA client + | original tool + prepared args + v + ComputerUsePreviewObserver + | started | completed/failed + v + ComputerUsePreviewPresenter + | + | validate -> decode -> resize -> JPEG + v + AgentPreviewCoordinator (one per process) + | claim/arbitrate/lifecycle + | + +------+------------------+ + | | + v v + NativeKit overlay renderer Canvas fallback + main-process image typed bounded frame event + Computer toolbar: Close Computer toolbar: Close +``` + +### Shared Native Overlay Ownership + +Extract the process-global concerns currently inside `AgentBrowserNativeOverlay` into one +`AgentPreviewCoordinator` (or equivalently focused shared overlay adapter), instantiated once in +`src/main/app/composition.ts`. + +The shared owner is responsible for: + +- one NativeKit `start()` / `stop()` lifecycle; +- capability detection and Canvas fallback selection; +- one attached host and one visible presentation; +- toolbar profile switching before a source is presented; +- display, host move/resize, focus, show/hide, minimize/restore, and close listeners; +- one eligible source claim and monotonic claim sequence per host/session; +- shared run-scoped dismissal; +- prepaint-before-show ordering; +- NativeKit latency instrumentation and failure fallback. + +Source profiles remain explicit: + +| Source | Toolbar | Native activation | Source action | +| --- | --- | --- | --- | +| Browser | **Open in panel**, **Close** | Open Browser panel | Browser presenter callback | +| Computer Use | **Close** | Ignore | Computer presenter dismissal | + +Do not instantiate two adapters that independently call `overlay.start()`. Do not broaden Browser +routes, frame contracts, or page ownership into a generic public preview API. + +### Source Arbitration + +There is one visible PiP per DeepChat process. + +- Claims from an inactive or background host/session never preempt the foreground active session. +- Within the foreground active host/session, the most recent explicit source claim wins. +- Browser tool activity is a Browser claim. +- An actual CUA `get_window_state` invocation start is a Computer Use claim. +- Frame arrival and periodic Browser capture are refreshes, not claims. +- A refresh may update only the source that still owns the claim. +- A source losing ownership retains only the state needed by its existing lifecycle; it must not + remain visible. +- Reclaim requires later explicit tool activity. +- When focus or active session changes, the coordinator reevaluates the latest retained claim for + that newly eligible host/session instead of assigning a new activity sequence. + +This distinction prevents Browser's 4 FPS capture loop from continuously stealing the PiP back from +a newly active Computer Use target, and prevents a background Agent from hiding or leaking into the +foreground session. + +### MCP Run Identity and Observer + +Add `runId?: string` to the internal MCP call options and forward it through: + +```text +ToolService.callTool + -> McpServicePort.callTool + -> McpService.callTool + -> ToolManager.callTool +``` + +The value is execution metadata only. Never inject `runId` into the CUA tool arguments. + +Inject a narrow `ComputerUsePreviewObserver` into `ToolManager`. It receives only resolved +execution facts: + +```ts +type ComputerUsePreviewCall = { + conversationId: string + runId: string + toolCallId: string + toolName: string + args: Record + source: { + serverName: string + ownerPluginId?: string + sourceId?: string + } +} + +interface ComputerUsePreviewObserver { + shouldCaptureAfterClick?(call: ComputerUsePreviewCall): boolean + started(call: ComputerUsePreviewCall): void + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void + failed(call: ComputerUsePreviewCall, error: unknown): void +} +``` + +Invoke `started` immediately before the resolved MCP client call. Invoke exactly one terminal +callback after success or failure. The optional post-click predicate is synchronous, failure-safe, +and may authorize only a PiP-eligible current target. Agent-requested observer work must not alter +the MCP response, permission flow, abort semantics, tool latency, or error propagation. Private +post-click work is asynchronous and must not delay or mutate the original click result. + +Calls without a conversation ID or run ID remain valid MCP calls but are not preview-eligible. + +### Computer Use Presenter + +Add a main-process `ComputerUsePreviewPresenter` that owns: + +```ts +type ComputerUsePreviewTarget = { + hostWindowId: number + sessionId: string + runId: string + toolCallId: string + pid: number + windowId: number + epoch: number + dismissedRunId: string | null +} +``` + +The presenter: + +- recognizes snapshot calls and target changes; +- advances `epoch` before each new run or target; +- resolves the host from the active renderer/session binding instead of trusting renderer-supplied + window IDs; +- validates and decodes supported inline images in the main process; +- resizes and encodes one bounded JPEG; +- keeps at most one transform in flight per active target; +- drops queued intermediate work and retains only the latest pending result; +- revalidates session, run, target, tool call, claim, and epoch before presentation; +- sends a data URL directly to NativeKit or a typed byte event only to the Canvas fallback; +- owns stop, suspend, resume, dismiss, and shutdown cleanup. + +### Renderer Coordination and Fallback + +Add a focused `AgentComputerUsePiP.vue` beside `AgentBrowserPiP.vue` in `ChatTabView.vue`. + +On the native path it is a headless lifecycle controller: + +- derive active session and `working` status from existing stores; +- observe host focus through the existing `WindowClient`; +- call typed mode routes only when derived state changes; +- render no image DOM and receive no frame bytes. + +On the fallback path it renders a small Canvas card with exactly one **Close** button. It retains +the previous Canvas pixels until a fully decoded new frame is ready and revokes all object URLs and +image resources on replacement/unmount. + +Keep the Computer Use component separate from `AgentBrowserPiP.vue`. Extract shared renderer logic +only if implementation reveals stable duplication; the source rules and controls are intentionally +different. + +## Typed Contracts + +Add focused Computer Use preview contracts rather than exposing generic MCP results. + +### Routes + +```ts +type ComputerUsePreviewMode = 'eligible' | 'suspended' | 'stopped' + +type SetComputerUsePreviewModeInput = { + sessionId: string + mode: ComputerUsePreviewMode +} + +type SetComputerUsePreviewModeOutput = { + updated: boolean + surface: 'native-overlay' | 'renderer-canvas' | 'none' +} + +type DismissComputerUsePreviewInput = { + sessionId: string + runId: string +} + +type DismissComputerUsePreviewOutput = { + dismissed: boolean +} +``` + +- `computerUse.setPreviewMode`: validate the route sender against the active session binding. +- `computerUse.dismissPreview`: dismiss only an exact current session/run match. + +`eligible` means the active renderer session is working. Native host focus listeners remain +authoritative for actual visibility. `suspended` retains the current valid frame while hiding it. +`stopped` removes presenter state for that session. + +### Fallback Event + +```ts +type ComputerUsePreviewFrame = { + sessionId: string + runId: string + sequence: number + width: number + height: number + mimeType: 'image/jpeg' + data: Uint8Array + timestamp: number +} +``` + +Publish `computerUse.preview.frame` only when the selected surface is `renderer-canvas`. The native +path must not send image bytes to the renderer. + +## Image Pipeline + +### Validation + +- Accept only inline PNG and JPEG image blocks. +- Reject unsupported MIME types, malformed base64, empty data, and invalid dimensions. +- Cap encoded input at 16 MiB before decode. +- Cap decoded width and height at 8192 px. +- Never use image metadata, target window title, or application name as user-visible or logged + content. + +### Transform + +- Preserve aspect ratio within 480 x 300. +- Never upscale the source. +- Encode JPEG at quality 72. +- Enforce a 512 KiB encoded output ceiling. +- Keep one transform in flight for the active target. +- If another valid result arrives during transform, retain only the latest pending result. +- Validate the active epoch after decode and again before presentation. + +The NativeKit surface continues to render with a maximum edge of 360 DIP. The larger encoded frame +retains useful detail on scaled displays without increasing overlay geometry. + +### Memory and Privacy + +- Keep frames in memory only. +- Do not write snapshots to disk. +- Do not add snapshots to settings, telemetry, crash metadata, or logs. +- Do not log base64, pixel buffers, window titles, or full CUA arguments. +- Release superseded buffers and renderer resources promptly. +- Reuse the original MCP response for conversation rendering; PiP must not create a second cached + artifact. +- Discard every non-image field from a private post-click response; do not publish, cache, persist, + or return that response to the Agent. + +## Platform Behavior + +The feature follows the existing NativeKit support matrix: + +| Runtime | Surface | +| --- | --- | +| macOS arm64/x64 | Native overlay | +| Windows x64 | Native overlay | +| Windows arm64 | Renderer Canvas fallback | +| Linux x64/arm64 under X11 or supported XWayland | Native overlay | +| Linux native Wayland | Renderer Canvas fallback | +| Missing/corrupt addon or native startup failure | Renderer Canvas fallback | + +No platform capture permission is added because both Agent-requested and private snapshots use the +existing Computer Use tool. Existing CUA permission behavior is unchanged. + +If NativeKit fails after initialization, hide/remove the native presentation and publish the +retained valid current frame to Canvas fallback without interrupting the Agent. + +## Performance Budget + +- Additional captures: at most one per eligible successful `click`; zero when PiP is not eligible. +- Additional idle polling: zero. +- Idle CPU when no result is being transformed: effectively zero. +- Maximum transforms in flight per active target: one. +- Pending-frame policy: latest wins; no unbounded queue. +- Valid result receipt to visible frame: p95 at or below 150 ms on the reference desktop. +- Synchronous NativeKit `pushImage()` warning threshold: 25 ms, preserving the Browser baseline. +- Encoded output: at most 512 KiB. +- Native path renderer image payloads: zero. + +These are acceptance budgets, not claims that every CUA action produces a snapshot. + +## Failure Behavior + +| Failure | Behavior | +| --- | --- | +| Permission response instead of invocation | Do not claim or show Computer Use PiP | +| Missing run/session identity | Execute tool normally; skip preview | +| Invalid `pid` or `window_id` | Execute tool normally; skip preview | +| Failed `get_window_state` | Retain valid frame for same target; never show error payload | +| Failed private post-click snapshot | Retain valid frame; do not change or delay the click result | +| Unsupported or malformed image | Ignore frame and retain prior valid frame | +| Oversized image | Reject frame and record a rate-limited metadata-only warning | +| Stale/out-of-order result | Drop by claim, epoch, run, target, and tool-call validation | +| Target changes | Hide/remove old presentation; wait for first valid new-target frame | +| Native overlay unavailable | Use Canvas fallback | +| Host loses focus | Hide while retaining current valid frame | +| Session/run stops | Remove presentation and release frame state | +| Presenter shutdown | Unsubscribe listeners, remove image, detach host, stop shared owner once | + +Preview failures never change the MCP tool result or Agent execution outcome. + +## Acceptance Criteria + +### User Behavior + +- A successful current-run `get_window_state` image produces one read-only latest-snapshot PiP when + the active DeepChat chat is foreground and working. +- The PiP is never blank and appears only after its first valid frame. +- Later valid snapshots for the same target replace the visible image. +- An eligible successful `click` schedules a private PiP-only snapshot and may refresh the visible + image without adding screenshot content to the Agent response. +- The Computer Use PiP contains exactly one **Close** button. +- Dragging is native on supported platforms and renderer-local on fallback platforms. +- Close suppresses only the current run and never changes the Agent or target application. +- A later run may show a new PiP after its first valid snapshot. +- Host blur/hide/minimize and route/session switch hide the PiP without leaking another session's + pixels. +- Terminal run/session state removes the PiP. + +### Architecture and Safety + +- Browser and Computer Use share one process-global NativeKit owner. +- Browser keeps **Open in panel** plus **Close** and all existing handoff behavior. +- Computer Use ignores NativeKit activation and exposes no open-panel action. +- Source ownership changes only on explicit tool activity; frame refresh does not steal ownership. +- `runId` reaches `ToolManager` as execution metadata and is not added to MCP arguments. +- CUA identity is determined from resolved plugin ownership metadata. +- No generic MCP pixel event is introduced. +- Native frames do not enter the renderer. +- No idle CUA polling or new capture subsystem is introduced; private capture is bounded to one + asynchronous request per eligible successful `click`. +- Oversize, malformed, stale, and out-of-order images cannot replace a current frame. +- All image bytes stay memory-only and are absent from logs and telemetry. + +### Verification + +- Unit tests cover foreground-scoped claim arbitration, run-scoped dismissal, target changes, stale + epochs, and lifecycle transitions. +- MCP integration tests cover run propagation, trusted CUA recognition, permission paths, valid + result observation, post-click private capture isolation, failure observation, and unchanged tool + responses. +- Image pipeline tests cover PNG/JPEG, malformed base64, unsupported MIME, dimension/input/output + limits, aspect ratio, no upscale, and latest-wins scheduling. +- Renderer tests cover native headless mode, Canvas fallback frame retention, close, focus, + unmount cleanup, and session switching. +- Browser regression tests cover toolbar profile switching, activation, dismissal, capture refresh, + and panel handoff. +- Packaged QA covers the existing NativeKit native/fallback platform matrix. + +## Rollout + +Land behind an internal feature gate if cross-platform packaged QA cannot complete in the same +change. The gate must disable only the Computer Use preview observer/presenter; it must not alter +CUA tool execution or Browser PiP. + +Do not call the feature complete until Browser regression coverage and at least one native plus one +Canvas fallback packaged run pass. Update this status and the linked Browser NativeKit architecture +document after implementation. diff --git a/docs/features/computer-use-snapshot-pip/tasks.md b/docs/features/computer-use-snapshot-pip/tasks.md new file mode 100644 index 000000000..be755ed6b --- /dev/null +++ b/docs/features/computer-use-snapshot-pip/tasks.md @@ -0,0 +1,160 @@ +# Computer Use Snapshot Picture-in-Picture Tasks + +## Status + +Implementation, focused automated verification, and the complete renderer suite are complete. +The complete main suite retains one unrelated provider-metadata expectation failure. Performance +measurement and packaged platform QA are still open. + +## Task List + +- [x] T01 - Propagate Agent run identity through MCP execution + - Add `runId` to internal `McpServicePort`, `McpService`, and `ToolManager` call options. + - Forward `ToolCallOptions.runId` from the MCP branch in `ToolService`. + - Keep the value out of `MCPToolCall` serialization and CUA arguments. + - Add focused propagation and unchanged-arguments tests. + +- [x] T02 - Extract the shared process-global PiP coordinator + - Move NativeKit lifecycle, capability detection, host/display synchronization, and presentation + ownership out of the Browser-only adapter. + - Instantiate one coordinator in the application composition root. + - Preserve the current Browser native and Canvas behavior. + - Keep one visible PiP invariant. + +- [x] T03 - Add source profiles and arbitration + - Keep Browser **Open in panel** plus **Close** controls. + - Add a Computer Use profile with **Close** only and ignored activation. + - Make Browser tool activity and CUA snapshot-call starts claim their source. + - Prevent frame refreshes from stealing ownership. + - Prevent background host/session claims from preempting the foreground session. + - Add shared run-scoped dismissal. + +- [x] T04 - Add the narrow Computer Use MCP observer + - Inject `started`, `completed`, and `failed` callbacks into `ToolManager`. + - Use resolved `ownerPluginId`/`sourceId`, original tool name, and prepared arguments. + - Observe only actual invocations after permission resolution. + - Preserve tool response, abort, progress, and error semantics. + +- [x] T05 - Implement Computer Use target and run state + - Add `ComputerUsePreviewPresenter`. + - Track host, session, run, tool call, `pid`, `windowId`, epoch, claim, and dismissal. + - Remove the prior presentation on a new run or target. + - Reject stale and out-of-order terminal results. + +- [x] T06 - Implement the bounded snapshot pipeline + - Accept successful `get_window_state` PNG/JPEG inline images only. + - Validate base64, input size, decoded dimensions, and output size. + - Resize within 480 x 300 without upscaling and encode JPEG quality 72. + - Keep one transform in flight and one latest pending result. + - Keep frames memory-only and logs pixel-free. + +- [x] T07 - Add typed Computer Use preview contracts + - Add `computerUse.setPreviewMode`. + - Add `computerUse.dismissPreview`. + - Add Canvas-only `computerUse.preview.frame`. + - Expose routes/events through the typed preload client. + - Validate route sender and active session in main. + +- [x] T08 - Add renderer lifecycle coordination + - Mount `AgentComputerUsePiP.vue` beside the Browser PiP controller. + - Derive active session, working state, route, and focus from existing stores/clients. + - Send `eligible`, `suspended`, and `stopped` transitions. + - Keep native mode headless and free of image events. + +- [x] T09 - Add the Canvas fallback UI + - Render the latest valid snapshot with one **Close** button. + - Make non-control points draggable without forwarding target input. + - Retain old pixels until a new frame is fully decoded. + - Reject stale frame sequences and release renderer resources on teardown. + - Reuse `common.close` without adding new copy. + +- [x] T10 - Complete lifecycle and privacy cleanup + - Hide on host blur/hide/minimize and inactive route/session. + - Remove on terminal run/session, target change, host close, and shutdown. + - Ensure another host/session can never see retained pixels. + - Ensure no disk, settings, telemetry, crash metadata, or generic MCP event stores frame bytes. + +- [x] T11 - Add main-process regression coverage + - Test shared coordinator startup/shutdown, toolbar switching, arbitration, and dismissal. + - Test Browser activation, close, capture refresh, and panel handoff after extraction. + - Test CUA recognition, permission paths, run identity, target changes, and stale results. + - Test valid, malformed, unsupported, oversized, failed, and aborted image results. + - Test native/fallback routing and latest-wins scheduling. + - Coordinator, Browser extraction, trust/permission/run propagation, native/fallback, + PNG/JPEG, malformed/unsupported/oversized/dimension validation, failed/aborted calls, target + epochs, dismissal, and latest-wins are covered. + +- [x] T12 - Add renderer regression coverage + - Test native headless mode and absence of frame subscriptions. + - Test Canvas first frame, frame retention, sequence rejection, drag, and close. + - Test focus, minimize, session switch, terminal state, and unmount cleanup. + - Verify Computer Use exposes no expand/open-panel behavior. + - Native headless mode, first Canvas frame, frame retention, target epoch/sequence rejection, + close-only controls, focus suspend/resume, terminal cleanup, drag, session switch, unmount, and + Browser supersession are covered. + +- [x] T13 - Add PiP-only post-click refresh + - After an eligible successful exact CUA `click`, asynchronously invoke one private + `get_window_state({ pid, window_id })`. + - Gate the call on trusted plugin ownership, valid session/run/target identity, active PiP, and + non-dismissed presenter state. + - Route the private result only through the preview observer; never publish, cache, or return it + to the Agent. + - Preserve click response, latency, permission, abort, and failure behavior. + - Cover successful isolation plus failed, untrusted, invalid-target, and inactive-preview paths. + +- [ ] T14 - Verify performance budgets + - Confirm at most one private capture per eligible successful `click` and zero idle polling. + - Confirm one transform in flight with no unbounded queue. + - Measure p95 valid-result-to-visible latency against the 150 ms budget. + - Keep NativeKit slow-push warning behavior at 25 ms. + - Confirm native mode sends zero image bytes to the renderer. + +- [ ] T15 - Run packaged platform QA + - Verify one NativeKit native-overlay runtime. + - Verify one renderer Canvas-fallback runtime. + - Exercise host focus/minimize/restore and target-app foreground behavior. + - Exercise Browser/Computer source transitions and toolbar profile changes. + - Exercise close, later run, route/session switch, and app shutdown. + +- [ ] T16 - Run repository verification + - Run `pnpm run format`. + - Run `pnpm run i18n`. + - Run `pnpm run lint`. + - Run `pnpm run typecheck`. + - Run focused main and renderer tests, then the complete affected suites. + - Partial: format, i18n, lint, typecheck, focused suites, and all 1,625 renderer tests pass. The + main suite passes 5,383 tests and fails only + `ModelConfigHelper > Configuration Priority > uses provider metadata until a user config + overrides it`; the unchanged provider test also fails in isolation. + +- [ ] T17 - Close implementation documentation + - Update the proposal status and record deviations. + - Update the Browser NativeKit architecture with the implemented shared-owner contract. + - Mark only verified tasks complete. + - Include BEFORE/AFTER ASCII layouts and platform QA status in the PR. + - Partial: implementation status, deviations, linked Browser architecture, verified tasks, ASCII + layouts, and open QA are recorded. PR handoff remains open. + +## Implementation Order + +1. T01-T03 establish identity and safe process-global ownership. +2. T04-T06 connect trusted CUA results to a bounded main-process snapshot pipeline. +3. T07-T09 add typed renderer coordination and fallback UI. +4. T10-T12 harden lifecycle, privacy, and regressions. +5. T13 adds event-driven post-click refresh. +6. T14-T17 measure, validate, and close documentation. + +## Done Definition + +- A current successful CUA `get_window_state` image can show one latest-snapshot PiP, and an + eligible successful `click` can schedule one private PiP-only refresh without idle polling. +- Browser and Computer Use safely share one NativeKit owner and retain source-specific controls. +- Computer Use PiP has only **Close**, is read-only, and never affects the Agent or target + application. +- Run, target, session, host, and epoch validation prevent stale pixel disclosure. +- Native delivery avoids renderer image traffic; unsupported runtimes use the bounded Canvas + fallback. +- Image work is memory-only, bounded, latest-wins, and covered by focused tests. +- Browser PiP behavior remains regression-tested. +- Required checks and at least one native plus one fallback packaged QA run pass. diff --git a/src/main/app/composition.ts b/src/main/app/composition.ts index 5b1cd8210..03e3e0d8c 100644 --- a/src/main/app/composition.ts +++ b/src/main/app/composition.ts @@ -57,6 +57,8 @@ import { TrayPresenter } from '../desktop/tray' import { OAuthService } from '../provider/auth' import { FloatingButtonPresenter } from '../desktop/floatingButton' import { YoBrowserPresenter } from '../desktop/browser/YoBrowserPresenter' +import { ComputerUsePreviewPresenter } from '@/desktop/computerUse/ComputerUsePreviewPresenter' +import { AgentPreviewCoordinator } from '@/desktop/preview/AgentPreviewCoordinator' import { KnowledgeService } from '../knowledge' import { WorkspaceService } from '../workspace' import { FileWatcherService } from '../platform/fileWatcher' @@ -262,6 +264,8 @@ export async function createMainProcessControl(dependencies: { let toolService: ToolServicePort let deepChatAgentHarness: DeepChatAgentHarness let yoBrowserPresenter: IYoBrowserPresenter + let computerUsePreviewPresenter: ComputerUsePreviewPresenter + let agentPreviewCoordinator: AgentPreviewCoordinator let dialogService: DialogServicePort let skillService: SkillServicePort let skillSyncService: SkillSyncServicePort @@ -610,7 +614,16 @@ export async function createMainProcessControl(dependencies: { notificationService = new NotificationService(desktopSettings, publishDeepchatEvent) trayPresenter = new TrayPresenter(desktopSettings, windowPresenter) dialogService = new DialogService(publishDeepchatEvent) - yoBrowserPresenter = new YoBrowserPresenter(windowPresenter, publishDeepchatEvent) + agentPreviewCoordinator = new AgentPreviewCoordinator() + yoBrowserPresenter = new YoBrowserPresenter( + windowPresenter, + publishDeepchatEvent, + agentPreviewCoordinator + ) + computerUsePreviewPresenter = new ComputerUsePreviewPresenter( + windowPresenter, + agentPreviewCoordinator + ) // Define the storage root for built-in knowledge databases. const dbDir = path.join(app.getPath('userData'), 'app_db') @@ -651,7 +664,8 @@ export async function createMainProcessControl(dependencies: { providerRuntime, () => deepChatAgentHarness.refreshToolRegistry(), publishDeepchatEvent, - (data) => deviceService.cacheImage(data) + (data) => deviceService.cacheImage(data), + computerUsePreviewPresenter ) const deeplinkActions = createDeeplinkActions({ window: windowPresenter, @@ -1631,7 +1645,13 @@ export async function createMainProcessControl(dependencies: { } await runDestroyStep('pluginService.shutdown', () => pluginService.shutdown()) await runDestroyStep('mcpService.shutdown', () => mcpService.shutdown()) + await runDestroyStep('computerUsePreviewPresenter.shutdown', () => + computerUsePreviewPresenter.shutdown() + ) await runDestroyStep('yoBrowserPresenter.shutdown', () => yoBrowserPresenter.shutdown()) + await runDestroyStep('agentPreviewCoordinator.shutdown', () => + agentPreviewCoordinator.shutdown() + ) await runDestroyStep('floatingButtonPresenter.destroy', () => floatingButtonPresenter.destroy()) await runDestroyStep('windowPresenter.destroyFloatingChatWindow', () => windowPresenter.destroyFloatingChatWindow() @@ -1743,6 +1763,8 @@ export async function createMainProcessControl(dependencies: { windowPresenter, shortcutPresenter, browserPresenter: yoBrowserPresenter, + computerUsePreviewPresenter, + desktopSessionBinding, tabPresenter, dialogService, settings: desktopSettings, diff --git a/src/main/desktop/browser/YoBrowserPresenter.ts b/src/main/desktop/browser/YoBrowserPresenter.ts index e81eb3de2..27b3c31ba 100644 --- a/src/main/desktop/browser/YoBrowserPresenter.ts +++ b/src/main/desktop/browser/YoBrowserPresenter.ts @@ -23,10 +23,10 @@ import { import type { DownloadInfo } from '@shared/types/browser' import type { IWindowPresenter, IYoBrowserPresenter } from '@shared/types/desktop' import { - AgentBrowserNativeOverlay, - type AgentBrowserNativeOverlayAction, - type AgentBrowserNativeOverlayTarget -} from './AgentBrowserNativeOverlay' + AgentPreviewCoordinator, + type AgentPreviewAction, + type AgentPreviewTarget +} from '@/desktop/preview/AgentPreviewCoordinator' import { BrowserTab as BrowserPage } from './BrowserTab' import { BrowserProfileImportService } from './BrowserProfileImportService' import { CDPManager } from './CDPManager' @@ -69,6 +69,7 @@ type SessionBrowserState = { previewTimer: ReturnType | null previewCapture: Promise | null previewEpoch: number + previewClaimSequence: number previewSequence: number previewBurstUntil: number } @@ -101,22 +102,25 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { ) private browserDataMutationActive = false private readonly windowPresenter: IWindowPresenter - private readonly nativeOverlay: AgentBrowserNativeOverlay + private readonly previewCoordinator: AgentPreviewCoordinator + private readonly unregisterPreviewHandler: () => void readonly toolHandler: YoBrowserToolHandler constructor( windowPresenter: IWindowPresenter, - private readonly publishEvent: DeepchatEventPublisher + private readonly publishEvent: DeepchatEventPublisher, + previewCoordinator: AgentPreviewCoordinator ) { this.windowPresenter = windowPresenter - this.nativeOverlay = new AgentBrowserNativeOverlay((action, target) => { + this.previewCoordinator = previewCoordinator + this.unregisterPreviewHandler = previewCoordinator.register('browser', (action, target) => { this.handleNativePreviewAction(action, target) }) this.toolHandler = new YoBrowserToolHandler(this) } async initialize(): Promise { - await this.nativeOverlay.initialize() + await this.previewCoordinator.initialize() } async getBrowserStatus(sessionId: string): Promise { @@ -192,7 +196,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return false } - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) this.detachOtherSessionBrowsers(hostWindowId, sessionId) @@ -284,9 +288,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { if (runId != null && state.agentRunId != null && runId !== state.agentRunId) { return { updated: false, surface: 'none' } } - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) await this.releasePreviewHost(state) - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.previewSurface = 'none' return { updated: true, surface: 'none' } } @@ -299,7 +303,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return { updated: false, surface: 'none' } } - await this.nativeOverlay.initialize() + await this.previewCoordinator.initialize() const targetWindow = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) if (hostWindowId != null && (!targetWindow || targetWindow.isDestroyed())) { @@ -307,7 +311,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } if (mode === 'rendering') { - this.nativeOverlay.hide(this.nativeTargetRef(state)) + this.previewCoordinator.hide(this.nativeTargetRef(state)) } await this.stopPreviewCapture(state) if (mode === 'capturing' && (!targetWindow || state.visible)) { @@ -322,34 +326,49 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewEpoch += 1 if (mode === 'rendering') { - this.nativeOverlay.hide(this.nativeTargetRef(state)) - state.previewSurface = this.nativeOverlay.isAvailable() ? 'native-overlay' : 'renderer-canvas' + this.previewCoordinator.hide(this.nativeTargetRef(state)) + state.previewSurface = this.previewCoordinator.isAvailable() + ? 'native-overlay' + : 'renderer-canvas' return { updated: true, surface: state.previewSurface } } if (!targetWindow) { return { updated: false, surface: 'none' } } - const nativeTarget = this.nativeTarget(state) - state.previewSurface = - nativeTarget && this.nativeOverlay.prepare(nativeTarget, targetWindow) - ? 'native-overlay' - : 'renderer-canvas' - - this.schedulePreviewCapture(state, 0, state.previewEpoch) + this.resumeClaimedPreview(state) return { updated: true, surface: state.previewSurface } } + dismissPreview(sessionId: string, runId: string): boolean { + const state = this.sessionBrowsers.get(sessionId) + if (!state || state.agentRunId !== runId) { + return false + } + + const dismissed = this.previewCoordinator.dismiss({ + source: 'browser', + sessionId, + runId + }) + if (dismissed) { + state.previewSurface = 'none' + void this.stopPreviewCapture(state) + this.emitPreviewSurface(state) + } + return dismissed + } + async destroySessionBrowser(sessionId: string): Promise { const state = this.sessionBrowsers.get(sessionId) if (!state) { return } - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) await this.releasePreviewHost(state) await this.detachSessionBrowser(sessionId) state.page.destroy() @@ -517,7 +536,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { for (const sessionId of Array.from(this.sessionBrowsers.keys())) { await this.destroySessionBrowser(sessionId) } - this.nativeOverlay.shutdown() + this.unregisterPreviewHandler() } private ensureSessionBrowserState(sessionId: string): SessionBrowserState { @@ -557,6 +576,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { previewTimer: null, previewCapture: null, previewEpoch: 0, + previewClaimSequence: 0, previewSequence: 0, previewBurstUntil: 0 } @@ -675,7 +695,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { return } - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) this.disposePreviewHost(state) state.page.destroy() state.overlay.destroy() @@ -964,13 +984,31 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { agentRunId?: string ): void { if (activitySource === 'agent') { + const previousRunId = state.agentRunId + const nextRunId = agentRunId?.trim() || previousRunId || nanoid(12) + if (previousRunId && previousRunId !== nextRunId) { + this.previewCoordinator.removeTarget({ + source: 'browser', + sessionId: state.sessionId, + runId: previousRunId + }) + state.previewEpoch += 1 + } state.owner = 'agent' - state.agentRunId = agentRunId?.trim() || state.agentRunId || nanoid(12) + state.agentRunId = nextRunId + state.previewClaimSequence = this.previewCoordinator.claim({ + source: 'browser', + sessionId: state.sessionId, + runId: nextRunId + }) + this.resumeClaimedPreview(state) return } + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.owner = 'user' state.agentRunId = undefined + state.previewClaimSequence = 0 } private emitWindowUpdated(sessionId: string): void { @@ -1320,7 +1358,7 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { state.previewHost = host host.once('closed', () => { if (state.previewHost === host) { - this.nativeOverlay.removeTarget(this.nativeTargetRef(state)) + this.previewCoordinator.releaseClaim(this.nativeTargetRef(state)) state.previewHost = null state.previewMode = 'stopped' state.previewSurface = 'none' @@ -1398,6 +1436,44 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { host.destroy() } + private resumeClaimedPreview(state: SessionBrowserState): void { + if (state.previewMode !== 'capturing' || state.previewTargetWindowId == null) { + return + } + const target = this.nativeTarget(state) + const host = BrowserWindow.fromId(state.previewTargetWindowId) + if (!target || !host || host.isDestroyed() || state.visible) { + return + } + + state.previewSurface = this.previewCoordinator.prepare(target, host) + this.emitPreviewSurface(state) + if ( + state.previewSurface !== 'none' && + state.previewTimer == null && + state.previewCapture == null + ) { + this.schedulePreviewCapture(state, 0, state.previewEpoch) + } + } + + private emitPreviewSurface(state: SessionBrowserState): void { + if (state.previewTargetWindowId == null || !state.agentRunId) { + return + } + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.surface.changed', { + windowId: state.previewTargetWindowId, + sessionId: state.sessionId, + runId: state.agentRunId, + surface: state.previewSurface, + version: Date.now() + }) + ) + } + private schedulePreviewCapture(state: SessionBrowserState, delayMs: number, epoch: number): void { if (state.previewMode !== 'capturing' || state.previewEpoch !== epoch) { return @@ -1409,6 +1485,9 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { void capture.finally(() => { if (state.previewCapture === capture) { state.previewCapture = null + if (state.previewTimer == null && state.previewMode === 'capturing') { + this.resumeClaimedPreview(state) + } } }) }, delayMs) @@ -1416,11 +1495,14 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } private async capturePreviewFrame(state: SessionBrowserState, epoch: number): Promise { + const initialTarget = this.nativeTarget(state) if ( state.previewMode !== 'capturing' || state.previewEpoch !== epoch || state.previewTargetWindowId == null || !state.agentRunId || + !initialTarget || + !this.previewCoordinator.isCurrent(initialTarget) || state.page.contents.isDestroyed() ) { return @@ -1438,24 +1520,15 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { if (data.byteLength <= PREVIEW_MAX_BYTES) { if (state.previewSurface === 'native-overlay') { const target = this.nativeTarget(state) - if (target && target.captureEpoch === epoch) { - this.nativeOverlay.present(target, data) + if (target && target.epoch === epoch && !this.previewCoordinator.present(target, data)) { + const host = BrowserWindow.fromId(target.windowId) + state.previewSurface = + host && !host.isDestroyed() ? this.previewCoordinator.prepare(target, host) : 'none' + this.emitPreviewSurface(state) } - } else if (state.previewSurface === 'renderer-canvas') { - state.previewSequence += 1 - this.windowPresenter.sendToWindow( - state.previewTargetWindowId, - DEEPCHAT_EVENT_CHANNEL, - createDeepchatEventEnvelope('browser.preview.frame', { - sessionId: state.sessionId, - runId: state.agentRunId, - sequence: state.previewSequence, - ...PREVIEW_FRAME, - mimeType: 'image/jpeg', - data, - timestamp: Date.now() - }) - ) + } + if (state.previewSurface === 'renderer-canvas') { + this.sendRendererPreviewFrame(state, data) } } } catch (error) { @@ -1466,6 +1539,10 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { } const active = state.page.contents.isLoading() || Date.now() < state.previewBurstUntil + const nextTarget = this.nativeTarget(state) + if (!nextTarget || !this.previewCoordinator.isCurrent(nextTarget)) { + return + } this.schedulePreviewCapture( state, active ? PREVIEW_ACTIVE_INTERVAL_MS : PREVIEW_IDLE_INTERVAL_MS, @@ -1473,45 +1550,78 @@ export class YoBrowserPresenter implements IYoBrowserPresenter { ) } - private nativeTarget(state: SessionBrowserState): AgentBrowserNativeOverlayTarget | null { - if (state.previewTargetWindowId == null || !state.agentRunId) { + private nativeTarget(state: SessionBrowserState): AgentPreviewTarget | null { + if ( + state.previewTargetWindowId == null || + !state.agentRunId || + state.previewClaimSequence <= 0 + ) { return null } return { + source: 'browser', windowId: state.previewTargetWindowId, sessionId: state.sessionId, runId: state.agentRunId, - captureEpoch: state.previewEpoch + epoch: state.previewEpoch, + claimSequence: state.previewClaimSequence } } + private sendRendererPreviewFrame(state: SessionBrowserState, data: Buffer): void { + if (state.previewTargetWindowId == null || !state.agentRunId) { + return + } + state.previewSequence += 1 + this.windowPresenter.sendToWindow( + state.previewTargetWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('browser.preview.frame', { + sessionId: state.sessionId, + runId: state.agentRunId, + sequence: state.previewSequence, + ...PREVIEW_FRAME, + mimeType: 'image/jpeg', + data, + timestamp: Date.now() + }) + ) + } + private nativeTargetRef(state: SessionBrowserState): { + source: 'browser' sessionId: string runId?: string } { return { + source: 'browser', sessionId: state.sessionId, ...(state.agentRunId ? { runId: state.agentRunId } : {}) } } - private handleNativePreviewAction( - action: AgentBrowserNativeOverlayAction, - target: AgentBrowserNativeOverlayTarget - ): void { + private handleNativePreviewAction(action: AgentPreviewAction, target: AgentPreviewTarget): void { const state = this.sessionBrowsers.get(target.sessionId) if ( !state || - state.previewSurface !== 'native-overlay' || + target.source !== 'browser' || state.previewTargetWindowId !== target.windowId || state.agentRunId !== target.runId || - state.previewEpoch !== target.captureEpoch + state.previewEpoch !== target.epoch || + state.previewClaimSequence !== target.claimSequence ) { return } - state.previewMode = 'rendering' + state.previewSurface = 'none' void this.stopPreviewCapture(state) + this.emitPreviewSurface(state) + + if (action === 'superseded') { + return + } + + state.previewMode = 'rendering' if (action === 'activate') { this.windowPresenter.show(target.windowId, true) diff --git a/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts new file mode 100644 index 000000000..23a5d7cf7 --- /dev/null +++ b/src/main/desktop/computerUse/ComputerUsePreviewPresenter.ts @@ -0,0 +1,632 @@ +import { BrowserWindow } from 'electron' +import { performance } from 'node:perf_hooks' +import sharp from 'sharp' +import { DEEPCHAT_EVENT_CHANNEL } from '@shared/contracts/channels' +import { createDeepchatEventEnvelope } from '@shared/contracts/events' +import logger from '@shared/logger' +import type { + ComputerUsePreviewMode, + ComputerUsePreviewModeResult, + ComputerUsePreviewSurface +} from '@shared/types/computerUse' +import type { IComputerUsePreviewPresenter, IWindowPresenter } from '@shared/types/desktop' +import type { MCPImageContent, MCPToolResponse } from '@shared/types/mcp' +import type { ComputerUsePreviewCall, ComputerUsePreviewObserver } from '@/mcp/toolManager' +import { + AgentPreviewCoordinator, + type AgentPreviewAction, + type AgentPreviewTarget +} from '@/desktop/preview/AgentPreviewCoordinator' + +type ComputerUseFrame = { + data: Buffer + width: number + height: number + sequence: number + timestamp: number +} + +type PendingSnapshot = { + call: ComputerUsePreviewCall + image: MCPImageContent + epoch: number + claimSequence: number +} + +type ComputerUsePreviewState = { + sessionId: string + mode: ComputerUsePreviewMode + hostWindowId: number | null + hostClosedListener: (() => void) | null + runId: string | null + toolCallId: string | null + toolCallEpoch: number + pid: number | null + targetWindowId: number | null + epoch: number + claimSequence: number + dismissedRunId: string | null + surface: ComputerUsePreviewSurface + frame: ComputerUseFrame | null + nextFrameSequence: number + transformActive: boolean + pendingSnapshot: PendingSnapshot | null +} + +const FRAME_MAX_WIDTH = 480 +const FRAME_MAX_HEIGHT = 300 +const FRAME_MAX_BYTES = 512 * 1024 +const INPUT_MAX_BYTES = 16 * 1024 * 1024 +const INPUT_MAX_DIMENSION = 8192 +const INPUT_MAX_PIXELS = INPUT_MAX_DIMENSION * INPUT_MAX_DIMENSION +const JPEG_QUALITY = 72 +const SLOW_TRANSFORM_WARNING_MS = 150 +const WARNING_INTERVAL_MS = 60_000 +const SUPPORTED_MIME_TYPES = new Set(['image/jpeg', 'image/png']) + +export class ComputerUsePreviewPresenter + implements IComputerUsePreviewPresenter, ComputerUsePreviewObserver +{ + private readonly states = new Map() + private readonly unregisterPreviewHandler: () => void + private lastValidationWarningAt = 0 + private lastSlowTransformWarningAt = 0 + + constructor( + private readonly windowPresenter: IWindowPresenter, + private readonly previewCoordinator: AgentPreviewCoordinator + ) { + this.unregisterPreviewHandler = previewCoordinator.register('computer-use', (action, target) => + this.handlePreviewAction(action, target) + ) + } + + shouldCaptureAfterClick(call: ComputerUsePreviewCall): boolean { + const state = this.states.get(call.conversationId) + if (state?.hostWindowId != null) { + const host = BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + this.stopState(state) + return false + } + } + const pid = this.readPositiveInteger(call.args.pid) + const targetWindowId = this.readPositiveInteger(call.args.window_id) + return Boolean( + call.toolName === 'click' && + state && + pid != null && + targetWindowId != null && + state.mode === 'eligible' && + state.hostWindowId != null && + state.runId === call.runId && + state.pid === pid && + state.targetWindowId === targetWindowId && + state.dismissedRunId !== call.runId + ) + } + + started(call: ComputerUsePreviewCall): void { + if (call.toolName !== 'get_window_state') { + return + } + const pid = this.readPositiveInteger(call.args.pid) + const targetWindowId = this.readPositiveInteger(call.args.window_id) + if (pid == null || targetWindowId == null) { + return + } + + const state = this.states.get(call.conversationId) ?? this.createState(call.conversationId) + const targetChanged = + state.runId !== call.runId || state.pid !== pid || state.targetWindowId !== targetWindowId + if (targetChanged) { + if (state.runId) { + this.previewCoordinator.removeTarget({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + state.surface = 'none' + if (state.runId !== call.runId) { + state.dismissedRunId = null + } + } + + state.runId = call.runId + state.toolCallId = call.toolCallId + state.toolCallEpoch = state.epoch + state.pid = pid + state.targetWindowId = targetWindowId + state.claimSequence = this.previewCoordinator.claim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: call.runId + }) + + if (state.mode === 'eligible') { + if (targetChanged || state.surface === 'none') { + this.presentCurrent(state) + } else { + this.prepareCurrent(state) + } + } + } + + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void { + const state = this.states.get(call.conversationId) + if ( + !state || + result.isError === true || + state.dismissedRunId === call.runId || + state.runId !== call.runId || + state.toolCallId !== call.toolCallId || + state.toolCallEpoch !== state.epoch + ) { + return + } + + const image = this.readInlineImage(result) + if (!image) { + return + } + this.enqueueSnapshot(state, { + call, + image, + epoch: state.epoch, + claimSequence: state.claimSequence + }) + } + + failed(_call: ComputerUsePreviewCall, _error: unknown): void { + // Keep the last valid frame for the same run and target. + } + + async setPreviewMode( + sessionId: string, + mode: ComputerUsePreviewMode, + hostWindowId?: number + ): Promise { + const normalizedSessionId = sessionId.trim() + if (!normalizedSessionId) { + return { updated: false, surface: 'none' } + } + + if (mode === 'stopped') { + const state = this.states.get(normalizedSessionId) + if ( + !state || + (hostWindowId != null && state.hostWindowId != null && state.hostWindowId !== hostWindowId) + ) { + return { updated: false, surface: 'none' } + } + this.stopState(state) + return { updated: true, surface: 'none' } + } + + const host = hostWindowId == null ? null : BrowserWindow.fromId(hostWindowId) + if (!host || host.isDestroyed()) { + const state = this.states.get(normalizedSessionId) + if (state && (state.hostWindowId == null || state.hostWindowId === hostWindowId)) { + this.stopState(state) + } + return { updated: false, surface: 'none' } + } + + this.stopOtherSessionsForHost(normalizedSessionId, host.id) + await this.previewCoordinator.initialize() + if (host.isDestroyed()) { + const state = this.states.get(normalizedSessionId) + if (state && (state.hostWindowId == null || state.hostWindowId === host.id)) { + this.stopState(state) + } + return { updated: false, surface: 'none' } + } + const state = this.states.get(normalizedSessionId) ?? this.createState(normalizedSessionId) + this.bindHost(state, host) + state.mode = mode + + if (mode === 'suspended') { + this.previewCoordinator.hide({ + source: 'computer-use', + sessionId: normalizedSessionId, + ...(state.runId ? { runId: state.runId } : {}) + }) + this.setSurface(state, 'none') + return { updated: true, surface: 'none' } + } + + const surface = this.presentCurrent(state, true) + return { updated: true, surface } + } + + dismissPreview(sessionId: string, runId: string): boolean { + const state = this.states.get(sessionId) + if (!state || state.runId !== runId) { + return false + } + + const dismissed = this.previewCoordinator.dismiss({ + source: 'computer-use', + sessionId, + runId + }) + if (dismissed) { + state.dismissedRunId = runId + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + this.setSurface(state, 'none') + } + return dismissed + } + + shutdown(): void { + for (const state of this.states.values()) { + this.stopState(state) + } + this.unregisterPreviewHandler() + } + + private createState(sessionId: string): ComputerUsePreviewState { + const state: ComputerUsePreviewState = { + sessionId, + mode: 'stopped', + hostWindowId: null, + hostClosedListener: null, + runId: null, + toolCallId: null, + toolCallEpoch: 0, + pid: null, + targetWindowId: null, + epoch: 0, + claimSequence: 0, + dismissedRunId: null, + surface: 'none', + frame: null, + nextFrameSequence: 0, + transformActive: false, + pendingSnapshot: null + } + this.states.set(sessionId, state) + return state + } + + private bindHost(state: ComputerUsePreviewState, host: BrowserWindow): void { + if (state.hostWindowId === host.id && state.hostClosedListener) { + return + } + this.unbindHost(state) + const closed = () => { + if (state.hostClosedListener !== closed) { + return + } + state.hostClosedListener = null + this.stopState(state) + } + state.hostWindowId = host.id + state.hostClosedListener = closed + host.once('closed', closed) + } + + private unbindHost(state: ComputerUsePreviewState): void { + const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) + if (host && state.hostClosedListener) { + host.removeListener('closed', state.hostClosedListener) + } + state.hostWindowId = null + state.hostClosedListener = null + } + + private stopState(state: ComputerUsePreviewState): void { + if (this.states.get(state.sessionId) !== state) { + return + } + state.epoch += 1 + state.mode = 'stopped' + state.frame = null + state.pendingSnapshot = null + if (state.runId) { + this.previewCoordinator.releaseClaim({ + source: 'computer-use', + sessionId: state.sessionId, + runId: state.runId + }) + } + this.unbindHost(state) + this.states.delete(state.sessionId) + } + + private readPositiveInteger(value: unknown): number | null { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null + } + + private readInlineImage(result: MCPToolResponse): MCPImageContent | null { + if (!Array.isArray(result.content)) { + return null + } + for (const item of result.content) { + if ( + item.type === 'image' && + typeof item.data === 'string' && + typeof item.mimeType === 'string' && + SUPPORTED_MIME_TYPES.has(item.mimeType.toLowerCase()) + ) { + return item + } + } + return null + } + + private enqueueSnapshot(state: ComputerUsePreviewState, snapshot: PendingSnapshot): void { + if (state.transformActive) { + state.pendingSnapshot = snapshot + return + } + state.transformActive = true + void this.transformSnapshot(state, snapshot).finally(() => { + const current = this.states.get(snapshot.call.conversationId) + if (current !== state) { + return + } + state.transformActive = false + const pending = state.pendingSnapshot + state.pendingSnapshot = null + if (pending) { + this.enqueueSnapshot(state, pending) + } + }) + } + + private async transformSnapshot( + owner: ComputerUsePreviewState, + snapshot: PendingSnapshot + ): Promise { + const startedAt = performance.now() + try { + const source = this.decodeBase64(snapshot.image.data) + const image = sharp(source, { + failOn: 'warning', + limitInputPixels: INPUT_MAX_PIXELS + }) + const metadata = await image.metadata() + const mimeType = snapshot.image.mimeType.toLowerCase() + const expectedFormat = mimeType === 'image/png' ? 'png' : 'jpeg' + if ( + metadata.format !== expectedFormat || + !metadata.width || + !metadata.height || + metadata.width > INPUT_MAX_DIMENSION || + metadata.height > INPUT_MAX_DIMENSION + ) { + throw new Error('unsupported image metadata') + } + + const transformed = await image + .resize({ + width: FRAME_MAX_WIDTH, + height: FRAME_MAX_HEIGHT, + fit: 'inside', + withoutEnlargement: true + }) + .jpeg({ quality: JPEG_QUALITY }) + .toBuffer({ resolveWithObject: true }) + if ( + transformed.data.byteLength === 0 || + transformed.data.byteLength > FRAME_MAX_BYTES || + transformed.info.width <= 0 || + transformed.info.height <= 0 + ) { + throw new Error('invalid transformed image') + } + + const state = this.states.get(snapshot.call.conversationId) + if ( + state !== owner || + state.runId !== snapshot.call.runId || + state.toolCallId !== snapshot.call.toolCallId || + state.epoch !== snapshot.epoch || + state.claimSequence !== snapshot.claimSequence || + state.dismissedRunId === snapshot.call.runId + ) { + return + } + + state.nextFrameSequence += 1 + state.frame = { + data: transformed.data, + width: transformed.info.width, + height: transformed.info.height, + sequence: state.nextFrameSequence, + timestamp: Date.now() + } + if (state.mode === 'eligible') { + this.presentCurrent(state) + } + } catch (error) { + this.warnValidation(snapshot.call, error) + } finally { + const durationMs = performance.now() - startedAt + if ( + durationMs > SLOW_TRANSFORM_WARNING_MS && + Date.now() - this.lastSlowTransformWarningAt >= WARNING_INTERVAL_MS + ) { + this.lastSlowTransformWarningAt = Date.now() + logger.warn('[ComputerUsePreview] Slow snapshot transform', { + durationMs: Math.round(durationMs) + }) + } + } + } + + private decodeBase64(data: string): Buffer { + const normalized = data.trim() + const maxBase64Length = Math.ceil((INPUT_MAX_BYTES * 4) / 3) + 4 + if ( + !normalized || + normalized.length > maxBase64Length || + normalized.length % 4 === 1 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(normalized) + ) { + throw new Error('invalid base64 image') + } + const buffer = Buffer.from(normalized, 'base64') + if (buffer.byteLength === 0 || buffer.byteLength > INPUT_MAX_BYTES) { + throw new Error('image input exceeds limit') + } + return buffer + } + + private prepareCurrent( + state: ComputerUsePreviewState, + announceSurface = false + ): ComputerUsePreviewSurface { + const target = this.previewTarget(state) + if (!target || state.mode !== 'eligible') { + return this.setSurface(state, 'none', announceSurface) + } + const host = state.hostWindowId == null ? null : BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + this.stopState(state) + return 'none' + } + return this.setSurface(state, this.previewCoordinator.prepare(target, host), announceSurface) + } + + private presentCurrent( + state: ComputerUsePreviewState, + announceSurface = false + ): ComputerUsePreviewSurface { + const surface = this.prepareCurrent(state, announceSurface) + const frame = state.frame + const target = this.previewTarget(state) + if (surface === 'none' || !frame || !target || state.hostWindowId == null) { + return surface + } + + if (surface === 'native-overlay') { + if (this.previewCoordinator.present(target, frame.data)) { + return surface + } + const host = BrowserWindow.fromId(state.hostWindowId) + if (!host || host.isDestroyed()) { + return this.setSurface(state, 'none') + } + const fallbackSurface = this.previewCoordinator.prepare(target, host) + this.setSurface(state, fallbackSurface) + if (fallbackSurface !== 'renderer-canvas') { + return fallbackSurface + } + } + + this.windowPresenter.sendToWindow( + state.hostWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('computerUse.preview.frame', { + sessionId: state.sessionId, + runId: target.runId, + epoch: state.epoch, + sequence: frame.sequence, + width: frame.width, + height: frame.height, + mimeType: 'image/jpeg', + data: frame.data, + timestamp: frame.timestamp + }) + ) + return state.surface + } + + private previewTarget(state: ComputerUsePreviewState): AgentPreviewTarget | null { + if ( + state.hostWindowId == null || + !state.runId || + state.pid == null || + state.targetWindowId == null || + state.claimSequence <= 0 || + state.dismissedRunId === state.runId + ) { + return null + } + return { + source: 'computer-use', + windowId: state.hostWindowId, + sessionId: state.sessionId, + runId: state.runId, + epoch: state.epoch, + claimSequence: state.claimSequence + } + } + + private setSurface( + state: ComputerUsePreviewState, + surface: ComputerUsePreviewSurface, + forceEvent = false + ): ComputerUsePreviewSurface { + if (state.surface === surface && !forceEvent) { + return surface + } + state.surface = surface + if (state.hostWindowId != null && state.runId) { + this.windowPresenter.sendToWindow( + state.hostWindowId, + DEEPCHAT_EVENT_CHANNEL, + createDeepchatEventEnvelope('computerUse.preview.surface.changed', { + windowId: state.hostWindowId, + sessionId: state.sessionId, + runId: state.runId, + epoch: state.epoch, + surface, + version: Date.now() + }) + ) + } + return surface + } + + private stopOtherSessionsForHost(sessionId: string, hostWindowId: number): void { + for (const [otherSessionId, state] of this.states) { + if (otherSessionId === sessionId || state.hostWindowId !== hostWindowId) { + continue + } + this.stopState(state) + } + } + + private handlePreviewAction(action: AgentPreviewAction, target: AgentPreviewTarget): void { + if (target.source !== 'computer-use') { + return + } + const state = this.states.get(target.sessionId) + if ( + !state || + state.runId !== target.runId || + state.epoch !== target.epoch || + state.claimSequence !== target.claimSequence + ) { + return + } + + if (action === 'dismiss') { + state.dismissedRunId = target.runId + state.epoch += 1 + state.frame = null + state.pendingSnapshot = null + } + this.setSurface(state, 'none') + } + + private warnValidation(call: ComputerUsePreviewCall, error: unknown): void { + if (Date.now() - this.lastValidationWarningAt < WARNING_INTERVAL_MS) { + return + } + this.lastValidationWarningAt = Date.now() + logger.warn('[ComputerUsePreview] Snapshot rejected', { + toolCallId: call.toolCallId, + reason: error instanceof Error ? error.message : String(error) + }) + } +} diff --git a/src/main/desktop/browser/AgentBrowserNativeOverlay.ts b/src/main/desktop/preview/AgentPreviewCoordinator.ts similarity index 62% rename from src/main/desktop/browser/AgentBrowserNativeOverlay.ts rename to src/main/desktop/preview/AgentPreviewCoordinator.ts index 1e85ad1aa..ab42c8824 100644 --- a/src/main/desktop/browser/AgentBrowserNativeOverlay.ts +++ b/src/main/desktop/preview/AgentPreviewCoordinator.ts @@ -4,25 +4,35 @@ import logger from '@shared/logger' type NativeKitOverlay = (typeof import('@zerob13/nativekit'))['overlay'] -export type AgentBrowserNativeOverlayTarget = { +export type AgentPreviewSource = 'browser' | 'computer-use' + +export type AgentPreviewTarget = { + source: AgentPreviewSource windowId: number sessionId: string runId: string - captureEpoch: number + epoch: number + claimSequence: number } -export type AgentBrowserNativeOverlayAction = 'activate' | 'dismiss' +export type AgentPreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' -type NativeOverlayActionHandler = ( - action: AgentBrowserNativeOverlayAction, - target: AgentBrowserNativeOverlayTarget -) => void +export type AgentPreviewAction = 'activate' | 'dismiss' | 'superseded' -type NativeOverlayTargetRef = { +type AgentPreviewTargetRef = { + source?: AgentPreviewSource sessionId: string runId?: string } +type AgentPreviewActionHandler = (action: AgentPreviewAction, target: AgentPreviewTarget) => void + +type AgentPreviewClaim = { + source: AgentPreviewSource + runId: string + sequence: number +} + type HostListeners = { focus: () => void blur: () => void @@ -48,25 +58,35 @@ const OPEN_PANEL_ICON_DATA_URL = const CLOSE_ICON_DATA_URL = 'data:image/png;base64,' + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAdklEQVQ4jbWTSwrAIAxE31G71F111d6z9DoWQSSKn1DbB64yM5gY4SM24AaMQmuSNnoyFxDS8fTxQhc9GSsKvRBpDslTsFeCU9RcVTtK6zhEbe5dddaaOkRtpnHteib/teAaAxu9TsFo2tMQu7pIy6u8/Jle8wCGAVmb4KsRPAAAAABJRU5ErkJggg==' -const TOOLBAR_OPTIONS: Parameters[0] = { + +const toolbarOptions = (source: AgentPreviewSource): Parameters[0] => ({ toolbar: { style: 'dark', - buttons: [ - { - id: OPEN_PANEL_CONTROL_ID, - imageData: OPEN_PANEL_ICON_DATA_URL, - tooltip: 'Open in side panel' - }, - { - id: CLOSE_CONTROL_ID, - imageData: CLOSE_ICON_DATA_URL, - tooltip: 'Close' - } - ] - } -} - -export class AgentBrowserNativeOverlay { + buttons: + source === 'browser' + ? [ + { + id: OPEN_PANEL_CONTROL_ID, + imageData: OPEN_PANEL_ICON_DATA_URL, + tooltip: 'Open in side panel' + }, + { + id: CLOSE_CONTROL_ID, + imageData: CLOSE_ICON_DATA_URL, + tooltip: 'Close' + } + ] + : [ + { + id: CLOSE_CONTROL_ID, + imageData: CLOSE_ICON_DATA_URL, + tooltip: 'Close' + } + ] + } +}) + +export class AgentPreviewCoordinator { private overlay: NativeKitOverlay | null = null private available = false private unavailable = false @@ -74,23 +94,30 @@ export class AgentBrowserNativeOverlay { private host: BrowserWindow | null = null private hostListeners: HostListeners | null = null private hostSyncTimer: ReturnType | null = null - private target: AgentBrowserNativeOverlayTarget | null = null + private target: AgentPreviewTarget | null = null private desiredVisible = false private overlayVisible: boolean | null = null private activeSessionId: string | null = null + private toolbarSource: AgentPreviewSource | null = null + private nextClaimSequence = 0 + private readonly claims = new Map() + private readonly dismissedRuns = new Map() + private readonly handlers = new Map() private lastSlowPushWarningAt = 0 private lastPushFailureWarningAt = 0 private listeningForDisplays = false private readonly handleActivate = () => { - this.emitAction('activate') + if (this.target?.source === 'browser') { + this.emitAction('activate') + } } private readonly handleControl = (controlId: string) => { - if (controlId === OPEN_PANEL_CONTROL_ID) { + if (controlId === OPEN_PANEL_CONTROL_ID && this.target?.source === 'browser') { this.emitAction('activate') } else if (controlId === CLOSE_CONTROL_ID) { - this.emitAction('dismiss') + this.dismissCurrent() } } @@ -98,7 +125,65 @@ export class AgentBrowserNativeOverlay { this.scheduleHostSync() } - constructor(private readonly onAction: NativeOverlayActionHandler) {} + register(source: AgentPreviewSource, handler: AgentPreviewActionHandler): () => void { + this.handlers.set(source, handler) + return () => { + if (this.handlers.get(source) === handler) { + this.handlers.delete(source) + } + } + } + + claim(input: { source: AgentPreviewSource; sessionId: string; runId: string }): number { + const sessionId = input.sessionId.trim() + const runId = input.runId.trim() + if (!sessionId || !runId) { + return 0 + } + + if (this.dismissedRuns.get(sessionId) !== runId) { + this.dismissedRuns.delete(sessionId) + } + + const sequence = ++this.nextClaimSequence + this.claims.set(sessionId, { + source: input.source, + runId, + sequence + }) + return sequence + } + + releaseClaim(target: AgentPreviewTargetRef): void { + const claim = this.claims.get(target.sessionId) + if ( + claim && + (target.source == null || claim.source === target.source) && + (target.runId == null || claim.runId === target.runId) + ) { + this.claims.delete(target.sessionId) + } + this.removeTarget(target) + } + + dismiss(target: AgentPreviewTargetRef): boolean { + const current = this.target + const claim = this.claims.get(target.sessionId) + const runId = target.runId ?? current?.runId ?? claim?.runId + if (!runId) { + return false + } + const currentOwnsRun = + current?.sessionId === target.sessionId && (target.runId == null || current.runId === runId) + const owner = currentOwnsRun ? current.source : claim?.source + if (target.source && owner !== target.source) { + return false + } + + this.dismissedRuns.set(target.sessionId, runId) + this.removeTarget({ sessionId: target.sessionId, runId }) + return true + } async initialize(): Promise { if (this.available) { @@ -119,32 +204,51 @@ export class AgentBrowserNativeOverlay { return this.available } - prepare(target: AgentBrowserNativeOverlayTarget, host: BrowserWindow): boolean { - if (!this.available || !this.overlay || host.isDestroyed()) { - return false + isCurrent(target: AgentPreviewTarget): boolean { + return this.matchesTarget(target) && this.matchesClaim(target) && !this.isDismissed(target) + } + + prepare(target: AgentPreviewTarget, host: BrowserWindow): AgentPreviewSurface { + if ( + host.isDestroyed() || + !this.matchesClaim(target) || + this.isDismissed(target) || + target.claimSequence <= 0 + ) { + return 'none' } + const previous = this.target const presentationChanged = - this.target != null && this.presentationId(this.target) !== this.presentationId(target) + previous != null && this.presentationId(previous) !== this.presentationId(target) if (presentationChanged) { this.desiredVisible = false this.setOverlayVisible(false) this.removeCurrentPresentation() - } - - if (!this.attachHost(host)) { - this.disable('host_attach_failed') - return false + this.target = null + this.handlers.get(previous.source)?.('superseded', { ...previous }) } this.target = { ...target } this.desiredVisible = true - return true + + if (!this.available || !this.overlay) { + return 'renderer-canvas' + } + if (!this.configureToolbar(target.source)) { + this.disableNative('toolbar_update_failed') + return 'renderer-canvas' + } + if (!this.attachHost(host)) { + this.disableNative('host_attach_failed') + return 'renderer-canvas' + } + return 'native-overlay' } - present(target: AgentBrowserNativeOverlayTarget, jpeg: Buffer): boolean { + present(target: AgentPreviewTarget, jpeg: Buffer): boolean { const overlay = this.overlay - if (!this.available || !overlay || !this.matchesTarget(target) || !this.host) { + if (!this.available || !overlay || !this.isCurrent(target) || !this.host) { return false } @@ -153,16 +257,17 @@ export class AgentBrowserNativeOverlay { const pushed = overlay.pushImage({ hostId: this.hostId(target.windowId), presentationId: this.presentationId(target), - sessionId: this.nativeSessionId(target.sessionId), + sessionId: this.nativeSessionId(target), imageData: `data:image/jpeg;base64,${jpeg.toString('base64')}` }) this.recordPushDuration(performance.now() - startedAt) if (!pushed) { this.warnFramePush(new Error('pushImage returned false')) + this.disableNative('frame_push_failed') return false } - const nativeSessionId = this.nativeSessionId(target.sessionId) + const nativeSessionId = this.nativeSessionId(target) if (this.activeSessionId !== nativeSessionId) { if (overlay.setActiveSession(nativeSessionId)) { this.activeSessionId = nativeSessionId @@ -178,20 +283,21 @@ export class AgentBrowserNativeOverlay { } catch (error) { this.recordPushDuration(performance.now() - startedAt) this.warnFramePush(error) + this.disableNative('frame_push_failed') return false } } - hide(target?: NativeOverlayTargetRef): void { - if (target && !this.matchesRun(target)) { + hide(target?: AgentPreviewTargetRef): void { + if (target && !this.matchesRef(target)) { return } this.desiredVisible = false this.setOverlayVisible(false) } - removeTarget(target?: NativeOverlayTargetRef): void { - if (target && !this.matchesRun(target)) { + removeTarget(target?: AgentPreviewTargetRef): void { + if (target && !this.matchesRef(target)) { return } this.desiredVisible = false @@ -202,27 +308,11 @@ export class AgentBrowserNativeOverlay { shutdown(): void { this.removeTarget() - this.unbindHost() - this.stopDisplayListeners() - - const overlay = this.overlay - this.overlay = null - this.available = false + this.claims.clear() + this.dismissedRuns.clear() + this.handlers.clear() this.unavailable = true - this.overlayVisible = null - this.activeSessionId = null - this.initialization = null - if (!overlay) { - return - } - - overlay.removeListener('activate', this.handleActivate) - overlay.removeListener('control', this.handleControl) - try { - overlay.stop() - } catch (error) { - this.warn('shutdown_failed', error) - } + this.stopNative() } private async start(): Promise { @@ -236,7 +326,7 @@ export class AgentBrowserNativeOverlay { const nativekit = await import('@zerob13/nativekit') const overlay = nativekit.overlay if ( - !overlay.start(TOOLBAR_OPTIONS) || + !overlay.start(toolbarOptions('browser')) || !overlay.setMaxSize(PREVIEW_MAX_EDGE) || !overlay.setVisible(false) ) { @@ -253,22 +343,43 @@ export class AgentBrowserNativeOverlay { overlay.on('activate', this.handleActivate) overlay.on('control', this.handleControl) this.overlay = overlay + this.toolbarSource = 'browser' this.overlayVisible = false this.available = true this.startDisplayListeners() return true } catch (error) { - this.shutdown() + this.unavailable = true + this.stopNative() this.logUnavailable('load_failed', error) return false } } - private attachHost(host: BrowserWindow): boolean { + private configureToolbar(source: AgentPreviewSource): boolean { + if (!this.overlay || this.toolbarSource === source) { + return Boolean(this.overlay) + } + try { + if (!this.overlay.start(toolbarOptions(source))) { + return false + } + this.toolbarSource = source + return true + } catch (error) { + this.warn('toolbar_update_failed', error) + return false + } + } + + private attachHost(host: BrowserWindow, force = false): boolean { const overlay = this.overlay if (!overlay || host.isDestroyed()) { return false } + if (!force && this.host?.id === host.id && this.hostListeners) { + return true + } if (this.host && this.host.id !== host.id) { this.unbindHost() @@ -367,7 +478,7 @@ export class AgentBrowserNativeOverlay { this.hostSyncTimer = null const host = this.host if (host && !host.isDestroyed()) { - this.attachHost(host) + this.attachHost(host, true) } }, HOST_SYNC_DELAY_MS) this.hostSyncTimer.unref?.() @@ -400,14 +511,27 @@ export class AgentBrowserNativeOverlay { screen.removeListener('display-metrics-changed', this.handleDisplayChange) } - private emitAction(action: AgentBrowserNativeOverlayAction): void { + private emitAction(action: 'activate' | 'dismiss'): void { + const target = this.target + if (!target) { + return + } + this.desiredVisible = false + this.setOverlayVisible(false) + this.handlers.get(target.source)?.(action, { ...target }) + } + + private dismissCurrent(): void { const target = this.target if (!target) { return } + this.dismissedRuns.set(target.sessionId, target.runId) this.desiredVisible = false this.setOverlayVisible(false) - this.onAction(action, { ...target }) + this.removeCurrentPresentation() + this.target = null + this.handlers.get(target.source)?.('dismiss', { ...target }) } private setOverlayVisible(visible: boolean): void { @@ -437,22 +561,38 @@ export class AgentBrowserNativeOverlay { } } - private matchesTarget(target: AgentBrowserNativeOverlayTarget): boolean { + private matchesTarget(target: AgentPreviewTarget): boolean { return ( - this.target?.windowId === target.windowId && + this.target?.source === target.source && + this.target.windowId === target.windowId && this.target.sessionId === target.sessionId && this.target.runId === target.runId && - this.target.captureEpoch === target.captureEpoch + this.target.epoch === target.epoch && + this.target.claimSequence === target.claimSequence ) } - private matchesRun(target: NativeOverlayTargetRef): boolean { + private matchesRef(target: AgentPreviewTargetRef): boolean { return ( this.target?.sessionId === target.sessionId && + (target.source == null || this.target.source === target.source) && (target.runId == null || this.target.runId === target.runId) ) } + private matchesClaim(target: AgentPreviewTarget): boolean { + const claim = this.claims.get(target.sessionId) + return ( + claim?.source === target.source && + claim.runId === target.runId && + claim.sequence === target.claimSequence + ) + } + + private isDismissed(target: Pick): boolean { + return this.dismissedRuns.get(target.sessionId) === target.runId + } + private isHostEligible(host: BrowserWindow): boolean { return !host.isDestroyed() && host.isVisible() && host.isFocused() && !host.isMinimized() } @@ -472,12 +612,12 @@ export class AgentBrowserNativeOverlay { return `chat-window:${windowId}` } - private presentationId(target: AgentBrowserNativeOverlayTarget): string { - return `agent-browser:${target.windowId}:${target.sessionId}` + private presentationId(target: AgentPreviewTarget): string { + return `agent-preview:${target.source}:${target.windowId}:${target.sessionId}` } - private nativeSessionId(sessionId: string): string { - return `agent-browser:${sessionId}` + private nativeSessionId(target: AgentPreviewTarget): string { + return `agent-preview:${target.source}:${target.sessionId}` } private normalizeBounds(bounds: Rectangle): Rectangle { @@ -489,10 +629,34 @@ export class AgentBrowserNativeOverlay { } } - private disable(reason: string): void { + private disableNative(reason: string): void { this.unavailable = true this.logUnavailable(reason) - this.shutdown() + this.stopNative() + } + + private stopNative(): void { + this.unbindHost() + this.stopDisplayListeners() + + const overlay = this.overlay + this.overlay = null + this.available = false + this.overlayVisible = null + this.activeSessionId = null + this.toolbarSource = null + this.initialization = null + if (!overlay) { + return + } + + overlay.removeListener('activate', this.handleActivate) + overlay.removeListener('control', this.handleControl) + try { + overlay.stop() + } catch (error) { + this.warn('shutdown_failed', error) + } } private recordPushDuration(durationMs: number): void { @@ -503,7 +667,7 @@ export class AgentBrowserNativeOverlay { return } this.lastSlowPushWarningAt = Date.now() - logger.warn('[AgentBrowserNativeOverlay] Slow frame presentation', { + logger.warn('[AgentPreviewCoordinator] Slow frame presentation', { durationMs: Math.round(durationMs) }) } @@ -517,7 +681,7 @@ export class AgentBrowserNativeOverlay { } private logUnavailable(reason: string, error?: unknown): void { - logger.info('[AgentBrowserNativeOverlay] Native overlay unavailable', { + logger.info('[AgentPreviewCoordinator] Native overlay unavailable', { platform: process.platform, arch: process.arch, reason, @@ -526,7 +690,7 @@ export class AgentBrowserNativeOverlay { } private warn(reason: string, error: unknown): void { - logger.warn('[AgentBrowserNativeOverlay] Native overlay operation failed', { + logger.warn('[AgentPreviewCoordinator] Native overlay operation failed', { reason, error: error instanceof Error ? error.message : String(error) }) diff --git a/src/main/desktop/routes.ts b/src/main/desktop/routes.ts index e53f3c924..461f55c1c 100644 --- a/src/main/desktop/routes.ts +++ b/src/main/desktop/routes.ts @@ -3,6 +3,7 @@ import type { IShortcutPresenter, ITabPresenter, IWindowPresenter, + IComputerUsePreviewPresenter, IYoBrowserPresenter } from '@shared/types/desktop' import type { DialogServicePort } from '@shared/types/dialog' @@ -11,6 +12,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -22,6 +24,8 @@ import { browserScanImportSourcesRoute, browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute, + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute, configGetFloatingButtonRoute, configGetLanguageRoute, configGetShortcutKeysRoute, @@ -56,18 +60,28 @@ import { } from '@shared/contracts/routes' import { DEV_EVENTS } from '../events' import { createRouteMap, type DeepchatRouteMap, type RouteContext } from '@/routes/routeRegistry' +import type { DesktopSessionBinding } from '@/desktop/sessionBinding' export function createDesktopRoutes(deps: { windowPresenter: IWindowPresenter shortcutPresenter: IShortcutPresenter browserPresenter: IYoBrowserPresenter + computerUsePreviewPresenter: IComputerUsePreviewPresenter + desktopSessionBinding: Pick tabPresenter: ITabPresenter dialogService: DialogServicePort settings: DesktopSettings setFloatingButtonEnabled(enabled: boolean): void recordActivity(input: SettingsActivityInput): void }): DeepchatRouteMap { - const { windowPresenter, shortcutPresenter, browserPresenter, tabPresenter, dialogService } = deps + const { + windowPresenter, + shortcutPresenter, + browserPresenter, + computerUsePreviewPresenter, + tabPresenter, + dialogService + } = deps const readWindowState = (context: RouteContext) => { const window = context.windowId == null ? null : BrowserWindow.fromId(context.windowId) const exists = Boolean(window && !window.isDestroyed()) @@ -402,6 +416,50 @@ export function createDesktopRoutes(deps: { ) } ], + [ + browserDismissPreviewRoute.name, + async (rawInput) => { + const input = browserDismissPreviewRoute.input.parse(rawInput) + return browserDismissPreviewRoute.output.parse({ + dismissed: browserPresenter.dismissPreview(input.sessionId, input.runId) + }) + } + ], + [ + computerUseSetPreviewModeRoute.name, + async (rawInput, context) => { + const input = computerUseSetPreviewModeRoute.input.parse(rawInput) + if ( + context.windowId == null || + (input.mode !== 'stopped' && + deps.desktopSessionBinding.getActiveId(context.webContentsId) !== input.sessionId) + ) { + return computerUseSetPreviewModeRoute.output.parse({ + updated: false, + surface: 'none' + }) + } + return computerUseSetPreviewModeRoute.output.parse( + await computerUsePreviewPresenter.setPreviewMode( + input.sessionId, + input.mode, + context.windowId + ) + ) + } + ], + [ + computerUseDismissPreviewRoute.name, + async (rawInput, context) => { + const input = computerUseDismissPreviewRoute.input.parse(rawInput) + const active = + deps.desktopSessionBinding.getActiveId(context.webContentsId) === input.sessionId + return computerUseDismissPreviewRoute.output.parse({ + dismissed: + active && computerUsePreviewPresenter.dismissPreview(input.sessionId, input.runId) + }) + } + ], [ browserDestroyRoute.name, async (rawInput) => { diff --git a/src/main/mcp/index.ts b/src/main/mcp/index.ts index fae6e66e5..287a7d65d 100644 --- a/src/main/mcp/index.ts +++ b/src/main/mcp/index.ts @@ -20,7 +20,7 @@ import { import type { ProviderRuntimePort } from '@shared/types/provider' import { ServerManager } from './serverManager' import type { McpClient as RuntimeMcpClient } from './mcpClient' -import { ToolManager } from './toolManager' +import { ToolManager, type ComputerUsePreviewObserver } from './toolManager' import { McpRouterManager } from './mcprouterManager' import { McpOAuthManager } from './mcpOAuthManager' import { getErrorMessageLabels } from '@shared/i18n' @@ -134,7 +134,8 @@ export class McpService implements McpServicePort { providerRuntime: Pick, onRegistryChanged: () => void, private readonly publishEvent: DeepchatEventPublisher, - cacheImage?: (data: string) => Promise + cacheImage?: (data: string) => Promise, + computerUsePreviewObserver?: ComputerUsePreviewObserver ) { logger.info('Initializing MCP service') @@ -167,7 +168,8 @@ export class McpService implements McpServicePort { this.locale, this.mcpSettings, this.serverManager, - this.publishEvent + this.publishEvent, + computerUsePreviewObserver ) // init mcprouter manager try { @@ -822,6 +824,7 @@ export class McpService implements McpServicePort { signal?: AbortSignal agentId?: string enabledServerIds?: string[] + runId?: string } ): Promise<{ content: string; rawData: MCPToolResponse }> { const toolCallResult = await this.toolManager.callTool(request, options) diff --git a/src/main/mcp/toolManager.ts b/src/main/mcp/toolManager.ts index a519fde80..a5d37c973 100644 --- a/src/main/mcp/toolManager.ts +++ b/src/main/mcp/toolManager.ts @@ -32,6 +32,26 @@ type McpToolAccessContext = { conversationId?: string } +export type ComputerUsePreviewCall = { + conversationId: string + runId: string + toolCallId: string + toolName: string + args: Record + source: { + serverName: string + ownerPluginId?: string + sourceId?: string + } +} + +export type ComputerUsePreviewObserver = { + shouldCaptureAfterClick?(call: ComputerUsePreviewCall): boolean + started(call: ComputerUsePreviewCall): void + completed(call: ComputerUsePreviewCall, result: MCPToolResponse): void + failed(call: ComputerUsePreviewCall, error: unknown): void +} + type ActiveToolDefinitionsRefresh = { completion: Promise settle: () => void @@ -76,7 +96,8 @@ export class ToolManager { locale: Pick, mcpSettings: McpSettings, serverManager: ServerManager, - private readonly publishEvent: DeepchatEventPublisher + private readonly publishEvent: DeepchatEventPublisher, + private readonly computerUsePreviewObserver?: ComputerUsePreviewObserver ) { this.agentSettings = agentSettings this.locale = locale @@ -104,11 +125,13 @@ export class ToolManager { private isCuaComputerUseServer(client: McpClient, serverConfig?: MCPServerConfig): boolean { const clientConfig = client.serverConfig as { ownerPluginId?: unknown + source?: unknown sourceId?: unknown } const ownerPluginId = serverConfig?.ownerPluginId ?? clientConfig.ownerPluginId + const source = serverConfig?.source ?? clientConfig.source const sourceId = serverConfig?.sourceId ?? clientConfig.sourceId - return ownerPluginId === CUA_PLUGIN_ID || sourceId === CUA_PLUGIN_ID + return ownerPluginId === CUA_PLUGIN_ID || (source === 'plugin' && sourceId === CUA_PLUGIN_ID) } public async getRunningClients(): Promise { @@ -582,8 +605,11 @@ export class ToolManager { toolCall: MCPToolCall, access?: Pick & { signal?: AbortSignal + runId?: string } ): Promise { + let previewCall: ComputerUsePreviewCall | null = null + let previewTerminalNotified = false try { access?.signal?.throwIfAborted() const finalName = toolCall.function.name @@ -772,37 +798,32 @@ export class ToolManager { } } + previewCall = + originalName === 'get_window_state' + ? this.createComputerUsePreviewCall({ + client: targetClient, + serverConfig, + toolCall, + toolName: originalName, + args: preparedArgs.args, + runId: access?.runId + }) + : null + if (previewCall) { + this.notifyComputerUsePreview('started', previewCall) + } + // Call the tool on the target client using the ORIGINAL name const result = access?.signal ? await targetClient.callTool(originalName, preparedArgs.args, { signal: access.signal }) : await targetClient.callTool(originalName, preparedArgs.args) access?.signal?.throwIfAborted() - // Format response - let formattedContent: string | MCPContentItem[] = '' - if (typeof result.content === 'string') { - formattedContent = result.content - } else if (Array.isArray(result.content)) { - formattedContent = result.content.map((item): MCPContentItem => { - if (typeof item === 'string') { - return { type: 'text', text: item } as MCPTextContent - } - if (item.type === 'text' || item.type === 'image' || item.type === 'resource') { - return item as MCPContentItem - } - if (item.type && item.text) { - return { type: 'text', text: item.text } as MCPTextContent - } - return { type: 'text', text: JSON.stringify(item) } as MCPTextContent - }) - } else if (result.content) { - formattedContent = JSON.stringify(result.content) - } + const response = this.formatToolResponse(toolCall.id, result) - const response: MCPToolResponse = { - toolCallId: toolCall.id, - content: formattedContent, - isError: result.isError + if (previewCall) { + this.notifyComputerUsePreview('completed', previewCall, response) + previewTerminalNotified = true } this.publishEvent('mcp.toolCall.result', { @@ -811,8 +832,22 @@ export class ToolManager { version: Date.now() }) + this.scheduleComputerUsePreviewAfterClick({ + client: targetClient, + serverConfig, + toolCall, + toolName: originalName, + args: preparedArgs.args, + runId: access?.runId, + response, + signal: access?.signal + }) + return response } catch (error: unknown) { + if (previewCall && !previewTerminalNotified) { + this.notifyComputerUsePreview('failed', previewCall, error) + } if (access?.signal?.aborted || isAbortError(error)) { throw error } @@ -827,6 +862,189 @@ export class ToolManager { } } + private createComputerUsePreviewCall(input: { + client: McpClient + serverConfig: MCPServerConfig + toolCall: MCPToolCall + toolName: string + args: Record + runId?: string + }): ComputerUsePreviewCall | null { + const conversationId = input.toolCall.conversationId?.trim() + const runId = input.runId?.trim() + if ( + !conversationId || + !runId || + !this.isCuaComputerUseServer(input.client, input.serverConfig) + ) { + return null + } + + const clientConfig = input.client.serverConfig as { + ownerPluginId?: unknown + sourceId?: unknown + } + const ownerPluginId = input.serverConfig.ownerPluginId ?? clientConfig.ownerPluginId + const sourceId = input.serverConfig.sourceId ?? clientConfig.sourceId + return { + conversationId, + runId, + toolCallId: input.toolCall.id, + toolName: input.toolName, + args: { ...input.args }, + source: { + serverName: input.client.serverName, + ...(typeof ownerPluginId === 'string' ? { ownerPluginId } : {}), + ...(typeof sourceId === 'string' ? { sourceId } : {}) + } + } + } + + private scheduleComputerUsePreviewAfterClick(input: { + client: McpClient + serverConfig: MCPServerConfig + toolCall: MCPToolCall + toolName: string + args: Record + runId?: string + response: MCPToolResponse + signal?: AbortSignal + }): void { + if ( + input.toolName !== 'click' || + input.response.isError === true || + getPluginToolPolicy(input.client.serverName, 'get_window_state') !== 'allow' || + !this.computerUsePreviewObserver?.shouldCaptureAfterClick + ) { + return + } + + const clickCall = this.createComputerUsePreviewCall(input) + const pid = this.readPositiveIntegerArg(input.args.pid) + const windowId = this.readPositiveIntegerArg(input.args.window_id) + if (!clickCall || pid == null || windowId == null) { + return + } + + let shouldCapture = false + try { + shouldCapture = this.computerUsePreviewObserver.shouldCaptureAfterClick(clickCall) + } catch (error) { + logger.warn('[ToolManager] Computer Use preview eligibility check failed', { + toolCallId: clickCall.toolCallId, + error: error instanceof Error ? error.message : String(error) + }) + } + if (!shouldCapture) { + return + } + + const snapshotCall: ComputerUsePreviewCall = { + ...clickCall, + toolCallId: `${clickCall.toolCallId}:pip-snapshot`, + toolName: 'get_window_state', + args: { + pid, + window_id: windowId + } + } + void this.captureComputerUsePreviewSnapshot(input.client, snapshotCall, input.signal) + } + + private async captureComputerUsePreviewSnapshot( + client: McpClient, + call: ComputerUsePreviewCall, + signal?: AbortSignal + ): Promise { + let started = false + try { + signal?.throwIfAborted() + this.notifyComputerUsePreview('started', call) + started = true + const result = signal + ? await client.callTool('get_window_state', call.args, { signal }) + : await client.callTool('get_window_state', call.args) + signal?.throwIfAborted() + this.notifyComputerUsePreview( + 'completed', + call, + this.formatToolResponse(call.toolCallId, result) + ) + } catch (error) { + if (started) { + this.notifyComputerUsePreview('failed', call, error) + } + } + } + + private formatToolResponse( + toolCallId: string, + result: { content?: unknown; isError?: boolean } + ): MCPToolResponse { + let formattedContent: string | MCPContentItem[] = '' + if (typeof result.content === 'string') { + formattedContent = result.content + } else if (Array.isArray(result.content)) { + formattedContent = result.content.map((item): MCPContentItem => { + if (typeof item === 'string') { + return { type: 'text', text: item } as MCPTextContent + } + if (item && typeof item === 'object' && ('type' in item || 'text' in item)) { + const contentItem = item as { type?: unknown; text?: unknown } + if ( + contentItem.type === 'text' || + contentItem.type === 'image' || + contentItem.type === 'resource' + ) { + return item as MCPContentItem + } + if (contentItem.type && contentItem.text) { + return { type: 'text', text: String(contentItem.text) } as MCPTextContent + } + } + return { type: 'text', text: JSON.stringify(item) } as MCPTextContent + }) + } else if (result.content) { + formattedContent = JSON.stringify(result.content) + } + + return { + toolCallId, + content: formattedContent, + isError: result.isError + } + } + + private readPositiveIntegerArg(value: unknown): number | null { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null + } + + private notifyComputerUsePreview( + phase: 'started' | 'completed' | 'failed', + call: ComputerUsePreviewCall, + value?: MCPToolResponse | unknown + ): void { + const observer = this.computerUsePreviewObserver + if (!observer) { + return + } + try { + if (phase === 'started') { + observer.started(call) + } else if (phase === 'completed') { + observer.completed(call, value as MCPToolResponse) + } else { + observer.failed(call, value) + } + } catch (error) { + logger.warn('[ToolManager] Computer Use preview observer failed', { + phase, + toolCallId: call.toolCallId, + error: error instanceof Error ? error.message : String(error) + }) + } + } + private async prepareToolArguments( client: McpClient, serverConfig: MCPServerConfig, diff --git a/src/main/tool/index.ts b/src/main/tool/index.ts index 9d124a0fc..fdbaad76c 100644 --- a/src/main/tool/index.ts +++ b/src/main/tool/index.ts @@ -355,6 +355,7 @@ export class ToolService implements ToolServicePort { return await this.options.mcpService.callTool(request, { agentId: options?.agentId ?? storedAccess?.agentId, enabledServerIds: options?.enabledMcpServerIds ?? storedAccess?.enabledMcpServerIds, + runId: options?.runId, signal: options?.signal }) } diff --git a/src/renderer/api/BrowserClient.ts b/src/renderer/api/BrowserClient.ts index 5358e8bd5..e48b2fcf0 100644 --- a/src/renderer/api/BrowserClient.ts +++ b/src/renderer/api/BrowserClient.ts @@ -4,6 +4,7 @@ import { browserOpenRequestedEvent, browserPreviewActionEvent, browserPreviewFrameEvent, + browserPreviewSurfaceChangedEvent, browserStatusChangedEvent, type DeepchatEventPayload } from '@shared/contracts/events' @@ -11,6 +12,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -88,6 +90,11 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return await bridge.invoke(browserSetPreviewModeRoute.name, { sessionId, mode, runId }) } + async function dismissPreview(sessionId: string, runId: string) { + const result = await bridge.invoke(browserDismissPreviewRoute.name, { sessionId, runId }) + return result.dismissed + } + async function destroy(sessionId: string) { const result = await bridge.invoke(browserDestroyRoute.name, { sessionId }) return result.destroyed @@ -211,6 +218,12 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() return bridge.on(browserPreviewActionEvent.name, listener) } + function onPreviewSurfaceChanged( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(browserPreviewSurfaceChangedEvent.name, listener) + } + return { getStatus, loadUrl, @@ -218,6 +231,7 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() updateCurrentWindowBounds, detach, setPreviewMode, + dismissPreview, destroy, goBack, goForward, @@ -232,7 +246,8 @@ export function createBrowserClient(bridge: DeepchatBridge = getDeepchatBridge() onStatusChanged, onActivityChanged, onPreviewFrame, - onPreviewAction + onPreviewAction, + onPreviewSurfaceChanged } } diff --git a/src/renderer/api/ComputerUseClient.ts b/src/renderer/api/ComputerUseClient.ts new file mode 100644 index 000000000..4516b64fc --- /dev/null +++ b/src/renderer/api/ComputerUseClient.ts @@ -0,0 +1,46 @@ +import type { DeepchatBridge } from '@shared/contracts/bridge' +import { + computerUsePreviewFrameEvent, + computerUsePreviewSurfaceChangedEvent, + type DeepchatEventPayload +} from '@shared/contracts/events' +import { + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute +} from '@shared/contracts/routes' +import type { ComputerUsePreviewMode } from '@shared/types/computerUse' +import { getDeepchatBridge } from './core' + +export function createComputerUseClient(bridge: DeepchatBridge = getDeepchatBridge()) { + async function setPreviewMode(sessionId: string, mode: ComputerUsePreviewMode) { + return await bridge.invoke(computerUseSetPreviewModeRoute.name, { sessionId, mode }) + } + + async function dismissPreview(sessionId: string, runId: string) { + const result = await bridge.invoke(computerUseDismissPreviewRoute.name, { sessionId, runId }) + return result.dismissed + } + + function onPreviewFrame( + listener: (payload: DeepchatEventPayload) => void + ) { + return bridge.on(computerUsePreviewFrameEvent.name, listener) + } + + function onPreviewSurfaceChanged( + listener: ( + payload: DeepchatEventPayload + ) => void + ) { + return bridge.on(computerUsePreviewSurfaceChangedEvent.name, listener) + } + + return { + setPreviewMode, + dismissPreview, + onPreviewFrame, + onPreviewSurfaceChanged + } +} + +export type ComputerUseClient = ReturnType diff --git a/src/renderer/src/apps/chat-main/ChatTabView.vue b/src/renderer/src/apps/chat-main/ChatTabView.vue index db48817d5..65373fd87 100644 --- a/src/renderer/src/apps/chat-main/ChatTabView.vue +++ b/src/renderer/src/apps/chat-main/ChatTabView.vue @@ -32,6 +32,9 @@ + void) | null = null let stopActivityChanged: (() => void) | null = null let stopPreviewFrame: (() => void) | null = null let stopPreviewAction: (() => void) | null = null +let stopPreviewSurfaceChanged: (() => void) | null = null let stopWindowStateChanged: (() => void) | null = null let dragState: | { @@ -275,8 +276,12 @@ const loadStatus = async () => { } const dismiss = () => { - dismissedRunId.value = currentRunId.value + const sessionId = currentSessionId.value + const runId = currentRunId.value + if (!sessionId || !runId) return + dismissedRunId.value = runId toolbarVisible.value = false + void browserClient.dismissPreview(sessionId, runId).catch(() => undefined) } const openInPanel = async () => { @@ -506,11 +511,26 @@ onMounted(() => { return } if (payload.action === 'dismiss') { - dismiss() + void dismiss() return } void openInPanel() }) + stopPreviewSurfaceChanged = browserClient.onPreviewSurfaceChanged((payload) => { + if ( + payload.windowId !== currentWindowId.value || + payload.sessionId !== currentSessionId.value || + payload.runId !== currentRunId.value + ) { + return + } + if (payload.surface === 'none') { + frameDecodeVersion += 1 + latestFrameSequence = -1 + hasFrame.value = false + } + previewSurface.value = payload.surface + }) stopWindowStateChanged = windowClient.onCurrentStateChanged((payload) => { windowStateVersion += 1 currentWindowId.value = payload.windowId @@ -533,6 +553,7 @@ onBeforeUnmount(() => { stopActivityChanged?.() stopPreviewFrame?.() stopPreviewAction?.() + stopPreviewSurfaceChanged?.() stopWindowStateChanged?.() if (currentSessionId.value) { pendingPreviewRequest = { diff --git a/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue new file mode 100644 index 000000000..d56ba7599 --- /dev/null +++ b/src/renderer/src/components/computerUse/AgentComputerUsePiP.vue @@ -0,0 +1,407 @@ + + + diff --git a/src/shared/contracts/events.ts b/src/shared/contracts/events.ts index be01e4dcd..697122f5f 100644 --- a/src/shared/contracts/events.ts +++ b/src/shared/contracts/events.ts @@ -21,8 +21,13 @@ import { browserOpenRequestedEvent, browserPreviewActionEvent, browserPreviewFrameEvent, + browserPreviewSurfaceChangedEvent, browserStatusChangedEvent } from './events/browser.events' +import { + computerUsePreviewFrameEvent, + computerUsePreviewSurfaceChangedEvent +} from './events/computerUse.events' import { chatPlanUpdatedEvent, chatStreamCompletedEvent, @@ -129,6 +134,7 @@ import { } from './events/workspace.events' export * from './events/browser.events' +export * from './events/computerUse.events' export * from './events/acp-terminal.events' export * from './events/app-runtime.events' export * from './events/chat.events' @@ -161,6 +167,9 @@ export const DEEPCHAT_EVENT_CATALOG = { [browserActivityChangedEvent.name]: browserActivityChangedEvent, [browserPreviewActionEvent.name]: browserPreviewActionEvent, [browserPreviewFrameEvent.name]: browserPreviewFrameEvent, + [browserPreviewSurfaceChangedEvent.name]: browserPreviewSurfaceChangedEvent, + [computerUsePreviewFrameEvent.name]: computerUsePreviewFrameEvent, + [computerUsePreviewSurfaceChangedEvent.name]: computerUsePreviewSurfaceChangedEvent, [browserOpenRequestedEvent.name]: browserOpenRequestedEvent, [browserStatusChangedEvent.name]: browserStatusChangedEvent, [settingsChangedEvent.name]: settingsChangedEvent, diff --git a/src/shared/contracts/events/browser.events.ts b/src/shared/contracts/events/browser.events.ts index 3a79c25de..c3113bd28 100644 --- a/src/shared/contracts/events/browser.events.ts +++ b/src/shared/contracts/events/browser.events.ts @@ -98,3 +98,14 @@ export const browserPreviewActionEvent = defineEventContract({ runId: z.string().min(1) }) }) + +export const browserPreviewSurfaceChangedEvent = defineEventContract({ + name: 'browser.preview.surface.changed', + payload: z.object({ + windowId: z.number().int().positive(), + sessionId: z.string().min(1), + runId: z.string().min(1), + surface: z.enum(['native-overlay', 'renderer-canvas', 'none']), + version: TimestampMsSchema + }) +}) diff --git a/src/shared/contracts/events/computerUse.events.ts b/src/shared/contracts/events/computerUse.events.ts new file mode 100644 index 000000000..4b8a29c51 --- /dev/null +++ b/src/shared/contracts/events/computerUse.events.ts @@ -0,0 +1,33 @@ +import { z } from 'zod' +import { TimestampMsSchema, defineEventContract } from '../common' + +const PreviewSurfaceSchema = z.enum(['native-overlay', 'renderer-canvas', 'none']) + +export const computerUsePreviewFrameEvent = defineEventContract({ + name: 'computerUse.preview.frame', + payload: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1), + epoch: z.number().int().nonnegative(), + sequence: z.number().int().nonnegative(), + width: z.number().int().positive().max(480), + height: z.number().int().positive().max(300), + mimeType: z.literal('image/jpeg'), + data: z.custom( + (value) => value instanceof Uint8Array && value.byteLength <= 512 * 1024 + ), + timestamp: TimestampMsSchema + }) +}) + +export const computerUsePreviewSurfaceChangedEvent = defineEventContract({ + name: 'computerUse.preview.surface.changed', + payload: z.object({ + windowId: z.number().int().positive(), + sessionId: z.string().min(1), + runId: z.string().min(1), + epoch: z.number().int().nonnegative(), + surface: PreviewSurfaceSchema, + version: TimestampMsSchema + }) +}) diff --git a/src/shared/contracts/routes.ts b/src/shared/contracts/routes.ts index c28f80404..658407605 100644 --- a/src/shared/contracts/routes.ts +++ b/src/shared/contracts/routes.ts @@ -5,6 +5,7 @@ import { browserAttachCurrentWindowRoute, browserApplyImportRoute, browserClearSandboxDataRoute, + browserDismissPreviewRoute, browserDestroyRoute, browserDetachRoute, browserGetStatusRoute, @@ -17,6 +18,10 @@ import { browserSetPreviewModeRoute, browserUpdateCurrentWindowBoundsRoute } from './routes/browser.routes' +import { + computerUseDismissPreviewRoute, + computerUseSetPreviewModeRoute +} from './routes/computerUse.routes' import { chatCancelSubmissionRoute, chatRespondToolInteractionRoute, @@ -508,6 +513,7 @@ import { } from './routes/workspace.routes' export * from './routes/browser.routes' +export * from './routes/computerUse.routes' export * from './routes/acp-terminal.routes' export * from './routes/chat.routes' export * from './routes/config.routes' @@ -693,8 +699,11 @@ const DEEPCHAT_ROUTE_CATALOG_PART_2 = { [browserClearSandboxDataRoute.name]: browserClearSandboxDataRoute, [browserScanImportSourcesRoute.name]: browserScanImportSourcesRoute, [browserSetPreviewModeRoute.name]: browserSetPreviewModeRoute, + [browserDismissPreviewRoute.name]: browserDismissPreviewRoute, [browserPreviewImportRoute.name]: browserPreviewImportRoute, [browserApplyImportRoute.name]: browserApplyImportRoute, + [computerUseSetPreviewModeRoute.name]: computerUseSetPreviewModeRoute, + [computerUseDismissPreviewRoute.name]: computerUseDismissPreviewRoute, [tabCaptureCurrentAreaRoute.name]: tabCaptureCurrentAreaRoute, [tabStitchImagesWithWatermarkRoute.name]: tabStitchImagesWithWatermarkRoute } satisfies Record diff --git a/src/shared/contracts/routes/browser.routes.ts b/src/shared/contracts/routes/browser.routes.ts index ce93a49ea..ba5bc6e20 100644 --- a/src/shared/contracts/routes/browser.routes.ts +++ b/src/shared/contracts/routes/browser.routes.ts @@ -69,6 +69,17 @@ export const browserSetPreviewModeRoute = defineRouteContract({ }) }) +export const browserDismissPreviewRoute = defineRouteContract({ + name: 'browser.dismissPreview', + input: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1) + }), + output: z.object({ + dismissed: z.boolean() + }) +}) + export const browserDestroyRoute = defineRouteContract({ name: 'browser.destroy', input: z.object({ diff --git a/src/shared/contracts/routes/computerUse.routes.ts b/src/shared/contracts/routes/computerUse.routes.ts new file mode 100644 index 000000000..5697bc9bc --- /dev/null +++ b/src/shared/contracts/routes/computerUse.routes.ts @@ -0,0 +1,27 @@ +import { z } from 'zod' +import { defineRouteContract } from '../common' + +const ComputerUsePreviewSurfaceSchema = z.enum(['native-overlay', 'renderer-canvas', 'none']) + +export const computerUseSetPreviewModeRoute = defineRouteContract({ + name: 'computerUse.setPreviewMode', + input: z.object({ + sessionId: z.string().min(1), + mode: z.enum(['eligible', 'suspended', 'stopped']) + }), + output: z.object({ + updated: z.boolean(), + surface: ComputerUsePreviewSurfaceSchema + }) +}) + +export const computerUseDismissPreviewRoute = defineRouteContract({ + name: 'computerUse.dismissPreview', + input: z.object({ + sessionId: z.string().min(1), + runId: z.string().min(1) + }), + output: z.object({ + dismissed: z.boolean() + }) +}) diff --git a/src/shared/types/computerUse.ts b/src/shared/types/computerUse.ts new file mode 100644 index 000000000..469ac65ca --- /dev/null +++ b/src/shared/types/computerUse.ts @@ -0,0 +1,8 @@ +export type ComputerUsePreviewMode = 'eligible' | 'suspended' | 'stopped' + +export type ComputerUsePreviewSurface = 'native-overlay' | 'renderer-canvas' | 'none' + +export interface ComputerUsePreviewModeResult { + updated: boolean + surface: ComputerUsePreviewSurface +} diff --git a/src/shared/types/desktop.ts b/src/shared/types/desktop.ts index 70b9eb7e2..e4fbb3cdd 100644 --- a/src/shared/types/desktop.ts +++ b/src/shared/types/desktop.ts @@ -11,6 +11,7 @@ import type { YoBrowserStatus } from './browser' import type { MCPToolDefinition } from './mcp' +import type { ComputerUsePreviewMode, ComputerUsePreviewModeResult } from './computerUse' import type { ProviderInstallPreview } from '@shared/providerDeeplink' import type { SettingsNavigationPayload } from '@shared/settingsNavigation' @@ -129,6 +130,7 @@ export interface IYoBrowserPresenter { hostWindowId?: number, runId?: string ): Promise + dismissPreview(sessionId: string, runId: string): boolean destroySessionBrowser(sessionId: string): Promise goBack(sessionId: string): Promise goForward(sessionId: string): Promise @@ -156,6 +158,16 @@ export interface IYoBrowserPresenter { } } +export interface IComputerUsePreviewPresenter { + setPreviewMode( + sessionId: string, + mode: ComputerUsePreviewMode, + hostWindowId?: number + ): Promise + dismissPreview(sessionId: string, runId: string): boolean + shutdown(): void +} + export interface ITabPresenter { createTab(windowId: number, url: string, options?: TabCreateOptions): Promise closeTab(tabId: number): Promise diff --git a/src/shared/types/mcp.ts b/src/shared/types/mcp.ts index 54e2348d1..cd417b57b 100644 --- a/src/shared/types/mcp.ts +++ b/src/shared/types/mcp.ts @@ -234,6 +234,7 @@ export interface McpServicePort { signal?: AbortSignal agentId?: string enabledServerIds?: string[] + runId?: string } ): Promise<{ content: string; rawData: MCPToolResponse }> preCheckToolPermission( diff --git a/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts b/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts deleted file mode 100644 index b7be12837..000000000 --- a/test/main/desktop/browser/AgentBrowserNativeOverlay.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -import { EventEmitter } from 'node:events' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -class MockBrowserWindow extends EventEmitter { - readonly id = 7 - destroyed = false - visible = true - focused = true - minimized = false - - isDestroyed() { - return this.destroyed - } - - isVisible() { - return this.visible - } - - isFocused() { - return this.focused - } - - isMinimized() { - return this.minimized - } - - getTitle() { - return 'DeepChat' - } - - getContentBounds() { - return { x: 10, y: 20, width: 800, height: 600 } - } - - getNativeWindowHandle() { - return Buffer.alloc(process.platform === 'linux' ? 4 : 8) - } -} - -describe('AgentBrowserNativeOverlay', () => { - beforeEach(() => { - vi.resetModules() - vi.useFakeTimers() - }) - - afterEach(() => { - vi.useRealTimers() - vi.restoreAllMocks() - }) - - const setup = async (startResult = true) => { - const screen = new EventEmitter() - const overlay = Object.assign(new EventEmitter(), { - start: vi.fn(() => startResult), - stop: vi.fn(() => true), - attachHost: vi.fn(() => true), - detachHost: vi.fn(() => true), - setVisible: vi.fn(() => true), - setMaxSize: vi.fn(() => true), - pushImage: vi.fn(() => true), - setActiveSession: vi.fn(() => true), - removeImage: vi.fn(() => true) - }) - - vi.doMock('electron', () => ({ - app: { - getName: () => 'DeepChat' - }, - screen - })) - vi.doMock('@zerob13/nativekit', () => ({ overlay })) - vi.doMock('@shared/logger', () => ({ - default: { - info: vi.fn(), - warn: vi.fn() - } - })) - - const { AgentBrowserNativeOverlay } = - await import('@/desktop/browser/AgentBrowserNativeOverlay') - const onAction = vi.fn() - const adapter = new AgentBrowserNativeOverlay(onAction) - - return { - adapter, - onAction, - overlay, - screen, - host: new MockBrowserWindow() - } - } - - it('pushes a current JPEG before showing the native panel', async () => { - const { adapter, overlay, host } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 3 - } - - await expect(adapter.initialize()).resolves.toBe(true) - expect(overlay.start).toHaveBeenCalledWith({ - toolbar: { - style: 'dark', - buttons: [ - { - id: 'open-panel', - imageData: expect.stringMatching(/^data:image\/png;base64,/), - tooltip: 'Open in side panel' - }, - { - id: 'close', - imageData: expect.stringMatching(/^data:image\/png;base64,/), - tooltip: 'Close' - } - ] - } - }) - expect(overlay.setMaxSize).toHaveBeenCalledWith(360) - expect(overlay.setVisible).toHaveBeenCalledWith(false) - expect(adapter.prepare(target, host as never)).toBe(true) - expect(overlay.setVisible).toHaveBeenLastCalledWith(false) - - expect(adapter.present(target, Buffer.from('frame'))).toBe(true) - - expect(overlay.pushImage).toHaveBeenCalledWith({ - hostId: 'chat-window:7', - presentationId: 'agent-browser:7:session-1', - sessionId: 'agent-browser:session-1', - imageData: `data:image/jpeg;base64,${Buffer.from('frame').toString('base64')}` - }) - expect(overlay.setActiveSession).toHaveBeenCalledWith('agent-browser:session-1') - expect(overlay.setVisible).toHaveBeenLastCalledWith(true) - expect(overlay.pushImage.mock.invocationCallOrder[0]).toBeLessThan( - overlay.setActiveSession.mock.invocationCallOrder[0] - ) - expect(overlay.setActiveSession.mock.invocationCallOrder[0]).toBeLessThan( - overlay.setVisible.mock.invocationCallOrder.at(-1)! - ) - - adapter.shutdown() - }) - - it('refreshes the same presentation without resetting native placement state', async () => { - const { adapter, overlay, host } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 3 - } - - await adapter.initialize() - adapter.prepare(target, host as never) - adapter.present(target, Buffer.from('frame-1')) - - overlay.attachHost.mockClear() - overlay.detachHost.mockClear() - overlay.removeImage.mockClear() - overlay.setActiveSession.mockClear() - overlay.setVisible.mockClear() - - expect(adapter.present(target, Buffer.from('frame-2'))).toBe(true) - - expect(overlay.pushImage).toHaveBeenCalledTimes(2) - expect(overlay.pushImage).toHaveBeenLastCalledWith( - expect.objectContaining({ - presentationId: 'agent-browser:7:session-1', - sessionId: 'agent-browser:session-1' - }) - ) - expect(overlay.attachHost).not.toHaveBeenCalled() - expect(overlay.detachHost).not.toHaveBeenCalled() - expect(overlay.removeImage).not.toHaveBeenCalled() - expect(overlay.setActiveSession).not.toHaveBeenCalled() - expect(overlay.setVisible).not.toHaveBeenCalled() - - adapter.shutdown() - }) - - it('keeps host movement native and maps unscoped controls to the current target', async () => { - const { adapter, onAction, overlay, host, screen } = await setup() - const target = { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 4 - } - await adapter.initialize() - adapter.prepare(target, host as never) - adapter.present(target, Buffer.from('frame')) - - host.focused = false - host.emit('blur') - expect(overlay.setVisible).toHaveBeenLastCalledWith(false) - - host.emit('move') - screen.emit('display-metrics-changed') - await vi.advanceTimersByTimeAsync(50) - expect(overlay.attachHost).toHaveBeenCalledTimes(2) - - overlay.emit('activate') - overlay.emit('control', 'open-panel') - overlay.emit('control', 'unknown') - overlay.emit('control', 'close') - expect(onAction).toHaveBeenNthCalledWith(1, 'activate', target) - expect(onAction).toHaveBeenNthCalledWith(2, 'activate', target) - expect(onAction).toHaveBeenNthCalledWith(3, 'dismiss', target) - expect(onAction).toHaveBeenCalledTimes(3) - - adapter.shutdown() - }) - - it('reports unavailable without attaching when native startup fails', async () => { - const { adapter, overlay, host } = await setup(false) - - await expect(adapter.initialize()).resolves.toBe(false) - expect( - adapter.prepare( - { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 1 - }, - host as never - ) - ).toBe(false) - expect(overlay.attachHost).not.toHaveBeenCalled() - }) - - it('falls back permanently when the first real host cannot attach', async () => { - const { adapter, overlay, host } = await setup() - overlay.attachHost.mockReturnValue(false) - await adapter.initialize() - - expect( - adapter.prepare( - { - windowId: 7, - sessionId: 'session-1', - runId: 'run-1', - captureEpoch: 1 - }, - host as never - ) - ).toBe(false) - expect(adapter.isAvailable()).toBe(false) - await expect(adapter.initialize()).resolves.toBe(false) - expect(overlay.stop).toHaveBeenCalledTimes(1) - }) -}) diff --git a/test/main/desktop/browser/YoBrowserPresenter.test.ts b/test/main/desktop/browser/YoBrowserPresenter.test.ts index edca8992e..805371d9f 100644 --- a/test/main/desktop/browser/YoBrowserPresenter.test.ts +++ b/test/main/desktop/browser/YoBrowserPresenter.test.ts @@ -9,13 +9,16 @@ const overlayHideMock = vi.fn() const overlayDestroyMock = vi.fn() const nativeInitializeMock = vi.fn(async () => false) const nativeIsAvailableMock = vi.fn(() => false) -const nativePrepareMock = vi.fn(() => false) +const nativePrepareMock = vi.fn(() => 'renderer-canvas') const nativePresentMock = vi.fn(() => true) const nativeHideMock = vi.fn() const nativeRemoveTargetMock = vi.fn() -const nativeShutdownMock = vi.fn() +const nativeReleaseClaimMock = vi.fn() +const nativeClaimMock = vi.fn(() => 1) +const nativeDismissMock = vi.fn(() => true) +const nativeIsCurrentMock = vi.fn(() => true) let nativeActionHandler: - | ((action: 'activate' | 'dismiss', target: Record) => void) + | ((action: 'activate' | 'dismiss' | 'superseded', target: Record) => void) | null = null class MockWebContents extends EventEmitter { @@ -167,8 +170,11 @@ describe('YoBrowserPresenter', () => { overlayDestroyMock.mockClear() nativeInitializeMock.mockResolvedValue(false) nativeIsAvailableMock.mockReturnValue(false) - nativePrepareMock.mockReturnValue(false) + nativePrepareMock.mockReturnValue('renderer-canvas') nativePresentMock.mockReturnValue(true) + nativeClaimMock.mockReturnValue(1) + nativeDismissMock.mockReturnValue(true) + nativeIsCurrentMock.mockReturnValue(true) nativeActionHandler = null vi.useFakeTimers() }) @@ -243,24 +249,6 @@ describe('YoBrowserPresenter', () => { } })) - vi.doMock('@/desktop/browser/AgentBrowserNativeOverlay', () => ({ - AgentBrowserNativeOverlay: class { - constructor( - onAction: (action: 'activate' | 'dismiss', target: Record) => void - ) { - nativeActionHandler = onAction - } - - initialize = nativeInitializeMock - isAvailable = nativeIsAvailableMock - prepare = nativePrepareMock - present = nativePresentMock - hide = nativeHideMock - removeTarget = nativeRemoveTargetMock - shutdown = nativeShutdownMock - } - })) - const { YoBrowserPresenter } = await import('@/desktop/browser/YoBrowserPresenter') const windowPresenter = { show: vi.fn((windowId: number) => { @@ -285,9 +273,37 @@ describe('YoBrowserPresenter', () => { }) } - const presenter = new YoBrowserPresenter(windowPresenter as any, (name, payload) => { - sendToAllWindowsMock('deepchat:event', createDeepchatEventEnvelope(name, payload)) - }) + const previewCoordinator = { + register: vi.fn( + ( + _source: 'browser', + handler: ( + action: 'activate' | 'dismiss' | 'superseded', + target: Record + ) => void + ) => { + nativeActionHandler = handler + return vi.fn() + } + ), + initialize: nativeInitializeMock, + isAvailable: nativeIsAvailableMock, + claim: nativeClaimMock, + releaseClaim: nativeReleaseClaimMock, + dismiss: nativeDismissMock, + prepare: nativePrepareMock, + present: nativePresentMock, + hide: nativeHideMock, + removeTarget: nativeRemoveTargetMock, + isCurrent: nativeIsCurrentMock + } + const presenter = new YoBrowserPresenter( + windowPresenter as any, + (name, payload) => { + sendToAllWindowsMock('deepchat:event', createDeepchatEventEnvelope(name, payload)) + }, + previewCoordinator as never + ) const getSessionWebContents = (sessionId: string) => { return ((presenter as any).sessionBrowsers.get(sessionId)?.view?.webContents ?? @@ -580,7 +596,7 @@ describe('YoBrowserPresenter', () => { windows.set(1, new MockBrowserWindow(1)) nativeInitializeMock.mockResolvedValue(true) nativeIsAvailableMock.mockReturnValue(true) - nativePrepareMock.mockReturnValue(true) + nativePrepareMock.mockReturnValue('native-overlay') await presenter.initialize() const loadPromise = presenter.loadUrl( diff --git a/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts new file mode 100644 index 000000000..49f56eaf6 --- /dev/null +++ b/test/main/desktop/computerUse/ComputerUsePreviewPresenter.test.ts @@ -0,0 +1,495 @@ +import { EventEmitter } from 'node:events' +import sharp from 'sharp' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +class MockBrowserWindow extends EventEmitter { + destroyed = false + + constructor(readonly id: number) { + super() + } + + isDestroyed() { + return this.destroyed + } +} + +describe('ComputerUsePreviewPresenter', () => { + beforeEach(() => { + vi.resetModules() + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const setup = async (nativeSurface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas') => { + const windows = new Map([[7, new MockBrowserWindow(7)]]) + vi.doMock('electron', () => ({ + BrowserWindow: { + fromId: (id: number) => windows.get(id) ?? null + } + })) + vi.doMock('@shared/logger', () => ({ + default: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn() + } + })) + + const { ComputerUsePreviewPresenter } = + await import('@/desktop/computerUse/ComputerUsePreviewPresenter') + let actionHandler: + | ((action: 'activate' | 'dismiss' | 'superseded', target: Record) => void) + | null = null + let nextClaimSequence = 0 + const coordinator = { + register: vi.fn( + ( + _source: 'computer-use', + handler: ( + action: 'activate' | 'dismiss' | 'superseded', + target: Record + ) => void + ) => { + actionHandler = handler + return vi.fn() + } + ), + initialize: vi.fn(async () => nativeSurface === 'native-overlay'), + claim: vi.fn(() => ++nextClaimSequence), + releaseClaim: vi.fn(), + dismiss: vi.fn(() => true), + prepare: vi.fn(() => nativeSurface), + present: vi.fn(() => true), + hide: vi.fn(), + removeTarget: vi.fn() + } + const windowPresenter = { + sendToWindow: vi.fn(() => true) + } + const presenter = new ComputerUsePreviewPresenter( + windowPresenter as never, + coordinator as never + ) + + return { + presenter, + coordinator, + windowPresenter, + windows, + getActionHandler: () => actionHandler + } + } + + const call = ( + toolCallId: string, + options: { + runId?: string + pid?: number + windowId?: number + toolName?: string + } = {} + ) => ({ + conversationId: 'session-1', + runId: options.runId ?? 'run-1', + toolCallId, + toolName: options.toolName ?? 'get_window_state', + args: { + pid: options.pid ?? 12, + window_id: options.windowId ?? 34 + }, + source: { + serverName: 'cua-driver', + ownerPluginId: 'com.deepchat.plugins.cua' + } + }) + + const createPng = async (width = 960, height = 600, red = 20) => + await sharp({ + create: { + width, + height, + channels: 3, + background: { r: red, g: 40, b: 60 } + } + }) + .png() + .toBuffer() + + const completeWithImage = async ( + presenter: { + completed: (call: ReturnType, result: never) => void + }, + currentCall: ReturnType, + image: Buffer, + mimeType = 'image/png' + ) => { + presenter.completed(currentCall, { + toolCallId: currentCall.toolCallId, + content: [ + { + type: 'image', + mimeType, + data: image.toString('base64') + } + ], + isError: false + } as never) + } + + it('allows post-click capture only for the eligible current non-dismissed target', async () => { + const { presenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + presenter.started(call('snapshot')) + const click = call('click', { toolName: 'click' }) + + expect(presenter.shouldCaptureAfterClick(click)).toBe(true) + expect( + presenter.shouldCaptureAfterClick(call('right-click', { toolName: 'right_click' })) + ).toBe(false) + expect( + presenter.shouldCaptureAfterClick(call('stale-target', { toolName: 'click', windowId: 35 })) + ).toBe(false) + + await presenter.setPreviewMode('session-1', 'suspended', 7) + expect(presenter.shouldCaptureAfterClick(click)).toBe(false) + + await presenter.setPreviewMode('session-1', 'eligible', 7) + expect(presenter.dismissPreview('session-1', 'run-1')).toBe(true) + expect(presenter.shouldCaptureAfterClick(click)).toBe(false) + }) + + it('releases preview state when the host closes without renderer cleanup', async () => { + const { presenter, coordinator, windows } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + presenter.started(call('snapshot')) + + const host = windows.get(7)! + host.destroyed = true + host.emit('closed') + + expect(coordinator.releaseClaim).toHaveBeenCalledWith({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + expect(presenter.shouldCaptureAfterClick(call('click', { toolName: 'click' }))).toBe(false) + await expect(presenter.setPreviewMode('session-1', 'stopped', 7)).resolves.toEqual({ + updated: false, + surface: 'none' + }) + }) + + it('publishes a bounded Canvas frame only after a valid current snapshot', async () => { + const { presenter, coordinator, windowPresenter } = await setup() + await expect(presenter.setPreviewMode('session-1', 'eligible', 7)).resolves.toEqual({ + updated: true, + surface: 'none' + }) + const currentCall = call('tool-1') + + presenter.started(currentCall) + + expect(coordinator.prepare).toHaveBeenCalledWith( + expect.objectContaining({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1', + epoch: 1 + }), + expect.objectContaining({ id: 7 }) + ) + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + + await completeWithImage(presenter, currentCall, await createPng()) + + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(true) + ) + const frameEnvelope = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: Record }) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frameEnvelope?.payload).toEqual( + expect.objectContaining({ + sessionId: 'session-1', + runId: 'run-1', + epoch: 1, + sequence: 1, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: expect.any(Uint8Array) + }) + ) + expect((frameEnvelope?.payload.data as Uint8Array).byteLength).toBeLessThanOrEqual(512 * 1024) + expect(coordinator.present).not.toHaveBeenCalled() + }) + + it('clears the old target epoch and rejects an out-of-order result', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-1') + presenter.started(first) + await completeWithImage(presenter, first, await createPng()) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const stale = call('tool-2', { windowId: 35 }) + const current = call('tool-3', { windowId: 35 }) + presenter.started(stale) + presenter.started(current) + await completeWithImage(presenter, stale, await createPng(320, 200, 80)) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + expect( + windowPresenter.sendToWindow.mock.calls.some(([, , envelope]) => { + const event = envelope as { name?: string; payload?: { epoch?: number } } + return event.name === 'computerUse.preview.surface.changed' && event.payload?.epoch === 2 + }) + ).toBe(true) + + await completeWithImage(presenter, current, await createPng(320, 200, 100)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + ) + const frames = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: { epoch?: number } }) + .filter((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frames[1].payload.epoch).toBe(2) + }) + + it('keeps native snapshot bytes out of renderer events', async () => { + const { presenter, coordinator, windowPresenter } = await setup('native-overlay') + await presenter.setPreviewMode('session-1', 'eligible', 7) + const currentCall = call('tool-native') + presenter.started(currentCall) + + await completeWithImage(presenter, currentCall, await createPng(320, 200)) + await vi.waitFor(() => expect(coordinator.present).toHaveBeenCalledOnce()) + + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + }) + + it('rejects malformed images and keeps the last valid same-target frame', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const validCall = call('tool-valid') + presenter.started(validCall) + await completeWithImage(presenter, validCall, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const invalidCall = call('tool-invalid') + presenter.started(invalidCall) + presenter.completed(invalidCall, { + toolCallId: invalidCall.toolCallId, + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'not-base64!' + } + ], + isError: false + }) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + }) + + it('enforces image bounds without upscaling the last valid frame', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const validCall = call('tool-small') + presenter.started(validCall) + await completeWithImage(presenter, validCall, await createPng(120, 80)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const frame = windowPresenter.sendToWindow.mock.calls + .map(([, , envelope]) => envelope as { name: string; payload: Record }) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frame?.payload).toEqual(expect.objectContaining({ width: 120, height: 80 })) + + const jpegCall = call('tool-jpeg') + presenter.started(jpegCall) + const jpeg = await sharp(await createPng(160, 90)) + .jpeg() + .toBuffer() + await completeWithImage(presenter, jpegCall, jpeg, 'image/jpeg') + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + ) + + const unsupportedCall = call('tool-unsupported') + presenter.started(unsupportedCall) + await completeWithImage(presenter, unsupportedCall, await createPng(120, 80), 'image/webp') + + const oversizedCall = call('tool-oversized') + presenter.started(oversizedCall) + presenter.completed(oversizedCall, { + toolCallId: oversizedCall.toolCallId, + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'A'.repeat(Math.ceil(((16 * 1024 * 1024 + 1) * 4) / 3)) + } + ], + isError: false + }) + await new Promise((resolve) => setImmediate(resolve)) + + const overDimensionCall = call('tool-dimension') + presenter.started(overDimensionCall) + await completeWithImage(presenter, overDimensionCall, await createPng(8193, 1)) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(2) + }) + + it('drops intermediate transforms and publishes only the latest overlapping snapshot', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-first') + const intermediate = call('tool-intermediate') + const latest = call('tool-latest') + + presenter.started(first) + await completeWithImage(presenter, first, await createPng(480, 300, 20)) + presenter.started(intermediate) + await completeWithImage(presenter, intermediate, await createPng(480, 300, 100)) + presenter.started(latest) + await completeWithImage(presenter, latest, await createPng(480, 300, 220)) + + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + const frame = windowPresenter.sendToWindow.mock.calls + .map( + ([, , envelope]) => + envelope as { name: string; payload: { data: Uint8Array; sequence: number } } + ) + .find((envelope) => envelope.name === 'computerUse.preview.frame') + expect(frame).toBeDefined() + const stats = await sharp(frame!.payload.data).stats() + + expect(frame!.payload.sequence).toBe(1) + expect(stats.channels[0].mean).toBeGreaterThan(180) + }) + + it('retains the last valid frame after failed and aborted snapshot calls', async () => { + const { presenter, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const valid = call('tool-valid') + presenter.started(valid) + await completeWithImage(presenter, valid, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + ) + + const failed = call('tool-failed') + presenter.started(failed) + presenter.failed(failed, new Error('snapshot failed')) + const aborted = call('tool-aborted') + presenter.started(aborted) + presenter.failed(aborted, new DOMException('Aborted', 'AbortError')) + await new Promise((resolve) => setImmediate(resolve)) + + expect( + windowPresenter.sendToWindow.mock.calls.filter( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toHaveLength(1) + }) + + it('dismisses only the current run and permits a later run', async () => { + const { presenter, coordinator, windowPresenter } = await setup() + await presenter.setPreviewMode('session-1', 'eligible', 7) + const first = call('tool-1') + presenter.started(first) + + expect(presenter.dismissPreview('session-1', 'run-1')).toBe(true) + await completeWithImage(presenter, first, await createPng()) + await new Promise((resolve) => setImmediate(resolve)) + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(false) + + const second = call('tool-2', { runId: 'run-2' }) + presenter.started(second) + await completeWithImage(presenter, second, await createPng(320, 200)) + await vi.waitFor(() => + expect( + windowPresenter.sendToWindow.mock.calls.some( + ([, , envelope]) => (envelope as { name?: string }).name === 'computerUse.preview.frame' + ) + ).toBe(true) + ) + expect(coordinator.dismiss).toHaveBeenCalledWith({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + }) +}) diff --git a/test/main/desktop/preview/AgentPreviewCoordinator.test.ts b/test/main/desktop/preview/AgentPreviewCoordinator.test.ts new file mode 100644 index 000000000..e282d22e3 --- /dev/null +++ b/test/main/desktop/preview/AgentPreviewCoordinator.test.ts @@ -0,0 +1,344 @@ +import { EventEmitter } from 'node:events' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +class MockBrowserWindow extends EventEmitter { + readonly id = 7 + destroyed = false + visible = true + focused = true + minimized = false + + isDestroyed() { + return this.destroyed + } + + isVisible() { + return this.visible + } + + isFocused() { + return this.focused + } + + isMinimized() { + return this.minimized + } + + getTitle() { + return 'DeepChat' + } + + getContentBounds() { + return { x: 10, y: 20, width: 800, height: 600 } + } + + getNativeWindowHandle() { + return Buffer.alloc(process.platform === 'linux' ? 4 : 8) + } +} + +describe('AgentPreviewCoordinator', () => { + beforeEach(() => { + vi.resetModules() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + const setup = async (startResult = true) => { + const screen = new EventEmitter() + const overlay = Object.assign(new EventEmitter(), { + start: vi.fn(() => startResult), + stop: vi.fn(() => true), + attachHost: vi.fn(() => true), + detachHost: vi.fn(() => true), + setVisible: vi.fn(() => true), + setMaxSize: vi.fn(() => true), + pushImage: vi.fn(() => true), + setActiveSession: vi.fn(() => true), + removeImage: vi.fn(() => true) + }) + + vi.doMock('electron', () => ({ + app: { + getName: () => 'DeepChat' + }, + screen + })) + vi.doMock('@zerob13/nativekit', () => ({ overlay })) + vi.doMock('@shared/logger', () => ({ + default: { + info: vi.fn(), + warn: vi.fn() + } + })) + + const { AgentPreviewCoordinator } = await import('@/desktop/preview/AgentPreviewCoordinator') + const browserAction = vi.fn() + const computerAction = vi.fn() + const coordinator = new AgentPreviewCoordinator() + coordinator.register('browser', browserAction) + coordinator.register('computer-use', computerAction) + + const browserTarget = (epoch = 3) => { + const claimSequence = coordinator.claim({ + source: 'browser', + sessionId: 'session-1', + runId: 'run-1' + }) + return { + source: 'browser' as const, + windowId: 7, + sessionId: 'session-1', + runId: 'run-1', + epoch, + claimSequence + } + } + + return { + coordinator, + browserAction, + computerAction, + browserTarget, + overlay, + screen, + host: new MockBrowserWindow() + } + } + + it('pushes a current JPEG before showing the native panel', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + const target = browserTarget() + + await expect(coordinator.initialize()).resolves.toBe(true) + expect(overlay.start).toHaveBeenCalledWith({ + toolbar: { + style: 'dark', + buttons: [ + { + id: 'open-panel', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Open in side panel' + }, + { + id: 'close', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Close' + } + ] + } + }) + expect(overlay.setMaxSize).toHaveBeenCalledWith(360) + expect(overlay.setVisible).toHaveBeenCalledWith(false) + expect(coordinator.prepare(target, host as never)).toBe('native-overlay') + expect(overlay.setVisible).toHaveBeenLastCalledWith(false) + + expect(coordinator.present(target, Buffer.from('frame'))).toBe(true) + + expect(overlay.pushImage).toHaveBeenCalledWith({ + hostId: 'chat-window:7', + presentationId: 'agent-preview:browser:7:session-1', + sessionId: 'agent-preview:browser:session-1', + imageData: `data:image/jpeg;base64,${Buffer.from('frame').toString('base64')}` + }) + expect(overlay.setActiveSession).toHaveBeenCalledWith('agent-preview:browser:session-1') + expect(overlay.setVisible).toHaveBeenLastCalledWith(true) + expect(overlay.pushImage.mock.invocationCallOrder[0]).toBeLessThan( + overlay.setActiveSession.mock.invocationCallOrder[0] + ) + expect(overlay.setActiveSession.mock.invocationCallOrder[0]).toBeLessThan( + overlay.setVisible.mock.invocationCallOrder.at(-1)! + ) + + coordinator.shutdown() + }) + + it('refreshes the same presentation without resetting native placement state', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + const target = browserTarget() + + await coordinator.initialize() + coordinator.prepare(target, host as never) + coordinator.present(target, Buffer.from('frame-1')) + coordinator.prepare(target, host as never) + expect(overlay.attachHost).toHaveBeenCalledOnce() + + overlay.attachHost.mockClear() + overlay.detachHost.mockClear() + overlay.removeImage.mockClear() + overlay.setActiveSession.mockClear() + overlay.setVisible.mockClear() + + expect(coordinator.present(target, Buffer.from('frame-2'))).toBe(true) + + expect(overlay.pushImage).toHaveBeenCalledTimes(2) + expect(overlay.pushImage).toHaveBeenLastCalledWith( + expect.objectContaining({ + presentationId: 'agent-preview:browser:7:session-1', + sessionId: 'agent-preview:browser:session-1' + }) + ) + expect(overlay.attachHost).not.toHaveBeenCalled() + expect(overlay.detachHost).not.toHaveBeenCalled() + expect(overlay.removeImage).not.toHaveBeenCalled() + expect(overlay.setActiveSession).not.toHaveBeenCalled() + expect(overlay.setVisible).not.toHaveBeenCalled() + + coordinator.shutdown() + }) + + it('switches to the close-only Computer Use toolbar and ignores activation', async () => { + const { coordinator, browserAction, browserTarget, computerAction, overlay, host } = + await setup() + const browser = browserTarget(4) + await coordinator.initialize() + coordinator.prepare(browser, host as never) + coordinator.present(browser, Buffer.from('browser')) + + const claimSequence = coordinator.claim({ + source: 'computer-use', + sessionId: 'session-1', + runId: 'run-1' + }) + const computer = { + source: 'computer-use' as const, + windowId: 7, + sessionId: 'session-1', + runId: 'run-1', + epoch: 5, + claimSequence + } + + expect(coordinator.prepare(computer, host as never)).toBe('native-overlay') + expect(browserAction).toHaveBeenCalledWith('superseded', browser) + expect(overlay.start).toHaveBeenLastCalledWith({ + toolbar: { + style: 'dark', + buttons: [ + { + id: 'close', + imageData: expect.stringMatching(/^data:image\/png;base64,/), + tooltip: 'Close' + } + ] + } + }) + + overlay.emit('activate') + overlay.emit('control', 'open-panel') + expect(computerAction).not.toHaveBeenCalled() + + overlay.emit('control', 'close') + expect(computerAction).toHaveBeenCalledWith('dismiss', computer) + + coordinator.shutdown() + }) + + it('keeps host movement native and maps Browser controls to the current target', async () => { + const { coordinator, browserAction, browserTarget, overlay, host, screen } = await setup() + const target = browserTarget(4) + await coordinator.initialize() + coordinator.prepare(target, host as never) + coordinator.present(target, Buffer.from('frame')) + + host.focused = false + host.emit('blur') + expect(overlay.setVisible).toHaveBeenLastCalledWith(false) + + host.emit('move') + screen.emit('display-metrics-changed') + await vi.advanceTimersByTimeAsync(50) + expect(overlay.attachHost).toHaveBeenCalledTimes(2) + + overlay.emit('activate') + overlay.emit('control', 'open-panel') + overlay.emit('control', 'unknown') + overlay.emit('control', 'close') + expect(browserAction).toHaveBeenNthCalledWith(1, 'activate', target) + expect(browserAction).toHaveBeenNthCalledWith(2, 'activate', target) + expect(browserAction).toHaveBeenNthCalledWith(3, 'dismiss', target) + expect(browserAction).toHaveBeenCalledTimes(3) + + coordinator.shutdown() + }) + + it('returns Canvas fallback when native startup fails', async () => { + const { coordinator, browserTarget, overlay, host } = await setup(false) + const target = browserTarget(1) + + await expect(coordinator.initialize()).resolves.toBe(false) + expect(coordinator.prepare(target, host as never)).toBe('renderer-canvas') + expect(overlay.attachHost).not.toHaveBeenCalled() + }) + + it('falls back permanently when the first real host cannot attach', async () => { + const { coordinator, browserTarget, overlay, host } = await setup() + overlay.attachHost.mockReturnValue(false) + await coordinator.initialize() + + expect(coordinator.prepare(browserTarget(1), host as never)).toBe('renderer-canvas') + expect(coordinator.isAvailable()).toBe(false) + await expect(coordinator.initialize()).resolves.toBe(false) + expect(overlay.stop).toHaveBeenCalledTimes(1) + }) + + it('keeps run dismissal scoped to the current source owner', async () => { + const { coordinator, browserTarget, host } = await setup(false) + const browser = browserTarget() + await coordinator.initialize() + coordinator.prepare(browser, host as never) + + expect( + coordinator.dismiss({ + source: 'computer-use', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(false) + expect(coordinator.isCurrent(browser)).toBe(true) + expect( + coordinator.dismiss({ + source: 'browser', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(true) + expect(coordinator.isCurrent(browser)).toBe(false) + }) + + it('uses the matching session claim when another session owns the visible surface', async () => { + const { coordinator, browserTarget, host } = await setup(false) + const browser = browserTarget() + await coordinator.initialize() + coordinator.prepare(browser, host as never) + + const computerClaimSequence = coordinator.claim({ + source: 'computer-use', + sessionId: 'session-2', + runId: 'run-2' + }) + coordinator.prepare( + { + source: 'computer-use', + windowId: 7, + sessionId: 'session-2', + runId: 'run-2', + epoch: 1, + claimSequence: computerClaimSequence + }, + host as never + ) + + expect( + coordinator.dismiss({ + source: 'browser', + sessionId: browser.sessionId, + runId: browser.runId + }) + ).toBe(true) + }) +}) diff --git a/test/main/mcp/toolManager.test.ts b/test/main/mcp/toolManager.test.ts index fd746e2c8..33dc3d36e 100644 --- a/test/main/mcp/toolManager.test.ts +++ b/test/main/mcp/toolManager.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { ToolManager } from '@/mcp/toolManager' +import { ToolManager, type ComputerUsePreviewObserver } from '@/mcp/toolManager' +import * as toolPolicyStore from '@/plugin/toolPolicyStore' function deferred() { let resolve!: (value: T) => void @@ -77,13 +78,18 @@ describe('ToolManager', () => { } } - function createToolManager(providerSettings: unknown, serverManager: unknown) { + function createToolManager( + providerSettings: unknown, + serverManager: unknown, + computerUsePreviewObserver?: ComputerUsePreviewObserver + ) { return new ToolManager( providerSettings as never, { getLanguage: vi.fn().mockReturnValue('en-US') }, providerSettings as never, serverManager as never, - vi.fn() + vi.fn(), + computerUsePreviewObserver ) } @@ -688,6 +694,543 @@ describe('ToolManager', () => { expect(providerSettings.getAgentMcpSelections).not.toHaveBeenCalled() }) + it('observes trusted CUA snapshots with run metadata without changing tool arguments', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + client.callTool.mockResolvedValue({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ], + isError: false + }) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([client]), + observer + ) + const toolCall = { + id: 'cua-state-1', + type: 'function' as const, + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + } + + const result = await manager.callTool(toolCall, { runId: 'run-1' }) + + expect(client.callTool).toHaveBeenCalledWith('get_window_state', { + pid: 12, + window_id: 34 + }) + expect(observer.started).toHaveBeenCalledWith({ + conversationId: 'session-1', + runId: 'run-1', + toolCallId: 'cua-state-1', + toolName: 'get_window_state', + args: { + pid: 12, + window_id: 34 + }, + source: { + serverName: 'cua-driver', + ownerPluginId: 'com.deepchat.plugins.cua' + } + }) + expect(observer.completed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-state-1' }), + result + ) + expect(observer.failed).not.toHaveBeenCalled() + expect(JSON.stringify(client.callTool.mock.calls)).not.toContain('run-1') + }) + + it('refreshes PiP after a trusted click without exposing or awaiting the private snapshot', async () => { + let snapshotPolicy: 'allow' | 'ask' = 'allow' + vi.spyOn(toolPolicyStore, 'getPluginToolPolicy').mockImplementation((_serverId, toolName) => + toolName === 'get_window_state' ? snapshotPolicy : null + ) + const client = createClient( + 'cua-driver', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + }, + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const privateSnapshots = [ + deferred<{ + content: Array<{ type: string; mimeType: string; data: string }> + isError: boolean + }>(), + deferred<{ + content: Array<{ type: string; mimeType: string; data: string }> + isError: boolean + }>() + ] + let privateSnapshotIndex = 0 + client.callTool.mockImplementation((toolName: string) => { + if (toolName === 'click') { + return Promise.resolve({ + content: 'clicked', + isError: false + }) + } + return privateSnapshots[privateSnapshotIndex++].promise + }) + const publishEvent = vi.fn() + const observer = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const providerSettings = createProviderSettings('cua-driver') + const manager = new ToolManager( + providerSettings as never, + { getLanguage: vi.fn().mockReturnValue('en-US') }, + providerSettings as never, + createServerManager([client]) as never, + publishEvent, + observer + ) + + const result = await manager.callTool( + { + id: 'cua-click-1', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result).toEqual({ + toolCallId: 'cua-click-1', + content: 'clicked', + isError: false + }) + expect(client.callTool).toHaveBeenNthCalledWith(1, 'click', { + pid: 12, + window_id: 34, + x: 10, + y: 20 + }) + expect(client.callTool).toHaveBeenNthCalledWith(2, 'get_window_state', { + pid: 12, + window_id: 34 + }) + expect(observer.shouldCaptureAfterClick).toHaveBeenCalledWith( + expect.objectContaining({ + toolCallId: 'cua-click-1', + toolName: 'click', + conversationId: 'session-1', + runId: 'run-1' + }) + ) + expect(observer.started).toHaveBeenCalledWith( + expect.objectContaining({ + toolCallId: 'cua-click-1:pip-snapshot', + toolName: 'get_window_state', + args: { + pid: 12, + window_id: 34 + } + }) + ) + expect(observer.completed).not.toHaveBeenCalled() + expect(publishEvent).toHaveBeenCalledOnce() + expect(publishEvent).toHaveBeenCalledWith( + 'mcp.toolCall.result', + expect.objectContaining({ + functionName: 'click', + content: 'clicked' + }) + ) + + privateSnapshots[0].resolve({ + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ], + isError: false + }) + await vi.waitFor(() => expect(observer.completed).toHaveBeenCalledOnce()) + + expect(observer.completed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-click-1:pip-snapshot' }), + expect.objectContaining({ + toolCallId: 'cua-click-1:pip-snapshot', + content: [ + { + type: 'image', + mimeType: 'image/png', + data: 'aW1hZ2U=' + } + ] + }) + ) + expect(publishEvent).toHaveBeenCalledOnce() + expect(observer.failed).not.toHaveBeenCalled() + + const laterResult = await manager.callTool( + { + id: 'cua-click-2', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":30,"y":40}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + expect(laterResult.content).toBe('clicked') + expect(publishEvent).toHaveBeenCalledTimes(2) + + const privateFailure = new Error('private snapshot failed') + privateSnapshots[1].reject(privateFailure) + await vi.waitFor(() => expect(observer.failed).toHaveBeenCalledOnce()) + + expect(observer.failed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-click-2:pip-snapshot' }), + privateFailure + ) + expect(publishEvent).toHaveBeenCalledTimes(2) + + snapshotPolicy = 'ask' + const guardedResult = await manager.callTool( + { + id: 'cua-click-3', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":50,"y":60}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(guardedResult.content).toBe('clicked') + expect(client.callTool).toHaveBeenCalledTimes(5) + expect(observer.shouldCaptureAfterClick).toHaveBeenCalledTimes(2) + expect(observer.started).toHaveBeenCalledTimes(2) + }) + + it('does not privately snapshot failed, invalid-target, or untrusted clicks', async () => { + const trustedClient = createClient( + 'cua-driver', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + trustedClient.callTool + .mockResolvedValueOnce({ content: 'click failed', isError: true }) + .mockResolvedValueOnce({ content: 'clicked', isError: false }) + const trustedObserver = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const trustedManager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([trustedClient]), + trustedObserver + ) + + await trustedManager.callTool( + { + id: 'failed-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + await trustedManager.callTool( + { + id: 'missing-window-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(trustedClient.callTool).toHaveBeenCalledTimes(2) + expect(trustedObserver.shouldCaptureAfterClick).not.toHaveBeenCalled() + expect(trustedObserver.started).not.toHaveBeenCalled() + + const untrustedClient = createClient( + 'manual-cua', + [ + { + name: 'click', + description: 'Click the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'manual', + sourceId: 'com.deepchat.plugins.cua' + } + ) + const untrustedObserver = { + shouldCaptureAfterClick: vi.fn(() => true), + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const untrustedManager = createToolManager( + createProviderSettings('manual-cua'), + createServerManager([untrustedClient]), + untrustedObserver + ) + + await untrustedManager.callTool( + { + id: 'untrusted-click', + type: 'function', + function: { + name: 'click', + arguments: '{"pid":12,"window_id":34,"x":10,"y":20}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(untrustedClient.callTool).toHaveBeenCalledOnce() + expect(untrustedObserver.shouldCaptureAfterClick).not.toHaveBeenCalled() + expect(untrustedObserver.started).not.toHaveBeenCalled() + }) + + it('does not observe a CUA permission response before actual invocation', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const providerSettings = createProviderSettings('cua-driver') + providerSettings.getMcpServers.mockResolvedValue({ + 'cua-driver': { + autoApprove: [] + } + }) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager(providerSettings, createServerManager([client]), observer) + + const result = await manager.callTool( + { + id: 'cua-state-permission', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result.requiresPermission).toBe(true) + expect(client.callTool).not.toHaveBeenCalled() + expect(observer.started).not.toHaveBeenCalled() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).not.toHaveBeenCalled() + }) + + it('does not trust a non-plugin server that spoofs the CUA source id', async () => { + const client = createClient( + 'spoofed-cua', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'manual', + sourceId: 'com.deepchat.plugins.cua' + } + ) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('spoofed-cua'), + createServerManager([client]), + observer + ) + + await manager.callTool( + { + id: 'spoofed-cua-state', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(client.callTool).toHaveBeenCalledOnce() + expect(observer.started).not.toHaveBeenCalled() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).not.toHaveBeenCalled() + }) + + it('reports a failed trusted CUA invocation without changing the tool error response', async () => { + const client = createClient( + 'cua-driver', + [ + { + name: 'get_window_state', + description: 'Read the current window', + inputSchema: { + properties: {}, + required: [] + } + } + ], + { + source: 'plugin', + ownerPluginId: 'com.deepchat.plugins.cua' + } + ) + const failure = new Error('driver failed') + client.callTool.mockRejectedValue(failure) + const observer = { + started: vi.fn(), + completed: vi.fn(), + failed: vi.fn() + } + const manager = createToolManager( + createProviderSettings('cua-driver'), + createServerManager([client]), + observer + ) + + const result = await manager.callTool( + { + id: 'cua-state-failed', + type: 'function', + function: { + name: 'get_window_state', + arguments: '{"pid":12,"window_id":34}' + }, + conversationId: 'session-1' + }, + { runId: 'run-1' } + ) + + expect(result).toEqual({ + toolCallId: 'cua-state-failed', + content: "Error: Failed to execute tool 'get_window_state': driver failed", + isError: true + }) + expect(observer.started).toHaveBeenCalledOnce() + expect(observer.completed).not.toHaveBeenCalled() + expect(observer.failed).toHaveBeenCalledWith( + expect.objectContaining({ toolCallId: 'cua-state-failed' }), + failure + ) + }) + it('normalizes CUA Windows launch bundle paths before dispatch', async () => { const client = createClient('cua-driver', [], { source: 'plugin', diff --git a/test/main/routes/contracts.test.ts b/test/main/routes/contracts.test.ts index ad9f5d2e9..5a5a96285 100644 --- a/test/main/routes/contracts.test.ts +++ b/test/main/routes/contracts.test.ts @@ -74,7 +74,10 @@ describe('main kernel contracts', () => { 'acpTerminal.kill', 'browser.attachCurrentWindow', 'browser.clearSandboxData', + 'browser.dismissPreview', 'browser.setPreviewMode', + 'computerUse.dismissPreview', + 'computerUse.setPreviewMode', 'databaseSecurity.repairSchema', 'debug.createMockChatSession', 'config.addManualAcpAgent', @@ -1788,7 +1791,10 @@ describe('main kernel contracts', () => { 'browser.open.requested', 'browser.preview.action', 'browser.preview.frame', + 'browser.preview.surface.changed', 'browser.status.changed', + 'computerUse.preview.frame', + 'computerUse.preview.surface.changed', 'chat.plan.updated', 'chat.stream.completed', 'chat.stream.failed', @@ -1927,6 +1933,39 @@ describe('main kernel contracts', () => { ).toBe(false) }) + it('validates bounded Computer Use preview frames with a target epoch', () => { + const payload = { + sessionId: 'session-1', + runId: 'run-1', + epoch: 2, + sequence: 3, + width: 480, + height: 300, + mimeType: 'image/jpeg', + timestamp: Date.now() + } as const + + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint8Array([1, 2, 3]) + }).success + ).toBe(true) + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + epoch: -1, + data: new Uint8Array([1, 2, 3]) + }).success + ).toBe(false) + expect( + DEEPCHAT_EVENT_CATALOG['computerUse.preview.frame'].payload.safeParse({ + ...payload, + data: new Uint8Array(512 * 1024 + 1) + }).success + ).toBe(false) + }) + it('validates browser preview actions against an exact target', () => { expect( DEEPCHAT_EVENT_CATALOG['browser.preview.action'].payload.safeParse({ diff --git a/test/main/routes/dispatcher.test.ts b/test/main/routes/dispatcher.test.ts index fdd49f1b3..270d54715 100644 --- a/test/main/routes/dispatcher.test.ts +++ b/test/main/routes/dispatcher.test.ts @@ -446,7 +446,7 @@ function createRuntime() { activate: vi.fn().mockResolvedValue(undefined), deactivate: vi.fn().mockResolvedValue(undefined), getActive: vi.fn().mockResolvedValue(null), - getActiveId: vi.fn(() => null) + getActiveId: vi.fn((): string | null => null) } const sessionTurnPort = { sendMessage: vi.fn().mockResolvedValue({ @@ -1260,12 +1260,22 @@ function createRuntime() { attachSessionBrowser: vi.fn().mockResolvedValue(true), updateSessionBrowserBounds: vi.fn().mockResolvedValue(undefined), detachSessionBrowser: vi.fn().mockResolvedValue(undefined), + setPreviewMode: vi.fn().mockResolvedValue({ updated: true, surface: 'renderer-canvas' }), + dismissPreview: vi.fn(() => true), destroySessionBrowser: vi.fn().mockResolvedValue(undefined), goBack: vi.fn().mockResolvedValue(undefined), goForward: vi.fn().mockResolvedValue(undefined), reload: vi.fn().mockResolvedValue(undefined), clearSandboxData: vi.fn().mockResolvedValue(undefined) } as unknown as IYoBrowserPresenter + const computerUsePreviewPresenter = { + setPreviewMode: vi.fn(async (_sessionId: string, mode: string) => ({ + updated: true, + surface: mode === 'stopped' ? 'none' : 'renderer-canvas' + })), + dismissPreview: vi.fn(() => true), + shutdown: vi.fn() + } const tabPresenter = { captureTabArea: vi.fn().mockResolvedValue('data:image/png;base64,capture'), @@ -1552,6 +1562,8 @@ function createRuntime() { windowPresenter, shortcutPresenter, browserPresenter: yoBrowserPresenter, + computerUsePreviewPresenter, + desktopSessionBinding, tabPresenter, dialogService: dialogService as unknown as DialogServicePort, settings: desktopSettings as never, @@ -1799,6 +1811,7 @@ function createRuntime() { knowledgeService, workspaceService, yoBrowserPresenter, + computerUsePreviewPresenter, tabPresenter, cronJobs, usageStatsService, @@ -5902,6 +5915,66 @@ describe('dispatchDeepchatRoute', () => { expect(clearSandboxResult).toEqual({ cleared: true }) }) + it('scopes Computer Use preview routes to the active sender session', async () => { + const { runtime, computerUsePreviewPresenter, desktopSessionBinding, yoBrowserPresenter } = + createRuntime() + const context = { webContentsId: 88, windowId: 3 } + desktopSessionBinding.getActiveId.mockReturnValue('session-1') + + const eligible = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'eligible' }, + context + ) + const browserDismissed = await dispatchDeepchatRoute( + runtime, + 'browser.dismissPreview', + { sessionId: 'session-1', runId: 'run-1' }, + context + ) + const dismissed = await dispatchDeepchatRoute( + runtime, + 'computerUse.dismissPreview', + { sessionId: 'session-1', runId: 'run-1' }, + context + ) + + desktopSessionBinding.getActiveId.mockReturnValue('session-2') + const rejected = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'eligible' }, + context + ) + const cleanup = await dispatchDeepchatRoute( + runtime, + 'computerUse.setPreviewMode', + { sessionId: 'session-1', mode: 'stopped' }, + context + ) + + expect(eligible).toEqual({ updated: true, surface: 'renderer-canvas' }) + expect(computerUsePreviewPresenter.setPreviewMode).toHaveBeenNthCalledWith( + 1, + 'session-1', + 'eligible', + 3 + ) + expect(browserDismissed).toEqual({ dismissed: true }) + expect(yoBrowserPresenter.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(dismissed).toEqual({ dismissed: true }) + expect(computerUsePreviewPresenter.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(rejected).toEqual({ updated: false, surface: 'none' }) + expect(cleanup).toEqual({ updated: true, surface: 'none' }) + expect(computerUsePreviewPresenter.setPreviewMode).toHaveBeenNthCalledWith( + 2, + 'session-1', + 'stopped', + 3 + ) + }) + it('dispatches phase3 tab routes through the renderer tab adapter', async () => { const { runtime, tabPresenter } = createRuntime() diff --git a/test/main/tool/toolService.test.ts b/test/main/tool/toolService.test.ts index 02302d0ef..5bb681abc 100644 --- a/test/main/tool/toolService.test.ts +++ b/test/main/tool/toolService.test.ts @@ -609,16 +609,23 @@ describe('ToolService', () => { expect((toolService as any).agentToolManager).toBe(runtimeManager) expect(runtimeManager.agentWorkspacePath).toBe('C:\\runtime-workspace') - await toolService.callTool({ - id: 'tool-1', - type: 'function', - function: { name: 'mcp_only', arguments: '{}' }, - conversationId: 'conv-1' - }) + await toolService.callTool( + { + id: 'tool-1', + type: 'function', + function: { name: 'mcp_only', arguments: '{}' }, + conversationId: 'conv-1' + }, + { runId: 'run-1' } + ) expect(mcpService.callTool).toHaveBeenCalledWith( expect.objectContaining({ function: expect.objectContaining({ name: 'mcp_only' }) }), - expect.objectContaining({ agentId: 'agent-1', enabledServerIds: ['mcp-server'] }) + expect.objectContaining({ + agentId: 'agent-1', + enabledServerIds: ['mcp-server'], + runId: 'run-1' + }) ) }) diff --git a/test/renderer/api/clients.test.ts b/test/renderer/api/clients.test.ts index a4c879a4e..921499108 100644 --- a/test/renderer/api/clients.test.ts +++ b/test/renderer/api/clients.test.ts @@ -4,6 +4,7 @@ import type { HooksNotificationsSettings } from '@shared/hooksNotifications' import { createAcpTerminalClient } from '../../../src/renderer/api/AcpTerminalClient' import { createAppRuntimeClient } from '../../../src/renderer/api/AppRuntimeClient' import { createBrowserClient } from '../../../src/renderer/api/BrowserClient' +import { createComputerUseClient } from '../../../src/renderer/api/ComputerUseClient' import { createChatClient } from '../../../src/renderer/api/ChatClient' import { createConfigClient } from '../../../src/renderer/api/ConfigClient' import { createContextMenuClient } from '../../../src/renderer/api/ContextMenuClient' @@ -877,6 +878,12 @@ describe('renderer api clients', () => { return { updated: true } case 'browser.setPreviewMode': return { updated: true, surface: 'renderer-canvas' } + case 'browser.dismissPreview': + return { dismissed: true } + case 'computerUse.setPreviewMode': + return { updated: true, surface: 'renderer-canvas' } + case 'computerUse.dismissPreview': + return { dismissed: true } case 'browser.clearSandboxData': return { cleared: true } case 'browser.import.scan': @@ -2236,6 +2243,39 @@ describe('renderer api clients', () => { }) }) + it('routes Browser and Computer Use preview dismissal through typed registry names', async () => { + const bridge = createBridge() + const browserClient = createBrowserClient(bridge) + const computerUseClient = createComputerUseClient(bridge) + + await expect(browserClient.dismissPreview('session-1', 'run-1')).resolves.toBe(true) + await expect(computerUseClient.dismissPreview('session-1', 'run-1')).resolves.toBe(true) + + expect(bridge.invoke).toHaveBeenNthCalledWith(1, 'browser.dismissPreview', { + sessionId: 'session-1', + runId: 'run-1' + }) + expect(bridge.invoke).toHaveBeenNthCalledWith(2, 'computerUse.dismissPreview', { + sessionId: 'session-1', + runId: 'run-1' + }) + }) + + it('routes Computer Use preview mode through the shared registry name', async () => { + const bridge = createBridge() + const client = createComputerUseClient(bridge) + + await expect(client.setPreviewMode('session-1', 'eligible')).resolves.toEqual({ + updated: true, + surface: 'renderer-canvas' + }) + + expect(bridge.invoke).toHaveBeenCalledWith('computerUse.setPreviewMode', { + sessionId: 'session-1', + mode: 'eligible' + }) + }) + it('routes browser website-data import through typed registry names', async () => { const bridge = createBridge() const browserClient = createBrowserClient(bridge) @@ -2912,4 +2952,21 @@ describe('renderer api clients', () => { expect(bridge.on).toHaveBeenCalledWith('browser.preview.action', listener) }) + + it('subscribes to Computer Use preview frames and surface changes', () => { + const bridge = createBridge() + const client = createComputerUseClient(bridge) + const frameListener = vi.fn() + const surfaceListener = vi.fn() + + client.onPreviewFrame(frameListener) + client.onPreviewSurfaceChanged(surfaceListener) + + expect(bridge.on).toHaveBeenNthCalledWith(1, 'computerUse.preview.frame', frameListener) + expect(bridge.on).toHaveBeenNthCalledWith( + 2, + 'computerUse.preview.surface.changed', + surfaceListener + ) + }) }) diff --git a/test/renderer/components/AgentBrowserPiP.test.ts b/test/renderer/components/AgentBrowserPiP.test.ts index 7bb689614..99757ff5c 100644 --- a/test/renderer/components/AgentBrowserPiP.test.ts +++ b/test/renderer/components/AgentBrowserPiP.test.ts @@ -66,6 +66,15 @@ const setup = async ( runId: string }) => void) | null = null + let previewSurfaceChangedHandler: + | ((payload: { + windowId: number + sessionId: string + runId: string + surface: 'native-overlay' | 'renderer-canvas' | 'none' + version: number + }) => void) + | null = null const status = createStatus() const browserClient = { getStatus: vi.fn(async () => status), @@ -73,6 +82,7 @@ const setup = async ( updated: true, surface: options.surface ?? 'renderer-canvas' })), + dismissPreview: vi.fn(async () => true), onOpenRequestedForCurrentWindow: vi.fn(() => vi.fn()), onStatusChanged: vi.fn( (handler: (payload: { sessionId: string; status: YoBrowserStatus | null }) => void) => { @@ -88,6 +98,10 @@ const setup = async ( onPreviewAction: vi.fn((handler: NonNullable) => { previewActionHandler = handler return vi.fn() + }), + onPreviewSurfaceChanged: vi.fn((handler: NonNullable) => { + previewSurfaceChangedHandler = handler + return vi.fn() }) } const windowClient = { @@ -142,6 +156,7 @@ const setup = async ( emitStatus: statusChangedHandler!, emitPreviewFrame: previewFrameHandler!, emitPreviewAction: previewActionHandler!, + emitPreviewSurfaceChanged: previewSurfaceChangedHandler!, emitWindowState: windowStateHandler! } } @@ -179,12 +194,14 @@ describe('AgentBrowserPiP', () => { }) it('stays dismissed for the current run', async () => { - const { wrapper } = await setup() + const { wrapper, browserClient } = await setup() + browserClient.dismissPreview.mockRejectedValueOnce(new Error('dismiss failed')) await wrapper.get('[aria-label="common.close"]').trigger('click') - await nextTick() + await flushPromises() expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + expect(browserClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') }) it('renders no Canvas surface and handles native panel actions', async () => { @@ -223,7 +240,7 @@ describe('AgentBrowserPiP', () => { await flushPromises() expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) - expect(browserClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'rendering', 'run-1') + expect(browserClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') }) it('ignores native actions targeted at another window', async () => { @@ -307,6 +324,52 @@ describe('AgentBrowserPiP', () => { expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) }) + it('clears retained Canvas pixels when another preview source supersedes Browser', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitPreviewFrame, emitPreviewSurfaceChanged } = await setup({ wide: true }) + + emitPreviewFrame({ + sessionId: 'session-1', + runId: 'run-1', + sequence: 1, + width: 480, + height: 300, + mimeType: 'image/jpeg', + data: new Uint8Array([1, 2, 3]), + timestamp: Date.now() + }) + await flushPromises() + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(false) + + emitPreviewSurfaceChanged({ + windowId: 1, + sessionId: 'session-1', + runId: 'run-1', + surface: 'none', + version: Date.now() + }) + await nextTick() + + expect(wrapper.find('[data-testid="agent-browser-pip"]').exists()).toBe(false) + + emitPreviewSurfaceChanged({ + windowId: 1, + sessionId: 'session-1', + runId: 'run-1', + surface: 'renderer-canvas', + version: Date.now() + }) + await nextTick() + + expect(wrapper.find('[data-testid="agent-browser-pip-placeholder"]').exists()).toBe(true) + }) + it('accepts a reset frame sequence when the Agent run changes', async () => { const drawImage = vi.fn() vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ drawImage } as never) diff --git a/test/renderer/components/AgentComputerUsePiP.test.ts b/test/renderer/components/AgentComputerUsePiP.test.ts new file mode 100644 index 000000000..44b46a2c4 --- /dev/null +++ b/test/renderer/components/AgentComputerUsePiP.test.ts @@ -0,0 +1,423 @@ +import { defineComponent, nextTick, reactive } from 'vue' +import { flushPromises, mount } from '@vue/test-utils' +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { + ComputerUsePreviewMode, + ComputerUsePreviewModeResult +} from '@shared/types/computerUse' + +const mountedWrappers: Array<{ unmount: () => void }> = [] + +const setup = async ( + surface: 'native-overlay' | 'renderer-canvas' = 'renderer-canvas', + setPreviewMode?: ( + sessionId: string, + mode: ComputerUsePreviewMode + ) => Promise +) => { + vi.resetModules() + vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue({ + x: 0, + y: 0, + top: 0, + left: 0, + right: 800, + bottom: 600, + width: 800, + height: 600, + toJSON: () => ({}) + }) + + type SurfacePayload = { + windowId: number + sessionId: string + runId: string + epoch: number + surface: 'native-overlay' | 'renderer-canvas' | 'none' + version: number + } + type FramePayload = { + sessionId: string + runId: string + epoch: number + sequence: number + width: number + height: number + mimeType: 'image/jpeg' + data: Uint8Array + timestamp: number + } + let surfaceHandler: ((payload: SurfacePayload) => void) | null = null + let frameHandler: ((payload: FramePayload) => void) | null = null + let windowStateHandler: + | ((payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => void) + | null = null + const stopFrameSubscription = vi.fn() + const stopSurfaceSubscription = vi.fn() + const stopWindowStateSubscription = vi.fn() + const computerUseClient = { + setPreviewMode: vi.fn( + setPreviewMode ?? + (async (_sessionId: string, mode: 'eligible' | 'suspended' | 'stopped') => ({ + updated: true, + surface: mode === 'eligible' ? surface : ('none' as const) + })) + ), + dismissPreview: vi.fn(async () => true), + onPreviewSurfaceChanged: vi.fn((handler: (payload: SurfacePayload) => void) => { + surfaceHandler = handler + return stopSurfaceSubscription + }), + onPreviewFrame: vi.fn((handler: (payload: FramePayload) => void) => { + frameHandler = handler + return stopFrameSubscription + }) + } + const windowClient = { + getCurrentState: vi.fn(async () => ({ windowId: 1, exists: true, isFocused: true })), + onCurrentStateChanged: vi.fn( + ( + handler: (payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => void + ) => { + windowStateHandler = handler + return stopWindowStateSubscription + } + ) + } + const sessionStore = reactive({ + sessions: [{ id: 'session-1', status: 'working' }] + }) + + vi.doMock('vue-i18n', () => ({ useI18n: () => ({ t: (key: string) => key }) })) + vi.doMock('@iconify/vue', () => ({ + Icon: defineComponent({ name: 'Icon', template: '' }) + })) + vi.doMock('@api/ComputerUseClient', () => ({ + createComputerUseClient: () => computerUseClient + })) + vi.doMock('@api/WindowClient', () => ({ createWindowClient: () => windowClient })) + vi.doMock('@/stores/ui/session', () => ({ useSessionStore: () => sessionStore })) + + const AgentComputerUsePiP = (await import('@/components/computerUse/AgentComputerUsePiP.vue')) + .default + const wrapper = mount(AgentComputerUsePiP, { + props: { sessionId: 'session-1' }, + global: { + stubs: { + Button: defineComponent({ + name: 'Button', + emits: ['click'], + template: '' + }) + } + } + }) + mountedWrappers.push(wrapper) + await flushPromises() + + return { + wrapper, + computerUseClient, + sessionStore, + stopFrameSubscription, + stopSurfaceSubscription, + stopWindowStateSubscription, + emitSurface: (payload: SurfacePayload) => surfaceHandler?.(payload), + emitFrame: (payload: FramePayload) => frameHandler?.(payload), + emitWindowState: (payload: { windowId: number | null; exists: boolean; isFocused: boolean }) => + windowStateHandler?.(payload) + } +} + +const surfacePayload = ( + overrides: Partial<{ + windowId: number + sessionId: string + runId: string + epoch: number + surface: 'native-overlay' | 'renderer-canvas' | 'none' + }> = {} +) => ({ + windowId: overrides.windowId ?? 1, + sessionId: overrides.sessionId ?? 'session-1', + runId: overrides.runId ?? 'run-1', + epoch: overrides.epoch ?? 1, + surface: overrides.surface ?? ('renderer-canvas' as const), + version: Date.now() +}) + +const framePayload = ( + overrides: Partial<{ + sessionId: string + runId: string + epoch: number + sequence: number + width: number + height: number + data: Uint8Array + }> = {} +) => ({ + sessionId: overrides.sessionId ?? 'session-1', + runId: overrides.runId ?? 'run-1', + epoch: overrides.epoch ?? 1, + sequence: overrides.sequence ?? 1, + width: overrides.width ?? 480, + height: overrides.height ?? 300, + mimeType: 'image/jpeg' as const, + data: overrides.data ?? new Uint8Array([1, 2, 3]), + timestamp: Date.now() +}) + +afterEach(async () => { + mountedWrappers.splice(0).forEach((wrapper) => wrapper.unmount()) + await flushPromises() + vi.restoreAllMocks() + vi.unstubAllGlobals() +}) + +describe('AgentComputerUsePiP', () => { + it('keeps the native path headless without subscribing to frame bytes', async () => { + const { wrapper, computerUseClient, emitSurface } = await setup('native-overlay') + + emitSurface(surfacePayload({ surface: 'native-overlay' })) + await nextTick() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.onPreviewFrame).not.toHaveBeenCalled() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'eligible') + }) + + it('shows the first decoded Canvas snapshot with Close as its only control', async () => { + const drawImage = vi.fn() + const bitmapClose = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: bitmapClose })) + ) + const { wrapper, computerUseClient, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + await nextTick() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload()) + await flushPromises() + + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 480, 300) + expect(bitmapClose).toHaveBeenCalledOnce() + expect(pip.attributes('style')).not.toContain('display: none') + expect(pip.findAll('button')).toHaveLength(1) + expect(pip.get('button').attributes('aria-label')).toBe('common.close') + expect(wrapper.find('[aria-label="common.open"]').exists()).toBe(false) + + computerUseClient.dismissPreview.mockRejectedValueOnce(new Error('dismiss failed')) + await pip.get('button').trigger('click') + await flushPromises() + + expect(computerUseClient.dismissPreview).toHaveBeenCalledWith('session-1', 'run-1') + expect(pip.attributes('style')).toContain('display: none') + }) + + it('keeps the current Canvas visible until a newer snapshot finishes decoding', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + let resolveSecondBitmap: ((bitmap: { close: () => void }) => void) | null = null + vi.stubGlobal( + 'createImageBitmap', + vi + .fn() + .mockResolvedValueOnce({ close: vi.fn() }) + .mockImplementationOnce( + () => + new Promise<{ close: () => void }>((resolve) => { + resolveSecondBitmap = resolve + }) + ) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + emitFrame(framePayload({ sequence: 1 })) + await flushPromises() + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + expect(pip.attributes('style')).not.toContain('display: none') + + emitFrame(framePayload({ sequence: 2 })) + await nextTick() + expect(drawImage).toHaveBeenCalledTimes(1) + expect(pip.attributes('style')).not.toContain('display: none') + + resolveSecondBitmap?.({ close: vi.fn() }) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(2) + }) + + it('drops stale target epochs and waits for the first new-target frame', async () => { + const drawImage = vi.fn() + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + + emitSurface(surfacePayload()) + emitFrame(framePayload({ sequence: 5 })) + await flushPromises() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).not.toContain( + 'display: none' + ) + + emitSurface(surfacePayload({ epoch: 2 })) + await nextTick() + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload({ epoch: 1, sequence: 6 })) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(1) + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).toContain( + 'display: none' + ) + + emitFrame(framePayload({ epoch: 2, sequence: 1 })) + await flushPromises() + expect(drawImage).toHaveBeenCalledTimes(2) + expect(wrapper.get('[data-testid="agent-computer-use-pip"]').attributes('style')).not.toContain( + 'display: none' + ) + }) + + it('suspends on blur, releases frame subscriptions, and resumes on focus', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { + wrapper, + computerUseClient, + stopFrameSubscription, + emitSurface, + emitFrame, + emitWindowState + } = await setup() + emitSurface(surfacePayload()) + emitFrame(framePayload()) + await flushPromises() + + emitWindowState({ windowId: 1, exists: true, isFocused: false }) + await flushPromises() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'suspended') + expect(stopFrameSubscription).toHaveBeenCalledOnce() + + emitWindowState({ windowId: 1, exists: true, isFocused: true }) + await flushPromises() + + expect(computerUseClient.setPreviewMode).toHaveBeenLastCalledWith('session-1', 'eligible') + expect(computerUseClient.onPreviewFrame).toHaveBeenCalledTimes(2) + }) + + it('drags the fallback without forwarding input to the target application', async () => { + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({ + drawImage: vi.fn() + } as never) + vi.stubGlobal( + 'createImageBitmap', + vi.fn(async () => ({ close: vi.fn() })) + ) + const { wrapper, emitSurface, emitFrame } = await setup() + emitSurface(surfacePayload()) + emitFrame(framePayload()) + await flushPromises() + const pip = wrapper.get('[data-testid="agent-computer-use-pip"]') + const initialStyle = pip.attributes('style') + + await pip.trigger('pointerdown', { button: 0, pointerId: 1, clientX: 500, clientY: 400 }) + await pip.trigger('pointermove', { pointerId: 1, clientX: 400, clientY: 300 }) + await pip.trigger('pointerup', { pointerId: 1, clientX: 400, clientY: 300 }) + + expect(pip.attributes('style')).not.toBe(initialStyle) + }) + + it('stops the previous session and releases subscriptions on teardown', async () => { + const { + wrapper, + computerUseClient, + sessionStore, + stopFrameSubscription, + stopSurfaceSubscription, + stopWindowStateSubscription, + emitSurface + } = await setup() + sessionStore.sessions.push({ id: 'session-2', status: 'working' }) + emitSurface(surfacePayload()) + await nextTick() + + await wrapper.setProps({ sessionId: 'session-2' }) + await flushPromises() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-1', 'stopped') + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-2', 'eligible') + expect(stopFrameSubscription).toHaveBeenCalledOnce() + + wrapper.unmount() + mountedWrappers.splice(mountedWrappers.indexOf(wrapper), 1) + await flushPromises() + expect(computerUseClient.setPreviewMode).toHaveBeenCalledWith('session-2', 'stopped') + expect(stopSurfaceSubscription).toHaveBeenCalledOnce() + expect(stopWindowStateSubscription).toHaveBeenCalledOnce() + }) + + it('queues teardown behind an in-flight preview mode update', async () => { + let resolveEligible: ((result: ComputerUsePreviewModeResult) => void) | undefined + const setPreviewMode = vi.fn(async (_sessionId: string, mode: ComputerUsePreviewMode) => { + if (mode === 'eligible') { + return await new Promise((resolve) => { + resolveEligible = resolve + }) + } + return { updated: true, surface: 'none' as const } + }) + const { wrapper } = await setup('renderer-canvas', setPreviewMode) + const callsBeforeUnmount = setPreviewMode.mock.calls.length + expect(setPreviewMode).toHaveBeenLastCalledWith('session-1', 'eligible') + + wrapper.unmount() + mountedWrappers.splice(mountedWrappers.indexOf(wrapper), 1) + await nextTick() + + expect(setPreviewMode).toHaveBeenCalledTimes(callsBeforeUnmount) + resolveEligible?.({ updated: true, surface: 'renderer-canvas' }) + await flushPromises() + + expect(setPreviewMode).toHaveBeenCalledTimes(callsBeforeUnmount + 1) + expect(setPreviewMode).toHaveBeenLastCalledWith('session-1', 'stopped') + }) + + it('stops and releases the fallback when the Agent run terminates', async () => { + const { wrapper, computerUseClient, sessionStore, emitSurface } = await setup() + emitSurface(surfacePayload()) + await nextTick() + + sessionStore.sessions[0].status = 'completed' + await flushPromises() + + expect(wrapper.find('[data-testid="agent-computer-use-pip"]').exists()).toBe(false) + expect(computerUseClient.setPreviewMode).toHaveBeenLastCalledWith('session-1', 'stopped') + }) +}) diff --git a/test/renderer/components/ChatTabView.test.ts b/test/renderer/components/ChatTabView.test.ts index e06557ee4..8d1fc8228 100644 --- a/test/renderer/components/ChatTabView.test.ts +++ b/test/renderer/components/ChatTabView.test.ts @@ -176,6 +176,12 @@ const setup = async (options: SetupOptions = {}) => { template: '
' }) })) + vi.doMock('@/components/computerUse/AgentComputerUsePiP.vue', () => ({ + default: defineComponent({ + name: 'AgentComputerUsePiP', + template: '
' + }) + })) vi.doMock('@/pages/AgentWelcomePage.vue', () => ({ default: defineComponent({ name: 'AgentWelcomePage', diff --git a/test/setup.ts b/test/setup.ts index a5ec3d04d..b29cb3dea 100644 --- a/test/setup.ts +++ b/test/setup.ts @@ -24,6 +24,12 @@ function getDefaultDeepchatInvokeResult( return { updated: true } case 'browser.setPreviewMode': return { updated: true, surface: 'renderer-canvas' } + case 'browser.dismissPreview': + return { dismissed: true } + case 'computerUse.setPreviewMode': + return { updated: true, surface: 'renderer-canvas' } + case 'computerUse.dismissPreview': + return { dismissed: true } case 'browser.detach': return { detached: true } case 'browser.destroy':