diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08105b5..e12c71c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,9 @@ jobs: - name: Typecheck run: bun run typecheck + - name: Lint + run: bun run lint + - name: Unit tests run: bun test tests/unit diff --git a/.github/workflows/stealth-bench.yml b/.github/workflows/stealth-bench.yml new file mode 100644 index 0000000..9390e89 --- /dev/null +++ b/.github/workflows/stealth-bench.yml @@ -0,0 +1,38 @@ +name: Stealth Bench + +# Separate, non-blocking benchmark: validates anti-bot stealth +# non-regression against public detection pages. Independent of CI — +# never gates merges. Manual dispatch + weekly schedule. +on: + workflow_dispatch: + schedule: + # Mondays 06:00 UTC + - cron: "0 6 * * 1" + +jobs: + stealth-bench: + runs-on: ubuntu-latest + timeout-minutes: 15 + # Network-dependent on third-party sites; do not fail the suite on flakes. + continue-on-error: true + steps: + - uses: actions/checkout@v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: 22 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Install Chromium (Patchright + system deps) + run: bunx patchright install --with-deps chromium + + - name: Run stealth benchmark + run: node --import tsx tests/live/stealth-bench.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4213db8..3e13f20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [0.1.55] - 11-06-2026 + +### Added + +- **Layout-agnostic price extraction** — captures a currency whether it sits before or after the amount (`CHF 6.90`, `6.90 CHF`, `10 €`, `350 kr`), even when the symbol and number land on separate DOM lines (`CHF\n6.90`, the digitec case that previously yielded zero prices). Handles non-breaking/narrow spaces and CH (`1'234.56`) / EU (`1.234,56`) decimal formats. Each price now carries a short `context` label of its surrounding line. +- **Structured per-card extraction** — `browser_products` MCP tool + `fuse-browser products ` CLI return `{title, price, currency, url}` grouped by repeated product card, plus an `extract_schema` container mode that iterates card-by-card. +- **New MCP tools** — `browser_tabs` (OAuth popups/multi-tab), `browser_dialog` + `browser_downloads`, `browser_console` + `browser_network`, `browser_autoscroll` (infinite-scroll until idle). Screenshots exposed as MCP resources (`screenshot://{sessionId}/last`). +- **CLI parity** — six one-shot page commands: `run` (multi-step plans via `--steps`/`--steps-file`/stdin), `products`, `extract`, `snapshot`, `screenshot`, `inspect`. `--help` now lists all 15 commands. +- **Config** — `FUSE_CAPS` tool-group filtering, named auth `profile`, `blockResources`, MCP progress notifications on batch tools, configurable network-log buffer (`FUSE_NETLOG_MAX`) that pins the main document. +- **Stealth** — self-healing selectors (role/text/re-snapshot fallback) and a weekly anti-bot benchmark workflow. + +### Fixed + +- **Booking currency** — `prepareBookingCurrency` no longer does an intermediate homepage navigation that landed on the consent wall and blanked the target page; cookies + the in-page picker apply the currency without it. +- **Probe robustness** — resilient load-state settle + a single re-extraction when the first text/title come back empty, fixing blank reports on heavy/consent-gated pages. +- **Tabs network capture** — a tab opened with a URL now wires its network log before navigating, so its document and subresources are captured. +- **mainText** — strips nav/aside/search/filter sub-trees so filter sidebars (e.g. Booking's budget slider) no longer leak into extracted prices, while product grids still yield every card. + +### Tooling + +- Biome linter wired into CI; full suite green (292 unit, 20 integration on real Chromium). + ## [0.1.54] - 08-06-2026 ### Fixed diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..6cd29ef --- /dev/null +++ b/biome.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.4.16/schema.json", + "files": { + "includes": ["src/**", "tests/**", "!dist/**", "!node_modules/**"] + }, + "formatter": { + "enabled": false + }, + "assist": { + "enabled": false + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "style": { + "useImportType": "off", + "useTemplate": "off" + }, + "complexity": { + "useOptionalChain": "off" + }, + "suspicious": { + "noAssignInExpressions": "off" + } + } + }, + "overrides": [ + { + "includes": ["tests/**"], + "linter": { + "rules": { + "style": { + "noNonNullAssertion": "off" + }, + "suspicious": { + "useIterableCallbackReturn": "off" + } + } + } + } + ] +} diff --git a/bun.lock b/bun.lock index 59cb49c..b705da0 100644 --- a/bun.lock +++ b/bun.lock @@ -17,6 +17,7 @@ "zod": "^4.4.3", }, "devDependencies": { + "@biomejs/biome": "^2.4.16", "@types/node": "^25.9.1", "tsx": "^4.22.4", "typescript": "^6.0.3", @@ -28,6 +29,24 @@ "playwright", ], "packages": { + "@biomejs/biome": ["@biomejs/biome@2.4.16", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.16", "@biomejs/cli-darwin-x64": "2.4.16", "@biomejs/cli-linux-arm64": "2.4.16", "@biomejs/cli-linux-arm64-musl": "2.4.16", "@biomejs/cli-linux-x64": "2.4.16", "@biomejs/cli-linux-x64-musl": "2.4.16", "@biomejs/cli-win32-arm64": "2.4.16", "@biomejs/cli-win32-x64": "2.4.16" }, "bin": { "biome": "bin/biome" } }, "sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA=="], + + "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.16", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wxPvu4XOA85YJk9ixSWUmq/QBHbid85BISbOAqqBM/5xQpPk9ayjk5375tOlSC0BeCwNSbPFafQBm+vBumXq0A=="], + + "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.16", "", { "os": "darwin", "cpu": "x64" }, "sha512-xFCqGPwYusQJp4N4NJLi1XJiZqjwFdjhT+KqtNy+Ug3qgfczqnTa6MSDvxJF6TkuDLoYJItMapz6tAf7kCekFw=="], + + "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-2kFb4//jxfZaP6D+Rj5VkHkxgyD9EoRAVBEQb8PKRv+s4NO2zYNJKXFaJmK1CmhufJOWEfpHKaRbOja7qjmdhQ=="], + + "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.16", "", { "os": "linux", "cpu": "arm64" }, "sha512-oYxnW0ARfJkr72ezzF2OR8N/rtkgLUQeYtF8cFhVswbknHxtTcmzSsanVJP8yQKnGpGpc2ck6c5zLvHahL6Cbg=="], + + "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-NbcBbi/nJqn5baae6wqRXdS7Gadf2uRpehSh6vMSYpG8OhkXl/Xg8aorWrJ+9VWqAT5ml90alLvorkpMW0nBwQ=="], + + "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.16", "", { "os": "linux", "cpu": "x64" }, "sha512-iHDS+MCM65DPqWGu+ECC3uoALyj2H7F4nVUPxIPjz/PIl94EUu+EDfGZDzFP+NY1EOPVt9NQvwFqq7HdMmowdg=="], + + "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.16", "", { "os": "win32", "cpu": "arm64" }, "sha512-0rgImMsNb5v/chhkIFe3wu7PEFClS6RBAYUijGL9UsYN3PanSaoK24HSSuSJb1pYbYYVjzAyZTl3gtjJ84BM8A=="], + + "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.16", "", { "os": "win32", "cpu": "x64" }, "sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], diff --git a/docs/cli.md b/docs/cli.md index 4c52f08..9e9a88e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,6 +1,6 @@ # CLI -`fuse-browser` is a command-line front-end for the browser agent. It exposes four subcommands that all share a single flag parser (`node:util` `parseArgs`, strict mode), so any flag is accepted globally but only consumed by the subcommands documented below. +`fuse-browser` is a command-line front-end for the browser agent. It exposes nine one-shot subcommands (`probe`, `fetch`, `fetch-batch`, `crawl`, `collect-batch`, `serp-batch`, `shots`, `shots-batch`, `site-shots`) that all share a single flag parser (`node:util` `parseArgs`, strict mode), so any flag is accepted globally but only consumed by the subcommands documented below. Session-based interaction (open/navigate/click/products/autoscroll/…) is exposed through the MCP server (`browser-mcp`), not the CLI. ``` fuse-browser probe [flags] @@ -109,6 +109,61 @@ Applicable flags: `--engine`, `--country`, `--headed`, `--output-dir`, `--viewpo --- +## Page commands (one-shot) + +These commands open a page, run one operation, print JSON on stdout (errors on stderr), and tear the browser down. Exit codes: `0` success, `1` functional/step failure, `2` bad usage or malformed JSON. They all share the page flags `--engine`, `--country`, `--currency`, `--headed`, `--human-mode`, `--proxy`, `--proxy-map`, `--output-dir`, `--storage-state`, `--no-robots`, `--wait-ms`, `--block-resources`. + +### `run ` + +Executes a multi-step plan in one session. Steps come from `--steps ''` (inline array) or `--steps-file ` (`-` reads stdin). Each step is `{type, …}`: `navigate`, `click`, `fill`, `scroll`, `press`, `wait`, `select`, `extract`. Prints `{ok, url, steps}`; on a failed step prints `{ok:false, error:{kind:"step_failed", step, message}}` and exits `1`. Malformed/non-array JSON exits `2`. + +```bash +fuse-browser run https://example.com \ + --steps '[{"type":"wait","ms":500},{"type":"extract","kind":"text"}]' +``` + +### `products ` + +Extracts repeated product cards from the rendered DOM. `--limit ` caps the result; `--container ` forces the card container. Prints `{url, count, products}`. + +```bash +fuse-browser products "https://www.digitec.ch/en/search?q=macbook" --limit 20 +``` + +### `extract ` + +Pulls page content. `--kind text` (default) returns main text, `prices` returns parsed prices, `markdown` returns LLM-ready markdown + metadata. Prints `{url, kind, …}`. + +```bash +fuse-browser extract https://example.com --kind markdown +``` + +### `snapshot ` + +Captures the indexed interactive-element snapshot (each element gets a stable `ref`). Add `--selectors` for per-element CSS selectors. Prints `{url, count, elements}`. + +```bash +fuse-browser snapshot https://example.com --selectors +``` + +### `screenshot ` + +Captures a PNG. Add `--full-page` for the full scroll height. With `--output ` it writes the file and prints `{url, path, bytes}`; otherwise it prints `{url, base64}`. + +```bash +fuse-browser screenshot https://example.com --full-page --output shot.png +``` + +### `inspect ` + +Snapshots the page (tagging elements with `data-fuse-ref`) then reports computed style, box, and WCAG contrast for the element named by `--ref ` (a `ref` from `snapshot`). `--ref` is required (else exits `2`); an unknown ref exits `1`. Prints `{url, ref, style}`. + +```bash +fuse-browser inspect https://example.com --ref 0 +``` + +--- + ## Approval guardrail Sensitive actions (pay / book / checkout / confirm) are blocked unless `--approved` is passed. When blocked, `probe` prints `BLOCKED: ` to stderr and exits `2`. `--approved` sets the `humanApproved` flag on the probe. diff --git a/docs/configuration.md b/docs/configuration.md index 4bf6566..f20cc13 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -25,6 +25,8 @@ Every field is optional. Defaults are applied by `resolveConfig` (`src/agent/con | `cdpCloseOnDone` | `boolean` | `true` for `ws/wss`, else `false` | Close the CDP session on teardown. Remote endpoints (Browserless) get a fresh context that is closed when done; a local attach never closes the user's browser (only the link is dropped). | | `cdpTimeoutMs` | `number` | `20000` | Timeout (ms) for the CDP connect. | | `storageStatePath` | `string` | `null` | Path to a Playwright storage-state JSON (cookies + localStorage) to load/persist a logged-in session. See [./sessions.md](./sessions.md). | +| `profile` | `string` | `null` | Named persistent auth profile — shorthand for `storageStatePath` at `~/.fuse-browser/profiles/.json` (`FUSE_BROWSER_HOME` overrides the home dir; ignored when `storageStatePath` is set). Name: letters/digits then `-`/`_`, max 41 chars. | +| `blockResources` | `string[]` | `null` (off) | Resource types aborted at the network layer to speed up batch runs: `image`, `media`, `font`, `stylesheet`, `script`, `xhr`, `fetch`, `websocket`, `manifest`, `other`. Unknown types are ignored. | | `harPath` | `string` | `null` | Record network traffic to this HAR file. See [./sessions.md](./sessions.md). | | `harMode` | `"minimal" \| "full"` | `"minimal"` | HAR recording detail. `minimal` records metadata only; `full` records response bodies. | | `harReplay` | `string` | `null` | Replay network traffic from this HAR file instead of hitting the live network. | diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index 7979c85..7f332d4 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -1,6 +1,6 @@ # MCP tools -Complete reference for the 32 `browser_*` tools exposed by the fuse-browser MCP server. +Complete reference for the 37 `browser_*` tools exposed by the fuse-browser MCP server. Tools fall into two families: @@ -11,6 +11,26 @@ Every field is optional unless **Required** says `yes`. Defaults shown below com The shared identity/profile options (the `agentOptionShape`) are listed once under [`browser_open`](#browser_open); tools that accept them say so and link back. +## Capability groups (`FUSE_CAPS`) + +By default all 37 tools are registered. Set the `FUSE_CAPS` env var (comma-separated group names) to expose fewer tools — a lighter context for the LLM client: + +| Group | Tools | +| --- | --- | +| `core` | Session lifecycle (`browser_open`/`browser_status`/`browser_close`/`browser_connect`), navigation (`browser_navigate`/`browser_back`/`browser_forward`), actions (`browser_click`/`browser_fill`/`browser_login`/`browser_scroll`/`browser_press`/`browser_select`), `browser_tabs`, `browser_dialog`/`browser_downloads`, `browser_snapshot`/`browser_act`, `browser_wait`/`browser_wait_for`, `browser_screenshot`. | +| `batch` | `browser_probe`, `browser_probe_html`, `browser_fetch`, `browser_fetch_batch`, `browser_crawl`, `browser_collect_batch`, `browser_shots_batch`, `browser_site_shots`, `browser_serp_batch`. | +| `extract` | `browser_collect`, `browser_run`, `browser_extract`, `browser_extract_schema`. | +| `debug` | `browser_inspect`, `browser_console`, `browser_network`, `browser_visual_diff`, `browser_metrics`. | +| `live` | `browser_handoff`, `browser_live_view`, `browser_live_view_stop`. | + +```sh +FUSE_CAPS=core,extract browser-mcp # only the core + extract groups +``` + +Parsing is forgiving: names are case-insensitive and whitespace-tolerant (`" CORE , Extract "` works), unknown names are reported on stderr and ignored, and a blank/unset value — or one containing **only** unknown names — falls back to all groups (never an empty server). + +**Progress notifications.** The batch tools (`browser_fetch_batch`, `browser_crawl`, `browser_collect_batch`, `browser_shots_batch`, `browser_site_shots`, `browser_serp_batch`) emit MCP `notifications/progress` (`progress`/`total` per finished item, with the item URL/query as `message`) when the client sends a `progressToken` with the request. Clients that don't request progress see no change. + --- ## One-shot @@ -285,6 +305,53 @@ Launch an installed browser with remote debugging, attach to it, and return a se { "browser": "chrome", "port": 9222 } ``` +### browser_tabs + +Manage the tabs of a live session: list them, open a new tab (optional `url`), select one as the active target of every other `browser_*` tool, or close one. Use it when a click spawned a popup (OAuth login, `target=_blank` link): `list` to find it, `select` to drive it, `close` then `select` to come back. + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | +| `action` | enum `list` \| `new` \| `select` \| `close` | yes | Tab action to perform. | +| `index` | number | no* | Tab index (**required** for `select`/`close`; `missing_index` error otherwise). | +| `url` | string | no | URL to open in the new tab (for `new`). | + +Always returns the tab list plus the active index after the action: `{ tabs, active }`. Closing the last tab is refused (`cannot_close_last_tab`) — use `browser_close` to end the session. An out-of-range `index` returns `invalid_tab_index`. + +```json +{ "sessionId": "s_abc123", "action": "select", "index": 1 } +``` + +### browser_dialog + +Set how native dialogs (`alert`/`confirm`/`prompt`/`beforeunload`) are handled on this session: accept or dismiss, with optional text for prompts. Applies to **upcoming** dialogs; the default policy (before any call) is **dismiss**. Dialog handling is auto-attached when the session opens, so dialogs never block a run. + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | +| `action` | enum `accept` \| `dismiss` | yes | Policy applied to upcoming dialogs. | +| `promptText` | string | no | Text typed into `prompt` dialogs when accepting. | + +Returns `{ policy, recent }` — `recent` is the last observed dialogs (max 20, oldest first), each `{ type, message, at, handled }`. + +```json +{ "sessionId": "s_abc123", "action": "accept", "promptText": "yes" } +``` + +### browser_downloads + +List the files downloaded by this session. Download capture is auto-attached when the session opens: every download is saved under `/downloads/` (suffixing `-1`, `-2` on name collisions) and recorded. + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | + +Returns `{ count, downloads }` — each download is `{ url, suggestedFilename, path, at, error? }` (`path` is empty while saving or when `error` is set). + +```json +{ "sessionId": "s_abc123" } +``` + --- ## Navigate @@ -678,6 +745,39 @@ Returned fields: `uptimeMs`, `probesOk`, `probesFailed`, `avgDurationMs`, `minDu { "reset": false } ``` +### browser_console + +Console messages captured in the session since open (last 80 kept). Use to debug JS errors, CSP violations, or why a page misbehaves. + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | +| `level` | enum `error` \| `warning` \| `info` \| `log` \| `debug` | no | Keep only this console message type. | +| `limit` | number | no | Last N entries returned (default `50`). | + +Returns `{ count, entries }`. + +```json +{ "sessionId": "s_abc123", "level": "error", "limit": 20 } +``` + +### browser_network + +Network requests captured in the session since open (last 80 kept), merged into one row per URL: `method`, `url`, `status`, `resourceType`. Use to debug why a page does not load: failed requests (`status: 404/500`), blocked APIs (`urlContains`). Entries without `status` got no response (pending/failed). + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | +| `status` | number | no | Keep only rows with this exact HTTP status. | +| `urlContains` | string | no | Keep only rows whose URL contains this substring. | +| `limit` | number | no | Last N rows returned (default `50`). | + +Returns `{ count, requests }`. + +```json +{ "sessionId": "s_abc123", "status": 404 } +``` + --- ## Live view diff --git a/docs/sessions.md b/docs/sessions.md index c679132..0a6eefa 100644 --- a/docs/sessions.md +++ b/docs/sessions.md @@ -63,6 +63,19 @@ browser_close { "sessionId": "" } browser_open { "storageStatePath": "./.auth/site.json" } ``` +> **Shorthand:** the `profile` option names a persistent auth profile (`profile: "github"` ⇒ `storageStatePath` at `~/.fuse-browser/profiles/github.json`). It is ignored when `storageStatePath` is set explicitly — see [configuration](./configuration.md). + +--- + +## Dialogs & downloads (auto-attached) + +Every session is wired for native dialogs and downloads **at open** (and re-wired after [crash recovery](#crash-recovery)) — no setup call needed: + +- **Dialogs** — `alert`/`confirm`/`prompt`/`beforeunload` are handled by a per-session policy so they never block a run. The default policy is **dismiss**; change it with `browser_dialog` (`accept`/`dismiss`, optional `promptText` for prompts), which also returns the last observed dialogs (max 20). +- **Downloads** — every download is saved under `/downloads/` (suffixing `-1`, `-2` on collisions). List them with `browser_downloads` (`{ url, suggestedFilename, path, at, error? }` each). + +See [MCP tools](./mcp-tools.md#browser_dialog) for the parameters. + --- ## HAR record / replay diff --git a/package.json b/package.json index c964a3a..0593950 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fusengine/browser-mcp", - "version": "0.1.54", + "version": "0.1.55", "description": "MCP server + CLI giving AI agents a real, stealth browser (Patchright/Playwright) — per-country identity, self-healing actions, snapshots, multi-step plans, structured extraction, CDP attach.", "license": "MIT", "author": "Fusengine", @@ -53,6 +53,7 @@ "prepublishOnly": "npm run build", "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", + "lint": "biome check src tests", "test": "bun test tests/unit", "test:integration": "node --test --import tsx tests/integration/mcp.test.ts tests/integration/probe.test.ts tests/integration/snapshot.test.ts tests/integration/snapshot-frames.test.ts tests/integration/collect.test.ts tests/integration/collect-batch.test.ts tests/integration/selectors.test.ts tests/integration/visual-diff.test.ts tests/integration/session-state.test.ts tests/integration/pipeline.test.ts tests/integration/run.test.ts tests/integration/extract-schema.test.ts tests/integration/recovery.test.ts tests/integration/live-view.test.ts", "browsers": "patchright install chromium", @@ -76,6 +77,7 @@ "zod": "^4.4.3" }, "devDependencies": { + "@biomejs/biome": "^2.4.16", "@types/node": "^25.9.1", "tsx": "^4.22.4", "typescript": "^6.0.3" diff --git a/src/actions/auto-scroll.ts b/src/actions/auto-scroll.ts new file mode 100644 index 0000000..a69f8ff --- /dev/null +++ b/src/actions/auto-scroll.ts @@ -0,0 +1,58 @@ +/** + * Auto-scroll a page to trigger lazy-load / infinite-scroll until the list + * stabilises, a target count is reached, or a cap is hit. + * @module actions/auto-scroll + */ +import type { Page } from "playwright"; +import type { AutoScrollOpts, ScrollProbe, StopInput, StopState } from "../interfaces/auto-scroll.js"; +import { evalScriptArg } from "../lib/evaluate.js"; + +/** + * Pure decision: should auto-scroll stop, and what is the new idle streak? + * Stops on reaching the selector target, the idle threshold, or the cap. + */ +export function decideStop(input: StopInput): StopState { + const { prev, curr, idle, rounds, idleRounds, maxScrolls, minCount, hasSelector } = input; + if (hasSelector && curr.count >= minCount) return { stop: true, idle }; + const grew = prev === null || curr.height > prev.height; + const nextIdle = grew ? 0 : idle + 1; + if (nextIdle >= idleRounds) return { stop: true, idle: nextIdle }; + if (rounds >= maxScrolls) return { stop: true, idle: nextIdle }; + return { stop: false, idle: nextIdle }; +} + +/** In-page probe: scroll to the bottom and report height + optional count. */ +const PROBE_SCRIPT = `(sel) => { + window.scrollTo(0, document.body.scrollHeight); + return { + height: document.body.scrollHeight, + count: sel ? document.querySelectorAll(sel).length : 0, + }; +}`; + +/** + * Scroll until the page stops growing, a selector reaches `minCount`, or + * `maxScrolls` is hit. Triggers lazy-load before bulk extraction. + */ +export async function autoScroll(page: Page, opts: AutoScrollOpts = {}): Promise<{ rounds: number; height: number }> { + const maxScrolls = opts.maxScrolls ?? 20; + const idleRounds = opts.idleRounds ?? 2; + const minCount = opts.minCount ?? 1; + const delayMs = opts.delayMs ?? 600; + const sel = opts.untilSelector ?? null; + let prev: ScrollProbe | null = null; + let idle = 0; + let rounds = 0; + let height = 0; + while (rounds < maxScrolls) { + const curr = await evalScriptArg(page, PROBE_SCRIPT, sel); + rounds += 1; + height = curr.height; + const next = decideStop({ prev, curr, idle, rounds, idleRounds, maxScrolls, minCount, hasSelector: sel !== null }); + idle = next.idle; + if (next.stop) break; + prev = curr; + await page.waitForTimeout(delayMs); + } + return { rounds, height }; +} diff --git a/src/actions/heal-locator.ts b/src/actions/heal-locator.ts new file mode 100644 index 0000000..18a406e --- /dev/null +++ b/src/actions/heal-locator.ts @@ -0,0 +1,84 @@ +/** + * Auto-healing locator resolution (Playwright "healer" pattern). When the + * primary selector/ref of a click/fill no longer matches a changed page, this + * re-resolves the target by accessible role+name, then visible text, then a + * fresh snapshot re-matched against the original label — returning the first + * visible locator (or `null` when nothing recovers). + * @module actions/heal-locator + */ +import type { Locator, Page } from "playwright"; +import type { InteractiveElement } from "../interfaces/extraction.js"; +import { logger } from "../lib/logger.js"; +import { escapeRegExp } from "../lib/text.js"; +import { refLocator } from "./ref-locator.js"; + +/** Re-resolve a changed page. `snapshotFn` re-captures the interactive tree. */ +export type SnapshotFn = (page: Page) => Promise; + +/** Return the locator if it resolves to a visible element, else `null`. */ +async function firstVisible(locator: Locator | null): Promise { + if (!locator) return null; + try { + if ((await locator.count()) > 0 && (await locator.first().isVisible())) { + return locator.first(); + } + } catch { + // Detached frame / mid-navigation: treat as a miss, let the next strategy try. + } + return null; +} + +/** Lowercased haystack of an element's accessible-ish text fields. */ +function elementText(el: InteractiveElement): string { + return `${el.text} ${el.name ?? ""} ${el.value ?? ""} ${el.placeholder ?? ""}` + .toLowerCase() + .trim(); +} + +/** + * Re-snapshot the page and resolve the first visible interactive element whose + * text/name/value/placeholder contains `target` (case-insensitive), via its ref. + */ +async function healViaSnapshot( + page: Page, + target: string, + snapshotFn: SnapshotFn, +): Promise { + let elements: InteractiveElement[]; + try { + elements = await snapshotFn(page); + } catch { + return null; + } + const needle = target.toLowerCase(); + for (const el of elements) { + if (el.ref === undefined || !elementText(el).includes(needle)) continue; + const hit = await firstVisible(refLocator(page, el.ref)); + if (hit) return hit; + } + return null; +} + +/** + * Recover a locator for `target` after the primary resolution failed. Tries, in + * order: (a) accessible role+name, (b) visible text, (c) fresh snapshot re-match. + * Returns the first visible {@link Locator}, or `null` if nothing recovers. + */ +export async function healLocator( + page: Page, + target: string, + snapshotFn: SnapshotFn, +): Promise { + if (!target.trim()) return null; + const rx = new RegExp(escapeRegExp(target), "i"); + const byRole = + (await firstVisible(page.getByRole("button", { name: rx }))) ?? + (await firstVisible(page.getByRole("link", { name: rx }))) ?? + (await firstVisible(page.getByRole("textbox", { name: rx }))); + const healed = + byRole ?? + (await firstVisible(page.getByText(rx))) ?? + (await healViaSnapshot(page, target, snapshotFn)); + if (healed) logger.debug("heal_locator: recovered target", { target }); + return healed; +} diff --git a/src/actions/smart-click.ts b/src/actions/smart-click.ts index 4356996..8104b85 100644 --- a/src/actions/smart-click.ts +++ b/src/actions/smart-click.ts @@ -3,9 +3,11 @@ * @module actions/smart-click */ import type { Locator, Page } from "playwright"; +import { captureSnapshot } from "../extraction/snapshot.js"; import type { ActionResult } from "../interfaces/types.js"; import { evalScriptArg } from "../lib/evaluate.js"; import { escapeRegExp } from "../lib/text.js"; +import { healLocator } from "./heal-locator.js"; import { humanPause } from "./human.js"; import { humanMoveTo } from "./human-mouse.js"; @@ -56,6 +58,15 @@ export async function smartClick( lastError = String(err).split("\n")[0] ?? "error"; } } + try { + const healed = await healLocator(page, target, captureSnapshot); + if (healed) { + await healed.click({ timeout: 2_000 }); + return { type: "click", target, ok: true, strategy: "heal" }; + } + } catch (err) { + lastError = String(err).split("\n")[0] ?? "error"; + } try { const clicked = await evalScriptArg(page, HEURISTIC_CLICK, target); if (clicked) return { type: "click", target, ok: true, strategy: "heuristic" }; diff --git a/src/actions/smart-fill.ts b/src/actions/smart-fill.ts index 378ab28..746618d 100644 --- a/src/actions/smart-fill.ts +++ b/src/actions/smart-fill.ts @@ -3,8 +3,10 @@ * @module actions/smart-fill */ import type { Locator, Page } from "playwright"; +import { captureSnapshot } from "../extraction/snapshot.js"; import type { ActionResult } from "../interfaces/types.js"; import { escapeRegExp, randInt } from "../lib/text.js"; +import { healLocator } from "./heal-locator.js"; import { humanPause } from "./human.js"; async function humanType(page: Page, locator: Locator, value: string): Promise { @@ -51,5 +53,15 @@ export async function smartFill( lastError = String(err).split("\n")[0] ?? "error"; } } + try { + const healed = await healLocator(page, target, captureSnapshot); + if (healed) { + if (humanMode) await humanType(page, healed, value); + else await healed.fill(value, { timeout: 2_000 }); + return { type: "fill", target, ok: true, strategy: "heal" }; + } + } catch (err) { + lastError = String(err).split("\n")[0] ?? "error"; + } return { type: "fill", target, ok: false, error: lastError }; } diff --git a/src/agent/collect-batch.ts b/src/agent/collect-batch.ts index a9cc660..fbce77f 100644 --- a/src/agent/collect-batch.ts +++ b/src/agent/collect-batch.ts @@ -21,6 +21,8 @@ const DEFAULT_CONCURRENCY = 2; export interface CollectBatchOptions extends CollectOptions { concurrency?: number; throttleMs?: number; + /** Called after each URL settles (success or failure): `(done, total, url)`. */ + onProgress?: (done: number, total: number, label?: string) => void; } /** One batch entry: the collected list for a URL, or an error. */ @@ -44,11 +46,16 @@ export async function collectBatch( const concurrency = opts.concurrency && opts.concurrency > 0 ? opts.concurrency : DEFAULT_CONCURRENCY; const throttleMs = opts.throttleMs ?? 250; const pool = new BrowserPool(config); + let done = 0; try { const outcomes = await mapConcurrent(urls, concurrency, async (url) => { - await throttleHost(url, jitterMs(throttleMs)); - const r = await pool.withPage((page) => collectOnPage(page, config, url, opts)); - return { url, count: r.items.length, steps: r.steps, reachedEnd: r.reachedEnd, items: r.items }; + try { + await throttleHost(url, jitterMs(throttleMs)); + const r = await pool.withPage((page) => collectOnPage(page, config, url, opts)); + return { url, count: r.items.length, steps: r.steps, reachedEnd: r.reachedEnd, items: r.items }; + } finally { + opts.onProgress?.(++done, urls.length, url); + } }); return outcomes.map((o, i) => (o.ok ? o.value : { url: urls[i] ?? "", error: String(o.error) })); } finally { diff --git a/src/agent/collect-run.ts b/src/agent/collect-run.ts index a802f03..ece1ad6 100644 --- a/src/agent/collect-run.ts +++ b/src/agent/collect-run.ts @@ -7,7 +7,7 @@ import type { Page } from "playwright"; import { selectEngineForConfig } from "../engine/registry.js"; import { teardownOpened } from "../engine/teardown.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; import { type CollectOptions, type CollectResult, scrollCollect } from "../state/scroll-collect.js"; import type { ResolvedConfig } from "./config.js"; @@ -16,7 +16,7 @@ import type { ResolvedConfig } from "./config.js"; * page-level work with no browser lifecycle (so a pool can drive it). */ export async function collectOnPage(page: Page, config: ResolvedConfig, url: string, opts: CollectOptions): Promise { - await gotoWithRetry(page, url, { waitUntil: "domcontentloaded", timeout: 30_000 }, config.retry); + await gotoWithRetry(page, url, DEFAULT_GOTO, config.retry); await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); return scrollCollect(page, opts); } diff --git a/src/agent/config-defaults.ts b/src/agent/config-defaults.ts new file mode 100644 index 0000000..9933fc7 --- /dev/null +++ b/src/agent/config-defaults.ts @@ -0,0 +1,42 @@ +/** + * Option resolvers that need lookups, kept out of `agent/config` for size: + * the proxy chain (explicit URL → country map → pool) and the storage-state + * path (explicit path → named auth profile). + * @module agent/config-defaults + */ +import { profileStoragePath } from "../identity/profiles.js"; +import type { AgentOptions } from "../interfaces/types.js"; +import { loadProxyCountryMap, type ProxySource, resolveProxy } from "../proxy/country-map.js"; +import { acquirePoolProxy } from "../proxy/pool.js"; + +/** + * Resolve the effective proxy: explicit `proxyUrl`, else the country-map entry + * for the resolved country, else a proxy from the fallback pool. + * + * @param opts - Agent options (proxy fields). + * @param countryCode - Resolved identity country code. + * @returns Proxy URL (or null) plus its provenance. + */ +export function resolveProxyChain( + opts: AgentOptions, + countryCode: string, +): { proxyUrl: string | null; proxySource: ProxySource } { + const map = loadProxyCountryMap(opts.proxyCountryMap, opts.proxyMapPath); + const resolved = resolveProxy(opts.proxyUrl, countryCode, map); + if (resolved.proxyUrl) return resolved; + const pooled = acquirePoolProxy(opts.proxiesPath); + if (pooled) return { proxyUrl: pooled, proxySource: "pool" }; + return resolved; +} + +/** + * Resolve the storage-state path: an explicit `storageStatePath` beats a + * named auth `profile` (`~/.fuse-browser/profiles/.json`). + * + * @param opts - Agent options (`storageStatePath`, `profile`). + * @returns The effective storage-state path, or null. + */ +export function resolveStorageStatePath(opts: AgentOptions): string | null { + if (opts.storageStatePath) return opts.storageStatePath; + return opts.profile ? profileStoragePath(opts.profile) : null; +} diff --git a/src/agent/config.ts b/src/agent/config.ts index ca743e5..94580fa 100644 --- a/src/agent/config.ts +++ b/src/agent/config.ts @@ -13,8 +13,8 @@ import { resolveRetry } from "../net/retry-config.js"; import type { AgentOptions } from "../interfaces/types.js"; import { ensureDir } from "../lib/fs.js"; import { resolveDefaultOutputDir } from "../lib/output-dir.js"; -import { loadProxyCountryMap, type ProxySource, resolveProxy } from "../proxy/country-map.js"; -import { acquirePoolProxy } from "../proxy/pool.js"; +import type { ProxySource } from "../proxy/country-map.js"; +import { resolveProxyChain, resolveStorageStatePath } from "./config-defaults.js"; /** Effective configuration used by every probe run. */ export interface ResolvedConfig { @@ -27,6 +27,7 @@ export interface ResolvedConfig { cdpCloseOnDone: boolean; cdpTimeoutMs: number; storageStatePath: string | null; + blockResources: string[] | null; harPath: string | null; harMode: "minimal" | "full"; harReplay: string | null; @@ -53,15 +54,7 @@ export function resolveConfig(opts: AgentOptions = {}): ResolvedConfig { const outputDir = opts.outputDir ?? resolveDefaultOutputDir(); ensureDir(outputDir); const identity = resolveIdentity(opts); - const map = loadProxyCountryMap(opts.proxyCountryMap, opts.proxyMapPath); - let { proxyUrl, proxySource } = resolveProxy(opts.proxyUrl, identity.countryCode, map); - if (!proxyUrl) { - const pooled = acquirePoolProxy(opts.proxiesPath); - if (pooled) { - proxyUrl = pooled; - proxySource = "pool"; - } - } + const { proxyUrl, proxySource } = resolveProxyChain(opts, identity.countryCode); const userDataDir = opts.userDataDir ?? null; if (userDataDir) ensureDir(userDataDir); const siteMemoryDir = opts.siteMemoryDir ?? join(outputDir, "site-memory"); @@ -75,7 +68,8 @@ export function resolveConfig(opts: AgentOptions = {}): ResolvedConfig { cdpHeaders: opts.cdpHeaders ?? null, cdpCloseOnDone: opts.cdpCloseOnDone ?? isRemoteCdp(opts.cdpEndpoint ?? ""), cdpTimeoutMs: opts.cdpTimeoutMs ?? 20_000, - storageStatePath: opts.storageStatePath ?? null, + storageStatePath: resolveStorageStatePath(opts), + blockResources: opts.blockResources?.length ? opts.blockResources : null, harPath: opts.harPath ?? null, harMode: opts.harMode ?? "minimal", harReplay: opts.harReplay ?? null, diff --git a/src/agent/crawl.ts b/src/agent/crawl.ts index 31b79cf..cd4d14b 100644 --- a/src/agent/crawl.ts +++ b/src/agent/crawl.ts @@ -26,6 +26,8 @@ export interface CrawlOptions { proxyUrl?: string; respectRobots?: boolean; throttleMs?: number; + /** Called after each page settles (success or failure): `(done, maxPages, url)`. */ + onProgress?: (done: number, total: number, label?: string) => void; } /** A crawled page: its rendered content plus its BFS depth from the seed. */ @@ -51,15 +53,20 @@ export async function crawl(seed: string, opts: CrawlOptions = {}): Promise<{ co const visited = new Set([seed]); const pages: CrawlPage[] = []; let frontier = [seed]; + let done = 0; for (let depth = 0; depth <= maxDepth && frontier.length > 0 && pages.length < maxPages; depth++) { const batch = frontier.slice(0, maxPages - pages.length); const fetched = await mapConcurrent(batch, concurrency, async (url) => { - if (robots && !(await robots.allowed(url))) throw new Error("robots-disallowed"); - await throttleHost(url, jitterMs(throttleMs)); - const r = await fetchFast(url, opts.proxyUrl); - const body = await resolveFetchBody(url, r, { browserFallback: opts.browserFallback, proxyUrl: opts.proxyUrl }); - return { rendered: await renderFetch(body, { format: opts.format, maxChars: opts.maxChars }), html: body.html, url: body.url }; + try { + if (robots && !(await robots.allowed(url))) throw new Error("robots-disallowed"); + await throttleHost(url, jitterMs(throttleMs)); + const r = await fetchFast(url, opts.proxyUrl); + const body = await resolveFetchBody(url, r, { browserFallback: opts.browserFallback, proxyUrl: opts.proxyUrl }); + return { rendered: await renderFetch(body, { format: opts.format, maxChars: opts.maxChars }), html: body.html, url: body.url }; + } finally { + opts.onProgress?.(++done, maxPages, url); + } }); const next: string[] = []; for (const o of fetched) { diff --git a/src/agent/fetch-batch.ts b/src/agent/fetch-batch.ts index 3df4897..f271b81 100644 --- a/src/agent/fetch-batch.ts +++ b/src/agent/fetch-batch.ts @@ -20,6 +20,8 @@ export interface FetchBatchOptions { browserFallback?: boolean; proxyUrl?: string; concurrency?: number; + /** Called after each URL settles (success or failure): `(done, total, url)`. */ + onProgress?: (done: number, total: number, label?: string) => void; } /** One batch result: a rendered fetch, or `{ url, error }` on failure. */ @@ -34,10 +36,15 @@ export type FetchBatchItem = RenderedFetch | { url: string; error: string }; */ export async function fetchBatch(urls: string[], opts: FetchBatchOptions = {}): Promise { const concurrency = opts.concurrency && opts.concurrency > 0 ? opts.concurrency : DEFAULT_CONCURRENCY; + let done = 0; const outcomes = await mapConcurrent(urls, concurrency, async (url) => { - const r = await fetchFast(url, opts.proxyUrl); - const body = await resolveFetchBody(url, r, { browserFallback: opts.browserFallback, proxyUrl: opts.proxyUrl }); - return renderFetch(body, { format: opts.format, maxChars: opts.maxChars }); + try { + const r = await fetchFast(url, opts.proxyUrl); + const body = await resolveFetchBody(url, r, { browserFallback: opts.browserFallback, proxyUrl: opts.proxyUrl }); + return await renderFetch(body, { format: opts.format, maxChars: opts.maxChars }); + } finally { + opts.onProgress?.(++done, urls.length, url); + } }); return outcomes.map((o, i) => (o.ok ? o.value : { url: urls[i] ?? "", error: String(o.error) })); } diff --git a/src/agent/network.ts b/src/agent/network.ts index 6db8a0a..bbefb7a 100644 --- a/src/agent/network.ts +++ b/src/agent/network.ts @@ -4,7 +4,13 @@ */ import type { Page } from "playwright"; -const MAX_ENTRIES = 80; +const DEFAULT_MAX_ENTRIES = 250; + +/** Resolve the network-log cap from `FUSE_NETLOG_MAX` (falls back to 250). */ +function resolveMax(): number { + const raw = Number(process.env.FUSE_NETLOG_MAX); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_MAX_ENTRIES; +} /** Collected network and console entries (live, mutated by the listeners). */ export interface NetworkLog { @@ -12,17 +18,47 @@ export interface NetworkLog { console: Array<{ type: string; text: string }>; } -function pushCapped(arr: T[], item: T): void { +/** True for the main-document request entry that must never be evicted. */ +function isDocumentEntry(item: Record): boolean { + return item.type === "request" && item.resourceType === "document"; +} + +/** + * Push with a FIFO cap. The first main-document request is pinned at index 0 + * and never evicted, so tools can still inspect the initial navigation even + * after the buffer has cycled on request-heavy sites. + */ +function pushCapped(arr: Array>, item: Record, max: number): void { + arr.push(item); + if (arr.length <= max) return; + const pinned = arr.length > 0 && isDocumentEntry(arr[0] as Record); + arr.splice(pinned ? 1 : 0, 1); +} + +/** Push a console entry with a plain FIFO cap. */ +function pushConsole(arr: T[], item: T, max: number): void { arr.push(item); - if (arr.length > MAX_ENTRIES) arr.shift(); + if (arr.length > max) arr.shift(); } /** Register request/response/console listeners and return their backing arrays. */ export function attachListeners(page: Page): NetworkLog { + const max = resolveMax(); const network: Array> = []; const consoleEntries: Array<{ type: string; text: string }> = []; - page.on("request", (req) => pushCapped(network, { type: "request", method: req.method(), url: req.url() })); - page.on("response", (res) => pushCapped(network, { type: "response", status: res.status(), url: res.url() })); - page.on("console", (msg) => pushCapped(consoleEntries, { type: msg.type(), text: msg.text() })); + page.on("request", (req) => + pushCapped( + network, + { + type: "request", + method: req.method(), + url: req.url(), + resourceType: req.resourceType(), + }, + max, + ), + ); + page.on("response", (res) => pushCapped(network, { type: "response", status: res.status(), url: res.url() }, max)); + page.on("console", (msg) => pushConsole(consoleEntries, { type: msg.type(), text: msg.text() }, max)); return { network, console: consoleEntries }; } diff --git a/src/agent/probe-run.ts b/src/agent/probe-run.ts index eb87b05..12864b6 100644 --- a/src/agent/probe-run.ts +++ b/src/agent/probe-run.ts @@ -16,7 +16,7 @@ import type { ProbeReport } from "../interfaces/report.js"; import type { ProbeOptions } from "../interfaces/types.js"; import { ensureDir, sha1 } from "../lib/fs.js"; import { withBreaker } from "../net/breaker-guard.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; import { assertRobotsAllowed } from "../net/robots-guard.js"; import { throttleHost } from "../net/throttle.js"; import { reportProxyBlocked } from "../proxy/pool.js"; @@ -24,6 +24,7 @@ import { domSignature } from "../state/dom-signature.js"; import { runActions } from "./actions-loop.js"; import type { ResolvedConfig } from "./config.js"; import { detectAndSolve } from "./detect.js"; +import { reExtractIfEmpty, settleLoad } from "./probe-settle.js"; import { attachListeners } from "./network.js"; import { extractSerpStep } from "./serp-step.js"; import { huntContacts } from "./contact-hunt.js"; @@ -48,8 +49,8 @@ export async function runProbe( const targetUrl = urlWithCurrency(url, config.currency); if (targetUrl.includes("booking.com") && config.currency) await prepareBookingCurrency(page, config.currency); await throttleHost(targetUrl, config.retry.throttleMs); - await withBreaker(targetUrl, config.circuitBreaker, () => gotoWithRetry(page, targetUrl, { waitUntil: "domcontentloaded", timeout: 30_000 }, config.retry)); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await withBreaker(targetUrl, config.circuitBreaker, () => gotoWithRetry(page, targetUrl, DEFAULT_GOTO, config.retry)); + await settleLoad(page); const consent = options.autoConsent ? await handleCommonConsent(page, config.humanMode) : { handled: false }; @@ -59,10 +60,10 @@ export async function runProbe( const outcome = await runActions(page, config, actions, targetUrl, runId); if (options.waitMs) await page.waitForTimeout(options.waitMs); const after = await domSignature(page); - const text = await mainText(page); + const first = await mainText(page); + const { text, title } = await reExtractIfEmpty(page, first, await page.title()); const { challenges, captcha } = await detectAndSolve(page, text, options, config); if (config.proxySource === "pool" && config.proxyUrl && "cloudflare" in challenges && (challenges.cloudflare || challenges.captcha)) reportProxyBlocked(config.proxyUrl); - const title = await page.title(); await page.screenshot({ path: screenshotPath, fullPage: true }); const visual = options.observeVisual ? await visualObservation(page, screenshotPath) : {}; const serp = await extractSerpStep(page, options, config); diff --git a/src/agent/probe-settle.ts b/src/agent/probe-settle.ts new file mode 100644 index 0000000..766ce14 --- /dev/null +++ b/src/agent/probe-settle.ts @@ -0,0 +1,39 @@ +/** + * Settle helpers for the one-shot probe: a resilient load-state wait and a + * single re-extraction pass when the first text/title came back empty — the + * failure mode seen on heavy/consent-gated pages (e.g. Booking) where the DOM + * was not ready at first extraction. + * @module agent/probe-settle + */ +import type { Page } from "playwright"; +import { mainText } from "../extraction/main-text.js"; +import { waitForRealtimeSettle } from "../state/realtime.js"; + +/** + * Wait for `networkidle`, falling back to `domcontentloaded` so a perpetually + * busy page (analytics/websocket traffic) still yields a loaded DOM instead of + * silently continuing on an empty page after the timeout. + */ +export async function settleLoad(page: Page): Promise { + await page + .waitForLoadState("networkidle", { timeout: 8_000 }) + .catch(() => page.waitForLoadState("domcontentloaded").catch(() => {})); +} + +/** + * Re-extract `text` + `title` once when either came back empty: give the page + * one more settle (domcontentloaded + realtime settle) then read again. No-op + * when the first pass already produced content. + * + * @returns The original values, or the re-read ones when the first were empty. + */ +export async function reExtractIfEmpty( + page: Page, + text: string, + title: string, +): Promise<{ text: string; title: string }> { + if (text.length > 0 && title.trim().length > 0) return { text, title }; + await page.waitForLoadState("domcontentloaded").catch(() => {}); + await waitForRealtimeSettle(page, 4_000).catch(() => {}); + return { text: await mainText(page), title: await page.title() }; +} diff --git a/src/agent/run-steps.ts b/src/agent/run-steps.ts index 29bb5f3..5ae3b83 100644 --- a/src/agent/run-steps.ts +++ b/src/agent/run-steps.ts @@ -11,7 +11,7 @@ import { detectChallenges } from "../extraction/challenges.js"; import { extractHotelOffers } from "../extraction/hotel-offers.js"; import { mainText } from "../extraction/main-text.js"; import { extractPrices } from "../extraction/prices.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; /** A single step. `type` selects the operation; other fields are step args. */ export type RunStep = Record & { type: string }; @@ -51,7 +51,7 @@ async function runExtract(page: Page, step: RunStep): Promise { async function runOne(page: Page, step: RunStep, humanMode: boolean): Promise { if (step.type === "navigate") { - await gotoWithRetry(page, String(step.url), { waitUntil: "domcontentloaded", timeout: 30_000 }); + await gotoWithRetry(page, String(step.url), DEFAULT_GOTO); return { index: 0, type: "navigate", ok: true, data: { url: page.url(), title: await page.title() } }; } if (step.type === "wait_for") return runWait(page, step); diff --git a/src/agent/serp-batch.ts b/src/agent/serp-batch.ts index 0b9bff5..1b9d6c9 100644 --- a/src/agent/serp-batch.ts +++ b/src/agent/serp-batch.ts @@ -12,7 +12,7 @@ import type { SerpBatchRow } from "../interfaces/extraction.js"; import { sleep } from "../lib/retry.js"; import { randInt } from "../lib/text.js"; import { withBreaker } from "../net/breaker-guard.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; import type { ResolvedConfig } from "./config.js"; import { collectSerp } from "./serp-paged.js"; @@ -25,6 +25,8 @@ export interface SerpBatchOptions { gl?: string; /** Fixed inter-query delay (ms); defaults to a 2–4s jitter. */ delayMs?: number; + /** Called after each query settles (success or failure): `(done, total, query)`. */ + onProgress?: (done: number, total: number, label?: string) => void; } /** Build a Google search URL. */ @@ -46,12 +48,13 @@ export async function serpBatch(config: ResolvedConfig, opts: SerpBatchOptions): if (i > 0) await sleep(opts.delayMs ?? randInt(2_000, 4_000)); try { const url = googleUrl(query, opts.hl ?? "en", opts.gl ?? "us"); - await withBreaker(url, config.circuitBreaker, () => gotoWithRetry(page, url, { waitUntil: "domcontentloaded", timeout: 30_000 }, config.retry)); + await withBreaker(url, config.circuitBreaker, () => gotoWithRetry(page, url, DEFAULT_GOTO, config.retry)); const serp = await collectSerp(page, opts.pages ?? 1, config.retry); rows.push({ query, results: serp.organic, rank: opts.rankDomain ? findDomainRanks(serp, opts.rankDomain) : undefined }); } catch (error) { rows.push({ query, results: [], error: String(error) }); } + opts.onProgress?.(i + 1, opts.queries.length, query); } } finally { await teardownOpened(opened); diff --git a/src/agent/serp-paged.ts b/src/agent/serp-paged.ts index 914f14d..c86b769 100644 --- a/src/agent/serp-paged.ts +++ b/src/agent/serp-paged.ts @@ -7,7 +7,7 @@ import type { Page } from "playwright"; import { extractSerp } from "../extraction/serp.js"; import type { Serp } from "../interfaces/extraction.js"; import type { RetryConfig } from "../interfaces/net.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; /** Build the URL for SERP page offset `start`. */ function withStart(url: string, start: number): string { @@ -23,7 +23,7 @@ export async function collectSerp(page: Page, pages: number, retry: RetryConfig) const seen = new Set(merged.organic.map((r) => r.url)); const base = page.url(); for (let i = 1; i < pages; i += 1) { - await gotoWithRetry(page, withStart(base, i * 10), { waitUntil: "domcontentloaded", timeout: 30_000 }, retry); + await gotoWithRetry(page, withStart(base, i * 10), DEFAULT_GOTO, retry); const next = await extractSerp(page); for (const r of next.organic) { if (seen.has(r.url)) continue; diff --git a/src/agent/shots-batch.ts b/src/agent/shots-batch.ts index fe10f24..f977eaa 100644 --- a/src/agent/shots-batch.ts +++ b/src/agent/shots-batch.ts @@ -25,6 +25,7 @@ export type ShotsBatchItem = { url: string; shots: Shot[] } | { url: string; err * @param viewports - Viewports per URL. * @param settleMs - Settle delay before each capture. * @param concurrency - Max browsers in flight (default 2). + * @param onProgress - Called after each URL settles: `(done, total, url)`. * @returns One item per input URL, in order. */ export async function captureShotsBatch( @@ -33,13 +34,19 @@ export async function captureShotsBatch( viewports: ViewportInput[], settleMs?: number, concurrency?: number, + onProgress?: (done: number, total: number, label?: string) => void, ): Promise { const limit = concurrency && concurrency > 0 ? concurrency : DEFAULT_CONCURRENCY; const pool = new BrowserPool(config); + let done = 0; try { - const outcomes = await mapConcurrent(urls, limit, (url) => - pool.withPage((page) => shotsOnPage(page, config, url, viewports, settleMs)), - ); + const outcomes = await mapConcurrent(urls, limit, async (url) => { + try { + return await pool.withPage((page) => shotsOnPage(page, config, url, viewports, settleMs)); + } finally { + onProgress?.(++done, urls.length, url); + } + }); return outcomes.map((o, i) => (o.ok ? { url: urls[i] ?? "", shots: o.value } : { url: urls[i] ?? "", error: String(o.error) })); } finally { await pool.close(); diff --git a/src/agent/shots.ts b/src/agent/shots.ts index 18f7b28..62544d0 100644 --- a/src/agent/shots.ts +++ b/src/agent/shots.ts @@ -9,7 +9,7 @@ import { selectEngineForConfig } from "../engine/registry.js"; import { teardownOpened } from "../engine/teardown.js"; import { type ViewportInput, resolveViewport } from "../engine/viewport.js"; import { ensureDir, sha1 } from "../lib/fs.js"; -import { gotoWithRetry } from "../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; import { detectScrollJack, settleForCapture } from "../state/settle-capture.js"; import type { ResolvedConfig } from "./config.js"; import { captureFilmstrip } from "./filmstrip.js"; @@ -41,7 +41,7 @@ export async function shotsOnPage( ensureDir(config.outputDir); const runId = sha1(`${url}-shots`).slice(0, 10); const shots: Shot[] = []; - await gotoWithRetry(page, url, { waitUntil: "domcontentloaded", timeout: 30_000 }, config.retry); + await gotoWithRetry(page, url, DEFAULT_GOTO, config.retry); for (const v of viewports) { const size = resolveViewport(v); await page.setViewportSize(size); diff --git a/src/agent/site-shots.ts b/src/agent/site-shots.ts index 18696ab..be9a0b0 100644 --- a/src/agent/site-shots.ts +++ b/src/agent/site-shots.ts @@ -53,7 +53,7 @@ export function mergePagesWithShots( export async function siteShots(config: ResolvedConfig, seed: string, opts: SiteShotsOptions = {}): Promise<{ count: number; pages: SitePage[] }> { const { pages } = await crawl(seed, opts); const viewports = opts.viewports ?? parseViewports(undefined); - const shots = await captureShotsBatch(config, pages.map((p) => p.url), viewports, opts.settleMs, opts.shotsConcurrency); + const shots = await captureShotsBatch(config, pages.map((p) => p.url), viewports, opts.settleMs, opts.shotsConcurrency, opts.onProgress); const merged = mergePagesWithShots(pages, shots); return { count: merged.length, pages: merged }; } diff --git a/src/bin/cli-config.ts b/src/bin/cli-config.ts new file mode 100644 index 0000000..b938f1c --- /dev/null +++ b/src/bin/cli-config.ts @@ -0,0 +1,44 @@ +/** + * Map raw CLI flags (`parseArgs` values) into {@link AgentOptions} for the + * one-shot page commands (`run`, `products`, `extract`, `snapshot`, + * `screenshot`, `inspect`). Mirrors the flag→option conventions of the existing + * runners (probe-cli, shots-cli): `--headed` flips `headless` off, `--country` + * feeds `countryCode`, `--no-robots` disables robots, etc. + * @module bin/cli-config + */ +import type { EngineName } from "../interfaces/engine-types.js"; +import type { AgentOptions } from "../interfaces/types.js"; + +type Values = Record; + +/** Narrow an unknown flag value to a string (or undefined when absent/non-string). */ +const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + +/** + * Build {@link AgentOptions} from parsed CLI `values`. + * + * @param values - The `parseArgs` `values` object for the current invocation. + * @returns Agent options consumable by `resolveConfig`. + */ +export function cliAgentOptions(values: Values): AgentOptions { + const blockRaw = str(values["block-resources"]); + const blockResources = blockRaw + ? blockRaw + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : undefined; + return { + engine: str(values.engine) as EngineName | undefined, + countryCode: str(values.country), + currency: str(values.currency), + headless: !values.headed, + humanMode: Boolean(values["human-mode"]), + proxyUrl: str(values.proxy), + proxyMapPath: str(values["proxy-map"]), + outputDir: str(values["output-dir"]), + storageStatePath: str(values["storage-state"]), + respectRobots: values["no-robots"] ? false : undefined, + ...(blockResources ? { blockResources } : {}), + }; +} diff --git a/src/bin/cli-options.ts b/src/bin/cli-options.ts new file mode 100644 index 0000000..4637f82 --- /dev/null +++ b/src/bin/cli-options.ts @@ -0,0 +1,64 @@ +/** + * The `parseArgs` option schema for the `fuse-browser` CLI. Extracted from + * `cli.ts` so the entry point stays under the file-size limit. Every flag used + * by any subcommand (batch or one-shot page command) is declared here. + * @module bin/cli-options + */ +import type { parseArgs } from "node:util"; + +/** Option map type accepted by Node's `parseArgs`. */ +type ParseArgsOptions = NonNullable[0]>["options"]>; + +/** Full flag schema shared by every CLI subcommand. */ +export const CLI_OPTIONS: ParseArgsOptions = { + engine: { type: "string" }, + country: { type: "string" }, + currency: { type: "string" }, + headed: { type: "boolean" }, + "auto-consent": { type: "boolean" }, + "extract-prices": { type: "boolean" }, + "detect-challenges": { type: "boolean" }, + "observe-visual": { type: "boolean" }, + "extract-serp": { type: "boolean" }, + "serp-pages": { type: "string" }, + "rank-domain": { type: "string" }, + csv: { type: "boolean" }, + viewports: { type: "string" }, + "settle-ms": { type: "string" }, + hl: { type: "string" }, + gl: { type: "string" }, + "delay-ms": { type: "string" }, + "human-mode": { type: "boolean" }, + approved: { type: "boolean" }, + replay: { type: "boolean" }, + "wait-ms": { type: "string" }, + "output-dir": { type: "string" }, + "storage-state": { type: "string" }, + proxy: { type: "string" }, + "browser-fallback": { type: "boolean" }, + text: { type: "boolean" }, + format: { type: "string" }, + concurrency: { type: "string" }, + "max-pages": { type: "string" }, + "max-depth": { type: "string" }, + "all-origins": { type: "boolean" }, + "no-robots": { type: "boolean" }, + "throttle-ms": { type: "string" }, + item: { type: "string" }, + container: { type: "string" }, + "max-steps": { type: "string" }, + "proxy-map": { type: "string" }, + "user-data-dir": { type: "string" }, + "site-memory-dir": { type: "string" }, + click: { type: "string", multiple: true }, + fill: { type: "string", multiple: true }, + steps: { type: "string" }, + "steps-file": { type: "string" }, + kind: { type: "string" }, + ref: { type: "string" }, + "full-page": { type: "boolean" }, + selectors: { type: "boolean" }, + output: { type: "string" }, + limit: { type: "string" }, + "block-resources": { type: "string" }, +}; diff --git a/src/bin/cli-page-routes.ts b/src/bin/cli-page-routes.ts new file mode 100644 index 0000000..c3e4788 --- /dev/null +++ b/src/bin/cli-page-routes.ts @@ -0,0 +1,41 @@ +/** + * Routing for the one-shot page commands (`run`, `products`, `extract`, + * `snapshot`, `screenshot`, `inspect`). Kept apart from `cli.ts` so the entry + * point stays small and the existing batch routing is untouched. + * @module bin/cli-page-routes + */ +import { runExtractCli } from "./extract-cli.js"; +import { runInspectCli } from "./inspect-cli.js"; +import { runProductsCli } from "./products-cli.js"; +import { runRunCli } from "./run-cli.js"; +import { runScreenshotCli } from "./screenshot-cli.js"; +import { runSnapshotCli } from "./snapshot-cli.js"; + +type Values = Record; + +/** Handler for a one-shot page command: `(url, values) => Promise`. */ +type PageRunner = (url: string, values: Values) => Promise; + +const PAGE_COMMANDS: Record = { + run: runRunCli, + products: runProductsCli, + extract: runExtractCli, + snapshot: runSnapshotCli, + screenshot: runScreenshotCli, + inspect: runInspectCli, +}; + +/** + * Dispatch a one-shot page command if `command` matches and a URL is present. + * + * @param command - The subcommand token. + * @param rest - Remaining positionals (`rest[0]` is the URL). + * @param values - Parsed CLI flags. + * @returns True when handled (caller should stop routing); false otherwise. + */ +export async function routePageCommand(command: string, rest: string[], values: Values): Promise { + const runner = PAGE_COMMANDS[command]; + if (!runner || !rest[0]) return false; + await runner(rest[0], values); + return true; +} diff --git a/src/bin/cli-page.ts b/src/bin/cli-page.ts new file mode 100644 index 0000000..658c463 --- /dev/null +++ b/src/bin/cli-page.ts @@ -0,0 +1,36 @@ +/** + * Shared open→navigate→teardown harness for the one-shot page commands. Opens a + * browser context from the CLI flags, navigates to `url`, hands a live page to + * the caller, and always tears the context down in a `finally`. + * @module bin/cli-page + */ +import type { Page } from "playwright"; +import { resolveConfig } from "../agent/config.js"; +import { selectEngineForConfig } from "../engine/registry.js"; +import { teardownOpened } from "../engine/teardown.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; +import { cliAgentOptions } from "./cli-config.js"; + +type Values = Record; + +/** + * Open a page on `url`, run `fn`, then tear the context down. + * + * @param url - Target URL to navigate to before invoking `fn`. + * @param values - Parsed CLI flags (mapped via {@link cliAgentOptions}). + * @param fn - Receives the navigated page; its result is returned. + * @returns Whatever `fn` resolves to. + */ +export async function withCliPage(url: string, values: Values, fn: (page: Page) => Promise): Promise { + const config = resolveConfig(cliAgentOptions(values)); + const opened = await selectEngineForConfig(config).open(config); + const page = opened.page ?? (await opened.context.newPage()); + try { + await gotoWithRetry(page, url, DEFAULT_GOTO, config.retry); + const waitMs = values["wait-ms"] ? Number(values["wait-ms"]) : 0; + if (waitMs > 0) await page.waitForTimeout(waitMs); + return await fn(page); + } finally { + await teardownOpened(opened); + } +} diff --git a/src/bin/cli-usage.ts b/src/bin/cli-usage.ts new file mode 100644 index 0000000..e2cfcc2 --- /dev/null +++ b/src/bin/cli-usage.ts @@ -0,0 +1,43 @@ +/** + * The `fuse-browser` CLI help text. Kept in sync with the command routing in + * `cli.ts` — every routed subcommand is listed here. + * @module bin/cli-usage + */ + +/** Full `--help` text: every one-shot subcommand plus the common options. */ +export const CLI_USAGE = `fuse-browser [options] + +One-shot commands: + probe Open a page and report text, screenshot, prices, challenges, SERP + fetch Fast HTTP fetch (add --browser-fallback to retry in a browser) + fetch-batch Concurrent fetch of many URLs (--concurrency ) + crawl Crawl same-origin pages (--max-pages --max-depth ) + collect-batch Extract structured data from many URLs (--item --container) + serp-batch Google SERP scrape (--rank-domain --serp-pages --csv) + shots Screenshot across --viewports mobile,tablet,desktop + shots-batch Screenshot many URLs (--concurrency ) + site-shots Crawl a site and screenshot every page (--max-pages ) + +Page commands (one-shot, JSON on stdout): + run Execute a step plan (--steps '' | --steps-file ) + products Extract product cards (--limit --container ) + extract Pull page content (--kind text|prices|markdown) + snapshot List interactive elements (--selectors for CSS selectors) + screenshot Capture a PNG (--full-page, --output or base64) + inspect Computed style + WCAG contrast for one element (--ref ) + +Common options: + --engine playwright | patchright | firefox | webkit + --country Geo/locale identity (e.g. CH, FR) --currency e.g. CHF, EUR + --proxy Proxy URL --proxy-map Country→proxy map + --headed Show the browser --human-mode Bézier cursor + human timing + --extract-prices Parse visible prices --auto-consent Dismiss cookie/consent walls + --detect-challenges Flag captchas/anti-bot --wait-ms Settle delay + --output-dir Where to write shots/reports --storage-state Reuse auth + --text Raw text instead of markdown --format Output format + +Session-based interaction (open/navigate/click/products/autoscroll/…) is exposed +through the MCP server: run the \`browser-mcp\` binary (or \`npm run mcp\`). + +Flags: --help / -h, --version / -v +`; diff --git a/src/bin/cli.ts b/src/bin/cli.ts index 660a400..2cd2e1d 100644 --- a/src/bin/cli.ts +++ b/src/bin/cli.ts @@ -1,9 +1,15 @@ #!/usr/bin/env node /** - * fuse-browser CLI entry point. Subcommands: `probe`, `fetch`, `fetch-batch`, `serp-batch`, `shots`. + * fuse-browser CLI entry point. Subcommands: `probe`, `fetch`, `fetch-batch`, + * `crawl`, `collect-batch`, `serp-batch`, `shots`, `shots-batch`, `site-shots`, + * plus the one-shot page commands `run`, `products`, `extract`, `snapshot`, + * `screenshot`, `inspect`. * @module bin/cli */ import { handleMetaFlags, parseArgsOrExit } from "./cli-meta.js"; +import { CLI_OPTIONS } from "./cli-options.js"; +import { routePageCommand } from "./cli-page-routes.js"; +import { CLI_USAGE } from "./cli-usage.js"; import { runCollectBatchCli } from "./collect-batch-cli.js"; import { runCrawlCli } from "./crawl-cli.js"; import { runFetchBatchCli } from "./fetch-batch-cli.js"; @@ -14,58 +20,13 @@ import { runShotsBatch } from "./shots-batch-cli.js"; import { runShots } from "./shots-cli.js"; import { runSiteShotsCli } from "./site-shots-cli.js"; -const USAGE = - "usage: fuse-browser probe [...] | fetch [--extract-prices --proxy ] | fetch-batch [--concurrency ] | serp-batch --rank-domain | shots --viewports mobile,desktop\n"; - const argv = process.argv.slice(2); -handleMetaFlags(argv, USAGE); +handleMetaFlags(argv, CLI_USAGE); const { positionals, values } = parseArgsOrExit({ args: argv, allowPositionals: true, - options: { - engine: { type: "string" }, - country: { type: "string" }, - currency: { type: "string" }, - headed: { type: "boolean" }, - "auto-consent": { type: "boolean" }, - "extract-prices": { type: "boolean" }, - "detect-challenges": { type: "boolean" }, - "observe-visual": { type: "boolean" }, - "extract-serp": { type: "boolean" }, - "serp-pages": { type: "string" }, - "rank-domain": { type: "string" }, - csv: { type: "boolean" }, - viewports: { type: "string" }, - "settle-ms": { type: "string" }, - hl: { type: "string" }, - gl: { type: "string" }, - "delay-ms": { type: "string" }, - "human-mode": { type: "boolean" }, - approved: { type: "boolean" }, - replay: { type: "boolean" }, - "wait-ms": { type: "string" }, - "output-dir": { type: "string" }, - "storage-state": { type: "string" }, - proxy: { type: "string" }, - "browser-fallback": { type: "boolean" }, - text: { type: "boolean" }, - format: { type: "string" }, - concurrency: { type: "string" }, - "max-pages": { type: "string" }, - "max-depth": { type: "string" }, - "all-origins": { type: "boolean" }, - "no-robots": { type: "boolean" }, - "throttle-ms": { type: "string" }, - item: { type: "string" }, - container: { type: "string" }, - "max-steps": { type: "string" }, - "proxy-map": { type: "string" }, - "user-data-dir": { type: "string" }, - "site-memory-dir": { type: "string" }, - click: { type: "string", multiple: true }, - fill: { type: "string", multiple: true }, - }, + options: CLI_OPTIONS, }); const [command, ...rest] = positionals; @@ -89,7 +50,9 @@ if (command === "serp-batch") { await runFetchCli(rest[0], opts); } else if (command === "probe" && rest[0]) { await runProbeCli(rest[0], opts); +} else if (command && (await routePageCommand(command, rest, opts))) { + // Handled by a one-shot page command (run/products/extract/snapshot/screenshot/inspect). } else { - process.stderr.write(USAGE); + process.stderr.write(CLI_USAGE); process.exit(1); } diff --git a/src/bin/extract-cli.ts b/src/bin/extract-cli.ts new file mode 100644 index 0000000..21e33f5 --- /dev/null +++ b/src/bin/extract-cli.ts @@ -0,0 +1,38 @@ +/** + * `extract` subcommand handler: pull `text`, `prices`, or `markdown` from a page. + * Prints `{url, kind, ...}` JSON. + * @module bin/extract-cli + */ +import type { Page } from "playwright"; +import { mainText } from "../extraction/main-text.js"; +import { extractPrices } from "../extraction/prices.js"; +import { htmlToMarkdown, renderMarkdown } from "../extraction/serialize/to-markdown.js"; +import { withCliPage } from "./cli-page.js"; + +type Values = Record; +type Kind = "text" | "prices" | "markdown"; + +/** Extract one payload of the requested `kind` from the live `page`. */ +async function extractKind(page: Page, kind: Kind): Promise> { + const url = page.url(); + if (kind === "prices") { + const prices = extractPrices(await mainText(page)); + return { url, kind, count: prices.length, prices }; + } + if (kind === "markdown") { + const doc = await htmlToMarkdown(await page.content(), { url }); + return { url, kind, meta: doc.meta, markdown: renderMarkdown(doc) }; + } + return { url, kind, text: await mainText(page) }; +} + +/** Run the `extract` subcommand against `url`. */ +export async function runExtractCli(url: string, values: Values): Promise { + const raw = typeof values.kind === "string" ? values.kind : "text"; + if (raw !== "text" && raw !== "prices" && raw !== "markdown") { + process.stderr.write("extract: --kind must be text | prices | markdown\n"); + process.exit(2); + } + const result = await withCliPage(url, values, (page) => extractKind(page, raw)); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} diff --git a/src/bin/inspect-cli.ts b/src/bin/inspect-cli.ts new file mode 100644 index 0000000..5ee0d80 --- /dev/null +++ b/src/bin/inspect-cli.ts @@ -0,0 +1,34 @@ +/** + * `inspect` subcommand handler: capture the snapshot (which tags elements with + * `data-fuse-ref`) then report computed style + box + WCAG contrast for the + * `--ref` element. Prints `{url, ref, style}` JSON. + * @module bin/inspect-cli + */ +import type { Page } from "playwright"; +import { captureSnapshot } from "../extraction/snapshot.js"; +import { inspectStyle } from "../extraction/style-probe.js"; +import { withCliPage } from "./cli-page.js"; + +type Values = Record; + +/** Snapshot then inspect the element identified by `ref`. */ +async function snapshotAndInspect(page: Page, ref: string): Promise> { + await captureSnapshot(page); + const style = await inspectStyle(page, ref); + return { url: page.url(), ref, style }; +} + +/** Run the `inspect` subcommand against `url`. */ +export async function runInspectCli(url: string, values: Values): Promise { + const ref = typeof values.ref === "string" ? values.ref : undefined; + if (!ref) { + process.stderr.write("inspect: --ref is required (from snapshot)\n"); + process.exit(2); + } + const result = await withCliPage(url, values, (page) => snapshotAndInspect(page, ref)); + if (!result.style) { + process.stdout.write(`${JSON.stringify({ ...result, error: "ref_not_found" }, null, 2)}\n`); + process.exit(1); + } + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} diff --git a/src/bin/products-cli.ts b/src/bin/products-cli.ts new file mode 100644 index 0000000..a986920 --- /dev/null +++ b/src/bin/products-cli.ts @@ -0,0 +1,18 @@ +/** + * `products` subcommand handler: extract repeated product cards from a page. + * Prints `{url, count, products}` JSON. + * @module bin/products-cli + */ +import { extractProducts } from "../extraction/products.js"; +import { withCliPage } from "./cli-page.js"; + +type Values = Record; +const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + +/** Run the `products` subcommand against `url`. */ +export async function runProductsCli(url: string, values: Values): Promise { + const limit = values.limit ? Number(values.limit) : undefined; + const containerSelector = str(values.container); + const products = await withCliPage(url, values, (page) => extractProducts(page, { limit, containerSelector })); + process.stdout.write(`${JSON.stringify({ url, count: products.length, products }, null, 2)}\n`); +} diff --git a/src/bin/run-cli.ts b/src/bin/run-cli.ts new file mode 100644 index 0000000..50d76d2 --- /dev/null +++ b/src/bin/run-cli.ts @@ -0,0 +1,54 @@ +/** + * `run` subcommand handler: execute a multi-step plan in one page session via + * {@link runSteps}. Steps come from `--steps ''` (inline) or + * `--steps-file ` (`-` = stdin). Prints `{ok, url, steps}` JSON. + * @module bin/run-cli + */ +import { readFileSync } from "node:fs"; +import { type RunStep, runSteps } from "../agent/run-steps.js"; +import { withCliPage } from "./cli-page.js"; + +type Values = Record; +const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + +/** Read the raw steps JSON from `--steps` or `--steps-file` (`-` = stdin). */ +function readStepsSource(values: Values): string { + const inline = str(values.steps); + if (inline) return inline; + const file = str(values["steps-file"]); + if (!file) { + process.stderr.write("run: provide --steps '' or --steps-file \n"); + process.exit(2); + } + return readFileSync(file === "-" ? 0 : file, "utf8"); +} + +/** Parse the steps JSON into an array, exiting 2 on malformed/non-array input. */ +function parseSteps(values: Values): RunStep[] { + let parsed: unknown; + try { + parsed = JSON.parse(readStepsSource(values)); + } catch (err) { + process.stderr.write(`run: malformed steps JSON: ${(err as Error).message}\n`); + process.exit(2); + } + if (!Array.isArray(parsed)) { + process.stderr.write("run: steps JSON must be an array\n"); + process.exit(2); + } + return parsed as RunStep[]; +} + +/** Run the `run` subcommand against `url`. */ +export async function runRunCli(url: string, values: Values): Promise { + const steps = parseSteps(values); + const humanMode = Boolean(values["human-mode"]); + const results = await withCliPage(url, values, (page) => runSteps(page, steps, humanMode)); + const failed = results.find((r) => !r.ok); + if (failed) { + const error = { kind: "step_failed", step: failed.index, message: failed.error ?? "failed" }; + process.stdout.write(`${JSON.stringify({ ok: false, url, error, steps: results }, null, 2)}\n`); + process.exit(1); + } + process.stdout.write(`${JSON.stringify({ ok: true, url, steps: results }, null, 2)}\n`); +} diff --git a/src/bin/screenshot-cli.ts b/src/bin/screenshot-cli.ts new file mode 100644 index 0000000..37860a3 --- /dev/null +++ b/src/bin/screenshot-cli.ts @@ -0,0 +1,24 @@ +/** + * `screenshot` subcommand handler: capture a PNG of a page. Writes to + * `--output ` (printing the path) or emits base64 JSON on stdout. Add + * `--full-page` for a full-page capture. + * @module bin/screenshot-cli + */ +import { withCliPage } from "./cli-page.js"; + +type Values = Record; +const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + +/** Run the `screenshot` subcommand against `url`. */ +export async function runScreenshotCli(url: string, values: Values): Promise { + const fullPage = Boolean(values["full-page"]); + const output = str(values.output); + const buffer = await withCliPage(url, values, (page) => + page.screenshot({ fullPage, ...(output ? { path: output } : {}) }), + ); + if (output) { + process.stdout.write(`${JSON.stringify({ url, path: output, bytes: buffer.length }, null, 2)}\n`); + return; + } + process.stdout.write(`${JSON.stringify({ url, base64: buffer.toString("base64") }, null, 2)}\n`); +} diff --git a/src/bin/snapshot-cli.ts b/src/bin/snapshot-cli.ts new file mode 100644 index 0000000..ea40e4c --- /dev/null +++ b/src/bin/snapshot-cli.ts @@ -0,0 +1,17 @@ +/** + * `snapshot` subcommand handler: capture the interactive-element snapshot of a + * page (add `--selectors` for per-element CSS selectors). Prints + * `{url, count, elements}` JSON. + * @module bin/snapshot-cli + */ +import { captureSnapshot } from "../extraction/snapshot.js"; +import { withCliPage } from "./cli-page.js"; + +type Values = Record; + +/** Run the `snapshot` subcommand against `url`. */ +export async function runSnapshotCli(url: string, values: Values): Promise { + const withSelectors = Boolean(values.selectors); + const elements = await withCliPage(url, values, (page) => captureSnapshot(page, withSelectors)); + process.stdout.write(`${JSON.stringify({ url, count: elements.length, elements }, null, 2)}\n`); +} diff --git a/src/consent/booking-currency.ts b/src/consent/booking-currency.ts index 61ff0e4..0d37040 100644 --- a/src/consent/booking-currency.ts +++ b/src/consent/booking-currency.ts @@ -6,8 +6,7 @@ import type { Page } from "playwright"; import { evalScript, evalScriptArg } from "../lib/evaluate.js"; import { waitForRealtimeSettle } from "../state/realtime.js"; -/** booking.com origin and cookie domain. */ -const BOOKING_ORIGIN = "https://www.booking.com"; +/** booking.com cookie domain. */ const BOOKING_COOKIE_DOMAIN = ".booking.com"; /** Header currency picker trigger selector. */ const CURRENCY_TRIGGER_SELECTOR = '[data-testid="header-currency-picker-trigger"]'; @@ -23,9 +22,6 @@ export const CURRENCY_LABELS: Record = { /** Currency codes derived from {@link CURRENCY_LABELS}. */ export const SUPPORTED_CURRENCIES = Object.keys(CURRENCY_LABELS); -const bookingSetupUrl = (currency: string): string => - `${BOOKING_ORIGIN}/?change_currency=1;selected_currency=${currency};top_currency=1`; - const bookingCookie = (name: string, value: string) => ({ name, value, @@ -46,7 +42,16 @@ const PICK_LABEL = `(labels) => { return true; }`; -/** Pre-position the currency via cookies + booking setup page. */ +/** + * Pre-position the currency by seeding the booking.com cookies BEFORE the real + * navigation (cur_curr is the one Booking actually reads). The earlier variant + * also did an intermediate `page.goto` to the booking.com homepage to "warm" the + * currency session, but that landed on the consent wall and left the page in a + * broken state, so the subsequent navigation to the target returned blank. The + * cookies + the in-URL `selected_currency` param apply the currency; when they + * miss, `selectBookingCurrency` (the header UI picker, run on the real page after + * navigation) corrects it — so no intermediate navigation is needed. + */ export async function prepareBookingCurrency(page: Page, currency: string): Promise { try { await page.context().addCookies([ @@ -54,8 +59,6 @@ export async function prepareBookingCurrency(page: Page, currency: string): Prom bookingCookie("selected_currency", currency), bookingCookie("changed_currency", "1"), ]); - await page.goto(bookingSetupUrl(currency), { waitUntil: "domcontentloaded", timeout: 20_000 }); - await page.waitForTimeout(800); } catch { /* best-effort */ } diff --git a/src/engine/configured-context.ts b/src/engine/configured-context.ts index 3c974ff..5a49cac 100644 --- a/src/engine/configured-context.ts +++ b/src/engine/configured-context.ts @@ -7,6 +7,7 @@ import { existsSync } from "node:fs"; import type { Browser, BrowserContext } from "playwright"; import type { ResolvedConfig } from "../agent/config.js"; +import { applyResourceBlocking } from "../net/block.js"; import { buildContextOptions } from "./context.js"; /** Cached coherent UA per browser (one real-UA read per browser process). */ @@ -55,5 +56,7 @@ export async function newConfiguredContext(browser: Browser, config: ResolvedCon const ua = await coherentUserAgent(browser); if (ua) contextOptions.userAgent = ua; } - return browser.newContext(contextOptions); + const context = await browser.newContext(contextOptions); + if (config.blockResources?.length) await applyResourceBlocking(context, config.blockResources); + return context; } diff --git a/src/extraction/main-text-clean.ts b/src/extraction/main-text-clean.ts new file mode 100644 index 0000000..59cba54 --- /dev/null +++ b/src/extraction/main-text-clean.ts @@ -0,0 +1,57 @@ +/** + * In-page innerText reader that subtracts non-content landmarks (nav, header, + * footer, aside, search/filter forms) before reading. The browser-side logic is + * authored as a string script (per {@link module:lib/evaluate}) so it stays + * isolated from Node DOM typing, and operates on cloned nodes so the live DOM + * is never mutated. + * @module extraction/main-text-clean + */ +import type { Page } from "playwright"; +import { evalScriptArg } from "../lib/evaluate.js"; + +/** Sub-trees stripped from every matched landmark before reading innerText. */ +const STRIP_SELECTORS = [ + "nav", + "header", + "footer", + "aside", + "[role=navigation]", + "[role=search]", + "[role=complementary]", + "[role=banner]", + "[role=contentinfo]", + "form[role=search]", + "[aria-label*='filter' i]", + "[data-testid*='filter' i]", + "[class*='filter' i]", + "[id*='filter' i]", +].join(","); + +/** + * Browser script: for each node matching `sel`, clone it, remove every `strip` + * sub-tree from the clone, then read its innerText. Joins all matches so a + * product grid of repeated `
` cards keeps every card. Returns `""` + * when nothing matches, letting the caller fall through to the next selector. + */ +const SCRIPT = `({ sel, strip }) => { + const nodes = Array.from(document.querySelectorAll(sel)); + if (nodes.length === 0) return ""; + return nodes.map((node) => { + const clone = node.cloneNode(true); + for (const junk of Array.from(clone.querySelectorAll(strip))) junk.remove(); + return clone.innerText || ""; + }).join("\\n").trim(); +}`; + +/** + * Read the joined, cleaned innerText of every node matching `selector`. + * @param page - Playwright page to evaluate against. + * @param selector - Landmark selector (e.g. `"main"`, `"article"`). + * @returns Newline-joined, trimmed text of all cleaned matches (`""` if none). + */ +export function cleanedInnerText(page: Page, selector: string): Promise { + return evalScriptArg(page, SCRIPT, { + sel: selector, + strip: STRIP_SELECTORS, + }); +} diff --git a/src/extraction/main-text.ts b/src/extraction/main-text.ts index af42a37..b10b457 100644 --- a/src/extraction/main-text.ts +++ b/src/extraction/main-text.ts @@ -5,22 +5,29 @@ * @module extraction/main-text */ import type { Page } from "playwright"; +import { cleanedInnerText } from "./main-text-clean.js"; /** Main-content selectors, most to least specific; `body` is the fallback. */ const MAIN_SELECTORS = ["main", "[role=main]", "article"]; /** - * Return the trimmed innerText of the first matching main-content landmark, - * or the whole body if none is present. + * Return the trimmed text of the matching main-content landmark, or the whole + * body if none is present. Each match is cloned and stripped of non-content + * sub-trees (nav/header/footer/aside/search + filter containers) before its + * innerText is read, so e.g. a Booking filter sidebar no longer leaks its + * budget slider into the extracted text. ALL matches of the first hitting + * selector are joined (not just the first), so a product grid whose cards are + * repeated `
` elements yields every card's text — not only the first. + * A page with a single `
` is unaffected (one match → identical output). */ export async function mainText(page: Page, timeout = 3_000): Promise { for (const selector of MAIN_SELECTORS) { - const loc = page.locator(selector).first(); try { - if ((await loc.count()) > 0) return await loc.innerText({ timeout }); + const joined = await cleanedInnerText(page, selector); + if (joined) return joined; } catch { /* try the next selector */ } } - return page.locator("body").innerText({ timeout }); + return (await page.locator("body").innerText({ timeout })).trim(); } diff --git a/src/extraction/prices-context.ts b/src/extraction/prices-context.ts new file mode 100644 index 0000000..4333364 --- /dev/null +++ b/src/extraction/prices-context.ts @@ -0,0 +1,52 @@ +/** + * Derive a short context label for a detected price from its neighbouring + * logical lines (the nearest significant non-price line before or after it). + * @module extraction/prices-context + */ + +/** Max length of a context label; longer neighbours are truncated. */ +const MAX_CONTEXT = 80; + +/** True when a line carries no useful text (empty or punctuation only). */ +function isBlank(line: string): boolean { + return line.trim().replace(/[\s.,;:|–—-]/g, "") === ""; +} + +/** + * Scan outward from `index` in `step` direction for the first qualifying line: + * non-blank and not itself a detected price line. Returns the trimmed text or + * `undefined` when the edge is reached first. + */ +function scan( + lines: string[], + index: number, + step: -1 | 1, + priceLineNos: ReadonlySet, +): string | undefined { + for (let i = index + step; i >= 0 && i < lines.length; i += step) { + if (priceLineNos.has(i)) continue; + const text = (lines[i] as string).trim(); + if (!isBlank(text)) return text; + } + return undefined; +} + +/** + * Pick the nearest significant non-price line around `lines[index]` as context. + * Scans upward first, then downward; price lines and blank lines are skipped so + * a stack of prices still resolves to the surrounding label/title. The result + * is trimmed and capped at {@link MAX_CONTEXT} characters. + * @param lines - All logical lines, same index space as the price's `lineNo`. + * @param index - Index of the price's own line. + * @param priceLineNos - Set of indices that are themselves price lines. + * @returns A short context label, or `undefined` when none qualifies. + */ +export function contextFor( + lines: string[], + index: number, + priceLineNos: ReadonlySet, +): string | undefined { + const text = scan(lines, index, -1, priceLineNos) ?? scan(lines, index, 1, priceLineNos); + if (!text) return undefined; + return text.length > MAX_CONTEXT ? `${text.slice(0, MAX_CONTEXT).trimEnd()}…` : text; +} diff --git a/src/extraction/prices-normalize.ts b/src/extraction/prices-normalize.ts new file mode 100644 index 0000000..7989a08 --- /dev/null +++ b/src/extraction/prices-normalize.ts @@ -0,0 +1,60 @@ +/** + * Layout-agnostic price text normalisation: special-space cleanup, locale-aware + * amount parsing and logical-line stitching for currencies split across DOM nodes. + * @module extraction/prices-normalize + */ + +/** Unicode spaces (NBSP, NNBSP, thin, figure) collapsed to a plain space. */ +const SPECIAL_SPACES = /[\u00A0\u202F\u2009\u2007]/g; + +/** + * Replace special whitespace with a regular space (keeps newlines intact). + * @param text - Raw page text possibly containing NBSP/NNBSP/thin/figure spaces. + * @returns Text with special spaces normalised to U+0020. + */ +export function normaliseSpaces(text: string): string { + return text.replace(SPECIAL_SPACES, " "); +} + +/** + * Locale-aware amount parse. The last `.`/`,` separator is the decimal point + * when it precedes 1–2 trailing digits; otherwise every separator is a thousands + * group and stripped. Apostrophes (U+0027/U+2019) and spaces are always thousands. + * @param raw - Numeric string such as "1'234.56", "1.234,56", "129". + * @returns The parsed number (float when a decimal part is present). + */ +export function normaliseAmount(raw: string): number { + const cleaned = raw.replace(/['’ ]/g, ""); + const lastSep = Math.max(cleaned.lastIndexOf("."), cleaned.lastIndexOf(",")); + if (lastSep === -1) return Number(cleaned); + const tail = cleaned.slice(lastSep + 1); + if (tail.length >= 1 && tail.length <= 2 && /^\d+$/.test(tail)) { + return Number(`${cleaned.slice(0, lastSep).replace(/[.,]/g, "")}.${tail}`); + } + return Number(cleaned.replace(/[.,]/g, "")); +} + +/** + * Stitch logical lines so a currency token isolated on its own line is rejoined + * to the amount on the neighbouring line, in both directions: + * `"CHF\n6.90"` → `"CHF 6.90"` (prefix) and `"6.90\nCHF"` → `"6.90 CHF"` (suffix). + * @param lines - Physical lines (already space-normalised). + * @param currencyTokens - Single regex alternation built from CURRENCY_PREFIXES. + * @returns One logical line per physical line (same length and index mapping). + */ +export function stitchLogicalLines(lines: string[], currencyTokens: string): string[] { + const lone = new RegExp(`^\\s*(?:${currencyTokens})\\s*$`, "i"); + const hasDigit = /[0-9]/; + const out = lines.slice(); + for (let i = 0; i < lines.length; i++) { + if (!lone.test(lines[i] as string)) continue; + const prev = i > 0 ? (lines[i - 1] as string) : ""; + const next = i + 1 < lines.length ? (lines[i + 1] as string) : ""; + if (next && hasDigit.test(next) && !lone.test(next)) { + out[i] = `${(lines[i] as string).trim()} ${next.trim()}`; + } else if (prev && hasDigit.test(prev) && !lone.test(prev)) { + out[i] = `${prev.trim()} ${(lines[i] as string).trim()}`; + } + } + return out; +} diff --git a/src/extraction/prices.ts b/src/extraction/prices.ts index 9d4a76b..a746f6e 100644 --- a/src/extraction/prices.ts +++ b/src/extraction/prices.ts @@ -1,10 +1,16 @@ /** - * Multi-currency visible price extraction from page text. + * Multi-currency, layout-agnostic visible price extraction from page text. + * Captures a currency whether it sits before or after the amount, and even when + * the symbol and amount land on separate DOM lines (e.g. "CHF\n6.90"). * @module extraction/prices */ import type { Price } from "../interfaces/extraction.js"; +import { contextFor } from "./prices-context.js"; +import { normaliseAmount, normaliseSpaces, stitchLogicalLines } from "./prices-normalize.js"; -const AMOUNT = "([0-9][0-9'’.,]*)"; +export { normaliseAmount } from "./prices-normalize.js"; + +const AMOUNT = "([0-9][0-9'\u2019.,\u00A0\u202F\u2009\u2007 ]*[0-9]|[0-9])"; /** [currency code, regex prefix alternative]. Order matters: generic $ goes last. */ const CURRENCY_PREFIXES: Array<[string, string]> = [ @@ -34,35 +40,39 @@ const CURRENCY_PREFIXES: Array<[string, string]> = [ ["USD", "USD|US\\$|\\$"], ]; +/** Single alternation of every currency token, for the logical-line stitcher. */ +const ALL_TOKENS = CURRENCY_PREFIXES.map(([, prefix]) => prefix).join("|"); + +/** Prefix ("CHF 6.90") or suffix ("6.90 CHF"); suffix skipped when the currency is followed by an amount so "Row 0 CHF 10" → "CHF 10", not "0 CHF". */ const PATTERNS: Array<[string, RegExp]> = CURRENCY_PREFIXES.map(([currency, prefix]) => [ currency, - new RegExp(`(?:${prefix})\\s*${AMOUNT}`, "gi"), + new RegExp(`(?:${prefix})\\s*${AMOUNT}|${AMOUNT}\\s*(?:${prefix})(?![A-Za-z])(?!\\s*[0-9])`, "gi"), ]); const SKIP_WORDS = ["restaurant", "parking", "breakfast", "déjeuner"]; -/** Normalize a raw amount: float when it has decimals, otherwise integer (separators stripped). */ -export function normaliseAmount(raw: string): number { - const cleaned = raw.replace(/['’]/g, ""); - if (/[.,]\d{2}$/.test(cleaned)) return Number(cleaned.replace(/,/g, "")); - return Number(cleaned.replace(/,/g, "").replace(/\./g, "")); +/** True for a numeric range like "20–30" / "20-30" (excluded from prices). */ +function isRange(line: string): boolean { + return (line.includes("–") || line.includes("-")) && /\d\s*[-–]\s*\d/.test(line); } /** - * Extract visible prices, skipping ranges (e.g. "20–30") and irrelevant lines + * Extract visible prices, skipping ranges and irrelevant lines * (restaurant, parking…). Deduplicates by currency+amount. + * @param text - Page (or row) innerText, possibly multi-line and split layout. + * @returns Detected prices, each tied to its trimmed logical line and index. */ export function extractPrices(text: string): Price[] { const prices: Price[] = []; const seen = new Set(); - const lines = text.split("\n"); - lines.forEach((line, lineNo) => { - if ((line.includes("–") || line.includes("-")) && /\d\s*[-–]\s*\d/.test(line)) return; - const lowered = line.toLowerCase(); - if (SKIP_WORDS.some((w) => lowered.includes(w))) return; + const physical = normaliseSpaces(text).split("\n"); + const logical = stitchLogicalLines(physical, ALL_TOKENS); + logical.forEach((line, lineNo) => { + if (isRange(line)) return; + if (SKIP_WORDS.some((w) => line.toLowerCase().includes(w))) return; for (const [currency, pattern] of PATTERNS) { for (const match of line.matchAll(pattern)) { - const amount = normaliseAmount(match[1] as string); + const amount = normaliseAmount((match[1] ?? match[2]) as string); const key = `${currency}:${amount}`; if (seen.has(key)) continue; seen.add(key); @@ -70,12 +80,20 @@ export function extractPrices(text: string): Price[] { } } }); + const priceLineNos = new Set(prices.map((p) => p.lineNo)); + for (const price of prices) { + const context = contextFor(logical, price.lineNo, priceLineNos); + if (context) price.context = context; + } return prices; } -/** First price of a line, or null. */ +/** + * First price of a single line, or null. Callers (hotel-offers) pre-split text. + * @param line - One already-isolated line of text. + * @returns The first detected currency/amount, or null when none is found. + */ export function parseSinglePrice(line: string): { currency: string; amount: number } | null { - const prices = extractPrices(line); - const first = prices[0]; + const first = extractPrices(line)[0]; return first ? { currency: first.currency, amount: first.amount } : null; } diff --git a/src/extraction/products-dom.ts b/src/extraction/products-dom.ts new file mode 100644 index 0000000..7663246 --- /dev/null +++ b/src/extraction/products-dom.ts @@ -0,0 +1,97 @@ +/** + * Pure, self-contained DOM heuristic for repeated product cards. All helpers + * are nested inside {@link collectProducts} so the whole function serializes + * cleanly for `page.evaluate`, yet runs identically against a linkedom + * Document in unit tests. Depends only on standard DOM APIs. + * @module extraction/products-dom + */ +import type { DomDocument, DomElement } from "../interfaces/dom.js"; +import type { Product, ProductsOptions } from "../interfaces/products.js"; + +/** + * Find repeated product cards in `document`. A card is the smallest element + * whose tag+class signature repeats ≥3 times AND whose AGGREGATED text holds + * both a price and a non-price title/link. Prices are matched on the container's + * full text (not a single leaf), so a currency token and amount split across + * separate DOM nodes — e.g. `CHF6.90` on React sites + * like digitec — are still detected. With `containerSelector`, the matched + * elements are the cards directly. Cards without a parseable price are dropped. + */ +export function collectProducts(document: DomDocument, opts: ProductsOptions = {}): Product[] { + const CURRENCIES: Array<[string, string]> = [ + ["CHF|Fr\\.?|SFr\\.?", "CHF"], + ["£|GBP", "GBP"], + ["€|EUR", "EUR"], + ["¥|JPY", "JPY"], + ["US\\$|USD|\\$", "USD"], + ]; + const AMOUNT = "([0-9][0-9'’., ]*[0-9]|[0-9])"; + // LAST `.`/`,` is the decimal when 1–2 trailing digits; else all are grouping. + // Mirrors prices-normalize#normaliseAmount, inlined (runs in-browser, no imports). + const toNumber = (raw: string): number => { + const c = raw.replace(/[^0-9.,]/g, ""); + const sep = Math.max(c.lastIndexOf("."), c.lastIndexOf(",")); + const tail = c.slice(sep + 1); + const dec = sep >= 0 && tail.length >= 1 && tail.length <= 2; + return Number.parseFloat(dec ? `${c.slice(0, sep).replace(/[.,]/g, "")}.${tail}` : c.replace(/[.,]/g, "")); + }; + // Aggregated, space-collapsed text of an element (currency + amount joined + // even when innerText inserts a newline between two child nodes). + const flat = (el: DomElement): string => (el.textContent || "").replace(/\s+/g, " ").trim(); + const parsePrice = (text: string): { currency: string; price: number } | null => { + for (const [tok, code] of CURRENCIES) { + const m = new RegExp(`(?:${tok})\\s*${AMOUNT}|${AMOUNT}\\s*(?:${tok})`, "i").exec(text); + if (m) { + const price = toNumber(m[1] ?? m[2] ?? ""); + if (Number.isFinite(price)) return { currency: code, price }; + } + } + return null; + }; + const signature = (el: DomElement): string => { + const cls = (el.getAttribute("class") || "").trim().split(/\s+/).filter(Boolean).sort().join("."); + return cls ? `${el.tagName}.${cls}` : el.tagName; + }; + // Longest non-price text from any link/heading/aria-label inside `card` ("" if none). + const bestTitle = (card: DomElement, priceText: string): string => { + let best = ""; + for (const el of [...card.querySelectorAll("a, h1, h2, h3, h4, [aria-label]")]) { + const t = (el.getAttribute("aria-label") || el.textContent || "").replace(/\s+/g, " ").trim(); + if (t && t !== priceText && !parsePrice(t) && t.length > best.length) best = t; + } + return best; + }; + const urlOf = (card: DomElement): string | undefined => { + const a = card.matches("a") ? card : card.querySelector("a"); + const href = a?.href || a?.getAttribute("href") || ""; + return href && !href.startsWith("javascript:") ? href : undefined; + }; + const toProduct = (card: DomElement): Product | null => { + const text = flat(card); + const hit = parsePrice(text); + if (!hit) return null; + const url = urlOf(card); + const name = bestTitle(card, text) || flat(card).slice(0, 120); + return { title: name, price: hit.price, currency: hit.currency, ...(url ? { url } : {}) }; + }; + let cards: DomElement[]; + if (opts.containerSelector) { + cards = [...document.querySelectorAll(opts.containerSelector)]; + } else { + const all = [...document.querySelectorAll("*")]; + const counts = new Map(); + for (const el of all) counts.set(signature(el), (counts.get(signature(el)) ?? 0) + 1); + // Candidate = repeated (≥3) container whose aggregated text holds price + title. + const candidates: DomElement[] = []; + for (const el of all) { + if ((counts.get(signature(el)) ?? 0) < 3) continue; + const text = flat(el); + if (parsePrice(text) && bestTitle(el, text)) candidates.push(el); + } + // Keep the SMALLEST card per cluster: drop any candidate containing another + // (nested grids → inner card wins, avoids matching the whole grid wrapper). + cards = candidates.filter((c) => !candidates.some((o) => o !== c && c.contains(o))); + } + const out = cards.map(toProduct).filter((p): p is Product => p !== null); + return opts.limit ? out.slice(0, opts.limit) : out; +} diff --git a/src/extraction/products.ts b/src/extraction/products.ts new file mode 100644 index 0000000..9830fdd --- /dev/null +++ b/src/extraction/products.ts @@ -0,0 +1,33 @@ +/** + * Structured product-card extraction from the live DOM. Detects repeated card + * containers that hold both a price and a title/link, returning one + * `{title, price, currency, url?}` per card. Generic across digitec, booking, + * amazon… because it groups by repeated tag+class signatures, not site CSS. + * The heuristic lives in {@link collectProducts} (pure, unit-tested); here it is + * serialized into the page so it runs on the rendered DOM after hydration. + * @module extraction/products + */ +import type { Page } from "playwright"; +import type { Product, ProductsOptions } from "../interfaces/products.js"; +import { evalScriptArg } from "../lib/evaluate.js"; +import { collectProducts } from "./products-dom.js"; + +/** + * Browser script: run the self-contained collector against `document`. + * A passthrough `__name` shim is injected first because tsx/esbuild rewrites + * named functions with `__name(fn, "…")` for `keepNames`; that helper is absent + * in the page, so without the shim the serialized source throws + * `ReferenceError: __name is not defined`. The shim is a harmless no-op when the + * source carries no such call (production/compiled builds). + */ +const SCRIPT = `(opts) => { var __name = (f) => f; return (${collectProducts.toString()})(document, opts); }`; + +/** + * Extract repeated product cards from the page's rendered DOM. + * @param page - A live Playwright page. + * @param opts - Optional limit / forced container selector. + * @returns One product per detected card (price-bearing cards only). + */ +export function extractProducts(page: Page, opts: ProductsOptions = {}): Promise { + return evalScriptArg(page, SCRIPT, opts); +} diff --git a/src/extraction/structured-cards.ts b/src/extraction/structured-cards.ts new file mode 100644 index 0000000..3d8b984 --- /dev/null +++ b/src/extraction/structured-cards.ts @@ -0,0 +1,55 @@ +/** + * Per-card variant of schema extraction: one record per repeated container, + * with every field resolved RELATIVE to that container. Keeps a card's title, + * price and link correlated, unlike the index-aligned parallel arrays of + * {@link extractStructured}. + * @module extraction/structured-cards + */ +import type { Page } from "playwright"; +import { evalScriptArg } from "../lib/evaluate.js"; +import type { ExtractedValue, ExtractionSchema } from "./structured.js"; + +// Each `container` match is scoped; fields use querySelector ON the container +// (not the document), so values stay correlated per card. `selector` is data, +// not interpolated; an invalid selector throws and is caught per field. +const CARDS_SCRIPT = `(a) => { + const read = (el, f) => { + if (!el) return null; + if (f.abs && f.attr) return el[f.attr] ?? null; + if (f.attr) return el.getAttribute(f.attr); + return (el.innerText || '').trim() || null; + }; + return [...document.querySelectorAll(a.container)].map((root) => { + const out = {}; + for (const key in a.fields) { + const f = a.fields[key]; + try { + out[key] = f.all + ? [...root.querySelectorAll(f.selector)].map((el) => read(el, f)) + : read(root.querySelector(f.selector), f); + } catch { + out[key] = null; + } + } + return out; + }); +}`; + +/** + * Read `schema` fields card-by-card: one record per `container` match. + * @param page - A live Playwright page. + * @param container - CSS selector matching each repeated card. + * @param schema - Field map; selectors resolve relative to each card. + * @returns One typed record per matched container. + */ +export async function extractStructuredPerCard( + page: Page, + container: string, + schema: ExtractionSchema, +): Promise>> { + return evalScriptArg>, { container: string; fields: ExtractionSchema }>( + page, + CARDS_SCRIPT, + { container, fields: schema }, + ); +} diff --git a/src/extraction/structured.ts b/src/extraction/structured.ts index 0d67987..27969ed 100644 --- a/src/extraction/structured.ts +++ b/src/extraction/structured.ts @@ -56,3 +56,5 @@ export async function extractStructured( ): Promise> { return evalScriptArg, ExtractionSchema>(page, READ_SCRIPT, schema); } + +export { extractStructuredPerCard } from "./structured-cards.js"; diff --git a/src/identity/profiles.ts b/src/identity/profiles.ts new file mode 100644 index 0000000..649f1fa --- /dev/null +++ b/src/identity/profiles.ts @@ -0,0 +1,33 @@ +/** + * Named persistent auth profiles: map a short profile name to the + * storage-state JSON it lives in (`/profiles/.json`). The save + * (session close / probe run) and load (configured context) cycle already + * works on `storageStatePath`; profiles are just a friendly path resolver. + * @module identity/profiles + */ +import { homedir } from "node:os"; +import { join } from "node:path"; + +/** Allowed profile names: alnum start, then alnum/`-`/`_`, 1-41 chars. */ +const PROFILE_NAME = /^[a-z0-9][a-z0-9_-]{0,40}$/i; + +/** Fuse-browser home dir (`FUSE_BROWSER_HOME` override, else `~/.fuse-browser`). */ +function fuseBrowserHome(): string { + return process.env.FUSE_BROWSER_HOME ?? join(homedir(), ".fuse-browser"); +} + +/** + * Resolve the storage-state file for a named auth profile. + * + * @param name - Profile name (letters/digits, then `-`/`_` allowed, max 41 chars). + * @returns Absolute path `/profiles/.json`. + * @throws Error when the name is empty or contains disallowed characters. + */ +export function profileStoragePath(name: string): string { + if (!PROFILE_NAME.test(name)) { + throw new Error( + `Invalid profile name "${name}" — use 1-41 chars: letters/digits, then "-" or "_" (must start with a letter or digit).`, + ); + } + return join(fuseBrowserHome(), "profiles", `${name}.json`); +} diff --git a/src/interfaces/auto-scroll.ts b/src/interfaces/auto-scroll.ts new file mode 100644 index 0000000..bf7d4ee --- /dev/null +++ b/src/interfaces/auto-scroll.ts @@ -0,0 +1,42 @@ +/** + * Types for the auto-scroll action. + * @module interfaces/auto-scroll + */ + +/** Tuning options for `autoScroll`. */ +export interface AutoScrollOpts { + /** Hard cap on scroll rounds (default 20). */ + maxScrolls?: number; + /** Consecutive rounds without growth before stopping (default 2). */ + idleRounds?: number; + /** CSS selector whose element count is the stop signal. */ + untilSelector?: string; + /** Element count that satisfies `untilSelector` (default 1). */ + minCount?: number; + /** Delay between scrolls in ms (default 600). */ + delayMs?: number; +} + +/** Outcome of one scroll round, measured in the page. */ +export interface ScrollProbe { + height: number; + count: number; +} + +/** Inputs for the pure stop decision. */ +export interface StopInput { + prev: ScrollProbe | null; + curr: ScrollProbe; + idle: number; + rounds: number; + idleRounds: number; + maxScrolls: number; + minCount: number; + hasSelector: boolean; +} + +/** Next loop state after evaluating a round. */ +export interface StopState { + stop: boolean; + idle: number; +} diff --git a/src/interfaces/dom.ts b/src/interfaces/dom.ts new file mode 100644 index 0000000..b6930a2 --- /dev/null +++ b/src/interfaces/dom.ts @@ -0,0 +1,30 @@ +/** + * Minimal structural DOM types. The project's tsconfig omits the `dom` lib on + * purpose, yet some pure helpers walk a DOM that exists both in the browser + * (via `page.evaluate`) and in tests (via linkedom). These interfaces cover + * only the read-only members those helpers touch, so the code stays lib-free + * while remaining type-checked. linkedom and browser nodes both satisfy them. + * @module interfaces/dom + */ + +/** Read-only element surface used by DOM-walking heuristics. */ +export interface DomElement { + readonly tagName: string; + readonly parentElement: DomElement | null; + readonly children: { readonly length: number }; + readonly textContent: string | null; + /** Present on anchors; the resolved absolute URL. */ + readonly href?: string; + getAttribute(name: string): string | null; + matches(selector: string): boolean; + /** True when `other` is this element or a descendant of it. */ + contains(other: DomElement): boolean; + querySelector(selector: string): DomElement | null; + querySelectorAll(selector: string): Iterable; +} + +/** Read-only document surface: just the query roots the heuristics need. */ +export interface DomDocument { + querySelector(selector: string): DomElement | null; + querySelectorAll(selector: string): Iterable; +} diff --git a/src/interfaces/extraction.ts b/src/interfaces/extraction.ts index 05ef445..bad4f29 100644 --- a/src/interfaces/extraction.ts +++ b/src/interfaces/extraction.ts @@ -9,6 +9,12 @@ export interface Price { amount: number; line: string; lineNo: number; + /** + * Best-effort short context label for the price: the nearest significant + * non-price neighbouring line (e.g. "2 nights, 2 adults", "Tickets from", + * or a product title). Optional and back-compatible. + */ + context?: string; } /** An aggregated hotel offer (provider + price). */ diff --git a/src/interfaces/options.ts b/src/interfaces/options.ts index 09f5397..63d414c 100644 --- a/src/interfaces/options.ts +++ b/src/interfaces/options.ts @@ -21,6 +21,10 @@ export interface AgentOptions { /** Timeout (ms) for the CDP connect. Default 20000. */ cdpTimeoutMs?: number; storageStatePath?: string; + /** Named persistent auth profile → storage state at `~/.fuse-browser/profiles/.json` (ignored when `storageStatePath` is set). */ + profile?: string; + /** Resource types to abort at the network layer (image, media, font, ... — unknown types ignored). */ + blockResources?: string[]; /** HAR: record traffic to `harPath` (`harMode` minimal/full), or replay from `harReplay`. */ harPath?: string; harMode?: "minimal" | "full"; diff --git a/src/interfaces/products.ts b/src/interfaces/products.ts new file mode 100644 index 0000000..7afb664 --- /dev/null +++ b/src/interfaces/products.ts @@ -0,0 +1,24 @@ +/** + * Types for structured product-card extraction (e-commerce / listings). + * @module interfaces/products + */ + +/** One product card: a repeated container holding both a price and a title/link. */ +export interface Product { + /** Card title / product name (best text node, never the raw price). */ + title: string; + /** Detected numeric amount (dot decimal, thousands stripped). */ + price: number; + /** ISO-ish currency code (CHF, EUR, USD, GBP…). */ + currency: string; + /** Absolute product URL when the card wraps or contains an anchor. */ + url?: string; +} + +/** Options for {@link extractProducts}. */ +export interface ProductsOptions { + /** Cap the number of returned cards (default: all). */ + limit?: number; + /** Force the repeated-card container selector instead of auto-detecting it. */ + containerSelector?: string; +} diff --git a/src/lib/safe-png.ts b/src/lib/safe-png.ts new file mode 100644 index 0000000..5e790c1 --- /dev/null +++ b/src/lib/safe-png.ts @@ -0,0 +1,25 @@ +/** + * Validation for client-supplied PNG file paths: rejects empty strings, + * NUL bytes, and non-`.png` extensions before any filesystem access. + * @module lib/safe-png + */ +import path from "node:path"; + +/** + * Assert that `p` is a plausible PNG path: non-empty, NUL-free, and ending + * in `.png` (case-insensitive) after `path.normalize()`. + * + * @param p - Raw path supplied by the MCP client. + * @param label - Field name used in error messages (e.g. `"baseline"`). + * @returns The normalized, resolved absolute path. + * @throws Error when the path is empty, contains a NUL byte, or does not end with `.png`. + */ +export function assertPngPath(p: string, label: string): string { + if (p === "") throw new Error(`${label}: path must be a non-empty string`); + if (p.includes("\0")) throw new Error(`${label}: path must not contain NUL characters`); + const normalized = path.normalize(p); + if (!normalized.toLowerCase().endsWith(".png")) { + throw new Error(`${label}: path must end with .png (got "${p}")`); + } + return path.resolve(normalized); +} diff --git a/src/net/block.ts b/src/net/block.ts new file mode 100644 index 0000000..025052b --- /dev/null +++ b/src/net/block.ts @@ -0,0 +1,38 @@ +/** + * Resource-type blocking on a browser context: abort requests whose Playwright + * `resourceType()` is in the blocked set, let everything else fall through to + * the next route handler (HAR replay, etc.). Speeds up batch runs by skipping + * images/fonts/media that extraction never reads. + * @module net/block + */ +import type { BrowserContext } from "playwright"; + +/** Playwright resource types that can be blocked. Unknown inputs are ignored. */ +export const BLOCKABLE_RESOURCE_TYPES = new Set([ + "image", + "media", + "font", + "stylesheet", + "script", + "xhr", + "fetch", + "websocket", + "manifest", + "other", +]); + +/** + * Install a catch-all route that aborts blocked resource types. + * Unknown type names are silently dropped; if nothing valid remains, + * no route is installed at all (zero overhead). + * + * @param context - Context to intercept (route is context-wide). + * @param types - Resource types to block (case-insensitive). + */ +export async function applyResourceBlocking(context: BrowserContext, types: string[]): Promise { + const blocked = new Set(types.map((t) => t.toLowerCase()).filter((t) => BLOCKABLE_RESOURCE_TYPES.has(t))); + if (blocked.size === 0) return; + await context.route("**/*", (route) => + blocked.has(route.request().resourceType()) ? route.abort() : route.fallback(), + ); +} diff --git a/src/net/navigate.ts b/src/net/navigate.ts index 513cbf4..4ea9b53 100644 --- a/src/net/navigate.ts +++ b/src/net/navigate.ts @@ -28,6 +28,9 @@ export interface GotoOptions { timeout?: number; } +/** Default goto options shared by every navigation call site. */ +export const DEFAULT_GOTO: GotoOptions = { waitUntil: "domcontentloaded", timeout: 30_000 }; + /** Navigate to `url`, retrying transient failures and rate-limit responses. */ export function gotoWithRetry( page: Page, diff --git a/src/server/caps.ts b/src/server/caps.ts new file mode 100644 index 0000000..d1aa7a8 --- /dev/null +++ b/src/server/caps.ts @@ -0,0 +1,36 @@ +/** + * Tool capability groups: opt-in filtering of which MCP tool groups are + * registered, via the `FUSE_CAPS` env var (comma-separated). Fewer exposed + * tools = a lighter context for the LLM client. Default: all groups. + * @module server/caps + */ + +/** Every tool capability group, in registration order. */ +export const CAP_GROUPS = ["core", "batch", "extract", "debug", "live"] as const; + +/** A single capability group name. */ +export type CapGroup = (typeof CAP_GROUPS)[number]; + +/** + * Parse a `FUSE_CAPS` value (e.g. `"core,extract"`) into the set of enabled + * groups. Blank/undefined enables everything; unknown names are reported on + * stderr (stdout is reserved for JSON-RPC) and ignored. An input with only + * unknown names falls back to all groups rather than an empty server. + * + * @param raw - The raw env value (comma-separated group names). + * @returns The set of enabled capability groups. + */ +export function parseCaps(raw: string | undefined): Set { + const all = new Set(CAP_GROUPS); + const names = (raw ?? "") + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + if (names.length === 0) return all; + const enabled = new Set(); + for (const name of names) { + if ((CAP_GROUPS as readonly string[]).includes(name)) enabled.add(name as CapGroup); + else console.error(`fuse-browser: unknown FUSE_CAPS group "${name}" (known: ${CAP_GROUPS.join(", ")})`); + } + return enabled.size > 0 ? enabled : all; +} diff --git a/src/server/map-options.ts b/src/server/map-options.ts index a25b052..0e8ab1a 100644 --- a/src/server/map-options.ts +++ b/src/server/map-options.ts @@ -2,6 +2,7 @@ * Map raw tool arguments to typed agent/probe options. * @module server/map-options */ +import { profileStoragePath } from "../identity/profiles.js"; import type { BrowserChannel } from "../interfaces/engine-types.js"; import type { AgentOptions, BrowserAction, ProbeOptions } from "../interfaces/types.js"; import { envAgentDefaults } from "./env-defaults.js"; @@ -9,6 +10,15 @@ import { envAgentDefaults } from "./env-defaults.js"; /** Server-wide browser defaults from `FUSE_*` env (per-call args override these). */ const ENV = envAgentDefaults(); +/** Storage state: explicit path beats named `profile`; env default is the last fallback. */ +function storageStateFrom(a: Record): string | undefined { + const explicit = a.storageStatePath as string | undefined; + if (explicit) return explicit; + const profile = a.profile as string | undefined; + if (profile) return profileStoragePath(profile); + return ENV.storageStatePath; +} + /** Extract {@link AgentOptions} from raw tool arguments, falling back to env. */ export function toAgentOptions(a: Record): AgentOptions { return { @@ -29,7 +39,9 @@ export function toAgentOptions(a: Record): AgentOptions { proxyUrl: a.proxyUrl as string | undefined, proxyMapPath: a.proxyMapPath as string | undefined, proxiesPath: a.proxiesPath as string | undefined, - storageStatePath: (a.storageStatePath as string | undefined) ?? ENV.storageStatePath, + storageStatePath: storageStateFrom(a), + profile: a.profile as string | undefined, + blockResources: a.blockResources as string[] | undefined, harPath: a.harPath as string | undefined, harMode: a.harMode as "minimal" | "full" | undefined, harReplay: a.harReplay as string | undefined, diff --git a/src/server/progress.ts b/src/server/progress.ts new file mode 100644 index 0000000..c708e10 --- /dev/null +++ b/src/server/progress.ts @@ -0,0 +1,35 @@ +/** + * MCP progress notifications for long-running batch tools. Builds a per-request + * reporter from the tool callback's `extra`: silent no-op when the client sent + * no `progressToken`, otherwise emits `notifications/progress` ("4/12") without + * ever throwing or blocking the batch. + * @module server/progress + */ +import type { RequestHandlerExtra } from "@modelcontextprotocol/sdk/shared/protocol.js"; +import type { ServerNotification, ServerRequest } from "@modelcontextprotocol/sdk/types.js"; + +/** The `extra` argument MCP tool callbacks receive from `registerTool`. */ +export type ToolExtra = RequestHandlerExtra; + +/** Progress callback: `done` items finished out of `total`, optional label. */ +export type ProgressFn = (done: number, total: number, message?: string) => void; + +/** + * Build a progress reporter bound to the current MCP request. + * + * @param extra - The tool callback's `extra` (carries `_meta.progressToken`). + * @returns A `(done, total, message?)` callback. No-op when the client did not + * request progress; otherwise fire-and-forget (send errors are swallowed). + */ +export function progressReporter(extra: ToolExtra): ProgressFn { + const progressToken = extra._meta?.progressToken; + if (progressToken === undefined) return () => {}; + return (done, total, message) => { + extra + .sendNotification({ + method: "notifications/progress", + params: { progressToken, progress: done, total, ...(message !== undefined ? { message } : {}) }, + }) + .catch(() => {}); + }; +} diff --git a/src/server/registry.ts b/src/server/registry.ts new file mode 100644 index 0000000..8c058fe --- /dev/null +++ b/src/server/registry.ts @@ -0,0 +1,85 @@ +/** + * Maps every MCP tool registration to its capability group. Used by the + * server factory to register only the groups enabled via `FUSE_CAPS`. + * @module server/registry + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { SessionManager } from "../session/manager.js"; +import type { CapGroup } from "./caps.js"; +import { registerActTools } from "./tools/act.js"; +import { registerAutoScrollTool } from "./tools/autoscroll.js"; +import { registerCollectTool } from "./tools/collect.js"; +import { registerCollectBatchTool } from "./tools/collect-batch.js"; +import { registerConnectTool } from "./tools/connect.js"; +import { registerCrawlTool } from "./tools/crawl.js"; +import { registerDialogTools } from "./tools/dialogs.js"; +import { registerExtractTool } from "./tools/extract.js"; +import { registerExtractSchemaTool } from "./tools/extract-schema.js"; +import { registerFetchTool } from "./tools/fetch.js"; +import { registerFetchBatchTool } from "./tools/fetch-batch.js"; +import { registerHandoffTool } from "./tools/handoff.js"; +import { registerInspectTool } from "./tools/inspect.js"; +import { registerLiveViewTool } from "./tools/live-view.js"; +import { registerLogTools } from "./tools/logs.js"; +import { registerMetricsTool } from "./tools/metrics.js"; +import { registerNavigateTool } from "./tools/navigate.js"; +import { registerProbeTools } from "./tools/probe.js"; +import { registerProductsTool } from "./tools/products.js"; +import { registerRunTool } from "./tools/run.js"; +import { registerScreenshotTool } from "./tools/screenshot.js"; +import { registerSerpBatchTool } from "./tools/serp-batch.js"; +import { registerSessionTools } from "./tools/session.js"; +import { registerShotsBatchTool } from "./tools/shots-batch.js"; +import { registerSiteShotsTool } from "./tools/site-shots.js"; +import { registerSnapshotTools } from "./tools/snapshot.js"; +import { registerTabsTool } from "./tools/tabs.js"; +import { registerVisualDiffTool } from "./tools/visual-diff.js"; +import { registerWaitTool } from "./tools/wait.js"; + +/** Build the per-group registration thunks for `server` + `sessions`. */ +export function toolGroups( + server: McpServer, + sessions: SessionManager, +): Record void>> { + return { + core: [ + () => registerSessionTools(server, sessions), + () => registerConnectTool(server, sessions), + () => registerNavigateTool(server, sessions), + () => registerActTools(server, sessions), + () => registerTabsTool(server, sessions), + () => registerDialogTools(server, sessions), + () => registerSnapshotTools(server, sessions), + () => registerWaitTool(server, sessions), + () => registerScreenshotTool(server, sessions), + () => registerAutoScrollTool(server, sessions), + ], + batch: [ + () => registerProbeTools(server), + () => registerFetchTool(server), + () => registerFetchBatchTool(server), + () => registerCrawlTool(server), + () => registerCollectBatchTool(server), + () => registerShotsBatchTool(server), + () => registerSiteShotsTool(server), + () => registerSerpBatchTool(server), + ], + extract: [ + () => registerCollectTool(server, sessions), + () => registerRunTool(server, sessions), + () => registerExtractTool(server, sessions), + () => registerExtractSchemaTool(server, sessions), + () => registerProductsTool(server, sessions), + ], + debug: [ + () => registerInspectTool(server, sessions), + () => registerLogTools(server, sessions), + () => registerVisualDiffTool(server, sessions), + () => registerMetricsTool(server), + ], + live: [ + () => registerHandoffTool(server, sessions), + () => registerLiveViewTool(server, sessions), + ], + }; +} diff --git a/src/server/resource-screenshot.ts b/src/server/resource-screenshot.ts new file mode 100644 index 0000000..c1bdb5e --- /dev/null +++ b/src/server/resource-screenshot.ts @@ -0,0 +1,34 @@ +/** + * Screenshot capture backing the `screenshot://{sessionId}/last` MCP resource. + * @module server/resource-screenshot + */ +import type { ReadResourceResult } from "@modelcontextprotocol/sdk/types.js"; +import { SessionNotFoundError } from "../lib/errors.js"; +import type { SessionManager } from "../session/manager.js"; + +/** MIME type of the resource blob. */ +const JPEG_MIME = "image/jpeg"; + +/** + * Capture a JPEG of the session's current page and return it as a base64 blob. + * @param sessions - Live session registry. + * @param sessionId - Target session id (from the resource URI). + * @param uriHref - Resolved resource URI to echo back in the content. + * @throws {SessionNotFoundError} when no live session matches `sessionId`. + */ +export async function captureSessionScreenshot( + sessions: SessionManager, + sessionId: string, + uriHref: string, +): Promise { + const session = sessions.get(sessionId); + const buffer = await session.page.screenshot({ type: "jpeg", quality: 80, fullPage: false }); + return { + contents: [{ uri: uriHref, mimeType: JPEG_MIME, blob: buffer.toString("base64") }], + }; +} + +/** True when `error` signals an unknown/expired session. */ +export function isSessionMissing(error: unknown): boolean { + return error instanceof SessionNotFoundError; +} diff --git a/src/server/resources.ts b/src/server/resources.ts index 222e398..3e1298b 100644 --- a/src/server/resources.ts +++ b/src/server/resources.ts @@ -1,9 +1,15 @@ /** - * Expose the run artifacts directory as an MCP resource. + * Expose run artifacts and live-session screenshots as MCP resources. * @module server/resources */ import { readdirSync } from "node:fs"; -import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { + McpServer, + ResourceTemplate, +} from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { ReadResourceResult } from "@modelcontextprotocol/sdk/types.js"; +import type { SessionManager } from "../session/manager.js"; +import { captureSessionScreenshot, isSessionMissing } from "./resource-screenshot.js"; const RUNS_DIR = "runs"; const RUNS_URI = "fuse-browser://runs"; @@ -16,8 +22,8 @@ function listRuns(): string[] { } } -/** Register a resource listing probe run artifacts (reports + screenshots). */ -export function registerResources(server: McpServer): void { +/** Register the static runs-index resource. */ +function registerRunsIndex(server: McpServer): void { server.registerResource( "runs-index", RUNS_URI, @@ -33,3 +39,46 @@ export function registerResources(server: McpServer): void { }), ); } + +/** Build the dynamic-screenshot resource template (lists every live session). */ +function screenshotTemplate(sessions: SessionManager): ResourceTemplate { + return new ResourceTemplate("screenshot://{sessionId}/last", { + list: async () => ({ + resources: sessions.list().map((s) => ({ + uri: `screenshot://${s.id}/last`, + name: `Session ${s.id} (last view)`, + mimeType: "image/jpeg", + })), + }), + }); +} + +/** Register the `screenshot://{sessionId}/last` JPEG resource. */ +function registerScreenshot(server: McpServer, sessions: SessionManager): void { + server.registerResource( + "session-screenshot", + screenshotTemplate(sessions), + { + title: "Session screenshot", + description: "Live JPEG of a session's current page, captured on read.", + mimeType: "image/jpeg", + }, + async (uri, { sessionId }): Promise => { + const id = Array.isArray(sessionId) ? sessionId[0] : sessionId; + try { + return await captureSessionScreenshot(sessions, String(id), uri.href); + } catch (error) { + if (isSessionMissing(error)) { + throw new Error(`No live session for id "${id}"`); + } + throw error; + } + }, + ); +} + +/** Register run-artifact and live-session-screenshot resources. */ +export function registerResources(server: McpServer, sessions: SessionManager): void { + registerRunsIndex(server); + registerScreenshot(server, sessions); +} diff --git a/src/server/result.ts b/src/server/result.ts index 59ff0c9..ce7bd6f 100644 --- a/src/server/result.ts +++ b/src/server/result.ts @@ -27,9 +27,14 @@ export function imageJsonResult( }; } -/** Error result (isError flag set). */ -export function errorResult(message: string): CallToolResult { - return { content: [{ type: "text", text: message }], isError: true }; +/** + * Error result (isError flag set). When a machine-readable `code` is given, + * it is exposed as `structuredContent: { code, message }`. + */ +export function errorResult(message: string, code?: string): CallToolResult { + const result: CallToolResult = { content: [{ type: "text", text: message }], isError: true }; + if (code) result.structuredContent = { code, message }; + return result; } /** Image result (PNG base64) with an optional text note. */ diff --git a/src/server/schemas.ts b/src/server/schemas.ts index 12fdee5..4ae47ed 100644 --- a/src/server/schemas.ts +++ b/src/server/schemas.ts @@ -43,6 +43,8 @@ export const agentOptionShape = { proxyMapPath: z.string().optional(), proxiesPath: z.string().optional(), storageStatePath: z.string().optional(), + profile: z.string().optional().describe("Named persistent auth profile — shorthand for storageStatePath at ~/.fuse-browser/profiles/.json (ignored when storageStatePath is set)."), + blockResources: z.array(z.string()).optional().describe("Resource types to abort (image, media, font, stylesheet, script, xhr, fetch, websocket, manifest, other) — speeds up batches; unknown types ignored."), harPath: z.string().optional(), harMode: z.enum(["minimal", "full"]).optional(), harReplay: z.string().optional(), diff --git a/src/server/server.ts b/src/server/server.ts index ad683b9..c298598 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -5,31 +5,9 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { VERSION } from "../lib/version.js"; import { SessionManager } from "../session/manager.js"; +import { CAP_GROUPS, parseCaps } from "./caps.js"; +import { toolGroups } from "./registry.js"; import { registerResources } from "./resources.js"; -import { registerActTools } from "./tools/act.js"; -import { registerCollectTool } from "./tools/collect.js"; -import { registerConnectTool } from "./tools/connect.js"; -import { registerExtractTool } from "./tools/extract.js"; -import { registerExtractSchemaTool } from "./tools/extract-schema.js"; -import { registerHandoffTool } from "./tools/handoff.js"; -import { registerInspectTool } from "./tools/inspect.js"; -import { registerLiveViewTool } from "./tools/live-view.js"; -import { registerMetricsTool } from "./tools/metrics.js"; -import { registerNavigateTool } from "./tools/navigate.js"; -import { registerCollectBatchTool } from "./tools/collect-batch.js"; -import { registerCrawlTool } from "./tools/crawl.js"; -import { registerFetchTool } from "./tools/fetch.js"; -import { registerFetchBatchTool } from "./tools/fetch-batch.js"; -import { registerProbeTools } from "./tools/probe.js"; -import { registerShotsBatchTool } from "./tools/shots-batch.js"; -import { registerSiteShotsTool } from "./tools/site-shots.js"; -import { registerSerpBatchTool } from "./tools/serp-batch.js"; -import { registerRunTool } from "./tools/run.js"; -import { registerScreenshotTool } from "./tools/screenshot.js"; -import { registerSessionTools } from "./tools/session.js"; -import { registerSnapshotTools } from "./tools/snapshot.js"; -import { registerVisualDiffTool } from "./tools/visual-diff.js"; -import { registerWaitTool } from "./tools/wait.js"; /** The built server and its session manager (for shutdown). */ export interface BuiltServer { @@ -37,34 +15,19 @@ export interface BuiltServer { sessions: SessionManager; } -/** Create the fuse-browser MCP server with every tool and resource wired. */ +/** + * Create the fuse-browser MCP server. Tool groups can be filtered with the + * `FUSE_CAPS` env var (e.g. `FUSE_CAPS=core,extract`) to expose fewer tools + * to the client; resources are always registered. + */ export function createServer(): BuiltServer { const server = new McpServer({ name: "fuse-browser", version: VERSION }); const sessions = new SessionManager(); - registerProbeTools(server); - registerFetchTool(server); - registerFetchBatchTool(server); - registerCrawlTool(server); - registerCollectBatchTool(server); - registerShotsBatchTool(server); - registerSiteShotsTool(server); - registerSerpBatchTool(server); - registerSessionTools(server, sessions); - registerConnectTool(server, sessions); - registerNavigateTool(server, sessions); - registerActTools(server, sessions); - registerSnapshotTools(server, sessions); - registerCollectTool(server, sessions); - registerWaitTool(server, sessions); - registerRunTool(server, sessions); - registerExtractTool(server, sessions); - registerExtractSchemaTool(server, sessions); - registerScreenshotTool(server, sessions); - registerInspectTool(server, sessions); - registerVisualDiffTool(server, sessions); - registerHandoffTool(server, sessions); - registerLiveViewTool(server, sessions); - registerMetricsTool(server); - registerResources(server); + const caps = parseCaps(process.env.FUSE_CAPS); + const groups = toolGroups(server, sessions); + for (const group of CAP_GROUPS) { + if (caps.has(group)) for (const register of groups[group]) register(); + } + registerResources(server, sessions); return { server, sessions }; } diff --git a/src/server/tools/autoscroll.ts b/src/server/tools/autoscroll.ts new file mode 100644 index 0000000..6c681d1 --- /dev/null +++ b/src/server/tools/autoscroll.ts @@ -0,0 +1,44 @@ +/** + * `browser_autoscroll`: drive a live session to the bottom of a long list, + * triggering lazy-load / infinite-scroll until it stabilises. + * @module server/tools/autoscroll + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { autoScroll } from "../../actions/auto-scroll.js"; +import type { SessionManager } from "../../session/manager.js"; +import { jsonResult } from "../result.js"; +import { withSession } from "./with-session.js"; + +/** Register `browser_autoscroll`. */ +export function registerAutoScrollTool(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_autoscroll", + { + title: "Auto-scroll a long list", + description: + "Repeatedly scroll a long/infinite list to the bottom to load every item before extracting. Stops after `idleRounds` rounds without growth, at `maxScrolls`, or once `untilSelector` reaches `minCount` elements. Run this before browser_extract/collect on lazy-loaded result pages.", + inputSchema: { + sessionId: z.string(), + maxScrolls: z.number().int().positive().optional(), + idleRounds: z.number().int().positive().optional(), + untilSelector: z.string().optional(), + minCount: z.number().int().positive().optional(), + delayMs: z.number().int().nonnegative().optional(), + }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + const { rounds, height } = await autoScroll(s.page, { + maxScrolls: a.maxScrolls as number | undefined, + idleRounds: a.idleRounds as number | undefined, + untilSelector: a.untilSelector as string | undefined, + minCount: a.minCount as number | undefined, + delayMs: a.delayMs as number | undefined, + }); + return jsonResult({ rounds, height, url: s.page.url() }); + }); + }, + ); +} diff --git a/src/server/tools/collect-batch.ts b/src/server/tools/collect-batch.ts index 4fa6666..4066286 100644 --- a/src/server/tools/collect-batch.ts +++ b/src/server/tools/collect-batch.ts @@ -10,6 +10,7 @@ import { z } from "zod"; import { collectBatch } from "../../agent/collect-batch.js"; import { resolveConfig } from "../../agent/config.js"; import { toAgentOptions } from "../map-options.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; /** Register `browser_collect_batch`. */ @@ -34,7 +35,7 @@ export function registerCollectBatchTool(server: McpServer): void { proxyUrl: z.string().optional(), }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const num = (v: unknown): number | undefined => (typeof v === "number" ? v : undefined); const config = resolveConfig(toAgentOptions(a)); @@ -46,6 +47,7 @@ export function registerCollectBatchTool(server: McpServer): void { extractPrices: a.extractPrices === true, concurrency: num(a.concurrency), throttleMs: num(a.throttleMs), + onProgress: progressReporter(extra), }); return jsonResult({ count: results.length, results }); }, diff --git a/src/server/tools/crawl.ts b/src/server/tools/crawl.ts index 186e661..dbec0b0 100644 --- a/src/server/tools/crawl.ts +++ b/src/server/tools/crawl.ts @@ -7,6 +7,7 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { crawl } from "../../agent/crawl.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; /** Register `browser_crawl`. */ @@ -31,10 +32,11 @@ export function registerCrawlTool(server: McpServer): void { proxyUrl: z.string().optional(), }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const num = (v: unknown): number | undefined => (typeof v === "number" ? v : undefined); const result = await crawl(String(a.url), { + onProgress: progressReporter(extra), maxPages: num(a.maxPages), maxDepth: num(a.maxDepth), sameOrigin: a.sameOrigin === false ? false : undefined, diff --git a/src/server/tools/dialogs.ts b/src/server/tools/dialogs.ts new file mode 100644 index 0000000..ad24265 --- /dev/null +++ b/src/server/tools/dialogs.ts @@ -0,0 +1,60 @@ +/** + * `browser_dialog` + `browser_downloads`: native dialog policy for a live + * session and the list of downloads captured on it. + * @module server/tools/dialogs + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { attachDialogs, recentDialogs, setDialogPolicy } from "../../session/dialogs.js"; +import { attachDownloads, listDownloads } from "../../session/downloads.js"; +import type { SessionManager } from "../../session/manager.js"; +import { jsonResult } from "../result.js"; +import { withSession } from "./with-session.js"; + +/** Register `browser_dialog` and `browser_downloads`. */ +export function registerDialogTools(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_dialog", + { + title: "Set dialog policy", + description: + "Set how native dialogs (alert/confirm/prompt/beforeunload) are handled on this session: accept or dismiss, with optional text for prompts. Applies to upcoming dialogs; also returns the recent ones.", + inputSchema: { + sessionId: z.string(), + action: z.enum(["accept", "dismiss"]), + promptText: z.string().optional(), + }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + // Idempotent: a no-op when the session wiring already attached it. + attachDialogs(s); + const policy = { + action: a.action as "accept" | "dismiss", + promptText: a.promptText as string | undefined, + }; + setDialogPolicy(s, policy); + return jsonResult({ policy, recent: recentDialogs(s) }); + }); + }, + ); + + server.registerTool( + "browser_downloads", + { + title: "List downloads", + description: "List the files downloaded by this session (saved under outputDir/downloads).", + inputSchema: { sessionId: z.string() }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + // Idempotent: a no-op when the session wiring already attached it. + attachDownloads(s); + const downloads = listDownloads(s); + return jsonResult({ count: downloads.length, downloads }); + }); + }, + ); +} diff --git a/src/server/tools/extract-schema.ts b/src/server/tools/extract-schema.ts index fb49424..cfa394f 100644 --- a/src/server/tools/extract-schema.ts +++ b/src/server/tools/extract-schema.ts @@ -4,7 +4,11 @@ */ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; -import { type ExtractionSchema, extractStructured } from "../../extraction/structured.js"; +import { + type ExtractionSchema, + extractStructured, + extractStructuredPerCard, +} from "../../extraction/structured.js"; import type { SessionManager } from "../../session/manager.js"; import { jsonResult } from "../result.js"; import { withSession } from "./with-session.js"; @@ -23,16 +27,22 @@ export function registerExtractSchemaTool(server: McpServer, sessions: SessionMa { title: "Extract by schema", description: - "Extract typed data from the live page via a field map (field -> {selector, attr?, all?, abs?}). Deterministic; reads the rendered DOM, so it works on Next.js/SPA pages.", + "Extract typed data from the live page via a field map (field -> {selector, attr?, all?, abs?}). Deterministic; reads the rendered DOM, so it works on Next.js/SPA pages. Pass `containerSelector` to extract CARD BY CARD: one record per matching container, every field resolved relative to it — so title/price/link stay correlated per card instead of returned as index-aligned parallel arrays.", inputSchema: { sessionId: z.string(), schema: z.record(z.string(), fieldSpec), + containerSelector: z.string().optional(), }, }, async (args) => { const a = args as Record; const schema = a.schema as ExtractionSchema; + const container = typeof a.containerSelector === "string" ? a.containerSelector : undefined; return withSession(sessions, String(a.sessionId), async (s) => { + if (container) { + const items = await extractStructuredPerCard(s.page, container, schema); + return jsonResult({ url: s.page.url(), count: items.length, items }); + } const data = await extractStructured(s.page, schema); return jsonResult({ url: s.page.url(), data }); }); diff --git a/src/server/tools/fetch-batch.ts b/src/server/tools/fetch-batch.ts index bc4c9d0..de542d3 100644 --- a/src/server/tools/fetch-batch.ts +++ b/src/server/tools/fetch-batch.ts @@ -7,6 +7,7 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { fetchBatch } from "../../agent/fetch-batch.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; /** Register `browser_fetch_batch`. */ @@ -26,7 +27,7 @@ export function registerFetchBatchTool(server: McpServer): void { concurrency: z.number().int().optional(), }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const urls = Array.isArray(a.urls) ? a.urls.map(String) : []; const results = await fetchBatch(urls, { @@ -35,6 +36,7 @@ export function registerFetchBatchTool(server: McpServer): void { browserFallback: a.browserFallback === true, proxyUrl: typeof a.proxyUrl === "string" ? a.proxyUrl : undefined, concurrency: typeof a.concurrency === "number" ? a.concurrency : undefined, + onProgress: progressReporter(extra), }); return jsonResult({ count: results.length, results }); }, diff --git a/src/server/tools/logs-filter.ts b/src/server/tools/logs-filter.ts new file mode 100644 index 0000000..04800db --- /dev/null +++ b/src/server/tools/logs-filter.ts @@ -0,0 +1,65 @@ +/** + * Pure filter/merge helpers for the session console and network log buffers. + * @module server/tools/logs-filter + */ +import type { NetworkLog } from "../../agent/network.js"; + +const DEFAULT_LIMIT = 50; + +/** One merged network row: request joined with its response by URL. */ +export interface NetworkEntry { + method?: string; + url: string; + status?: number; + resourceType?: string; +} + +/** Console-level filter values (Playwright console message types). */ +export const CONSOLE_LEVELS = ["error", "warning", "info", "log", "debug"] as const; + +/** + * Filter console entries by level and keep the last `limit`. + * @param entries - Raw console buffer from the session's NetworkLog. + * @param level - Optional exact console type to keep (e.g. "error"). + * @param limit - Max entries returned, most recent last (default 50). + */ +export function filterConsole( + entries: NetworkLog["console"], + level?: string, + limit: number = DEFAULT_LIMIT, +): NetworkLog["console"] { + const matched = level ? entries.filter((e) => e.type === level) : entries; + return matched.slice(-Math.max(0, limit)); +} + +/** + * Merge raw request/response events into one row per URL (order preserved). + * @param events - Raw network buffer from the session's NetworkLog. + */ +export function mergeNetwork(events: Array>): NetworkEntry[] { + const byUrl = new Map(); + for (const ev of events) { + const url = String(ev.url ?? ""); + const row = byUrl.get(url) ?? { url }; + if (typeof ev.method === "string") row.method = ev.method; + if (typeof ev.resourceType === "string") row.resourceType = ev.resourceType; + if (typeof ev.status === "number") row.status = ev.status; + if (!byUrl.has(url)) byUrl.set(url, row); + } + return [...byUrl.values()]; +} + +/** + * Filter merged network rows by status / URL substring, keep the last `limit`. + * @param rows - Merged rows from {@link mergeNetwork}. + * @param f - Optional filters: exact `status`, `urlContains` substring, `limit`. + */ +export function filterNetwork( + rows: NetworkEntry[], + f: { status?: number; urlContains?: string; limit?: number } = {}, +): NetworkEntry[] { + let out = rows; + if (f.status !== undefined) out = out.filter((r) => r.status === f.status); + if (f.urlContains !== undefined) out = out.filter((r) => r.url.includes(f.urlContains as string)); + return out.slice(-Math.max(0, f.limit ?? DEFAULT_LIMIT)); +} diff --git a/src/server/tools/logs.ts b/src/server/tools/logs.ts new file mode 100644 index 0000000..65e7005 --- /dev/null +++ b/src/server/tools/logs.ts @@ -0,0 +1,71 @@ +/** + * `browser_console` / `browser_network`: query the console and network logs + * already captured for a live session — debug JS errors and failed requests. + * @module server/tools/logs + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import type { SessionManager } from "../../session/manager.js"; +import { jsonResult } from "../result.js"; +import { CONSOLE_LEVELS, filterConsole, filterNetwork, mergeNetwork } from "./logs-filter.js"; +import { withSession } from "./with-session.js"; + +const CONSOLE_DESC = + "Console messages captured in the session since open (last 80). Use to debug JS errors, CSP " + + 'violations, or why a page misbehaves. Filter with `level` (e.g. "error"); `limit` = last N (default 50).'; + +const NETWORK_DESC = + "Network requests captured in the session since open (last 80): method, url, status, resourceType. " + + "Use to debug why a page does not load: failed requests (`status: 404/500`), blocked APIs " + + "(`urlContains`). `limit` = last N (default 50). Entries without `status` got no response (pending/failed)."; + +/** Register `browser_console` and `browser_network`. */ +export function registerLogTools(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_console", + { + title: "Read console logs", + description: CONSOLE_DESC, + inputSchema: { + sessionId: z.string(), + level: z.enum(CONSOLE_LEVELS).optional(), + limit: z.number().optional(), + }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + const entries = filterConsole( + s.logs.console, + a.level as string | undefined, + a.limit as number | undefined, + ); + return jsonResult({ count: entries.length, entries }); + }); + }, + ); + server.registerTool( + "browser_network", + { + title: "Read network logs", + description: NETWORK_DESC, + inputSchema: { + sessionId: z.string(), + status: z.number().optional(), + urlContains: z.string().optional(), + limit: z.number().optional(), + }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + const requests = filterNetwork(mergeNetwork(s.logs.network), { + status: a.status as number | undefined, + urlContains: a.urlContains as string | undefined, + limit: a.limit as number | undefined, + }); + return jsonResult({ count: requests.length, requests }); + }); + }, + ); +} diff --git a/src/server/tools/navigate.ts b/src/server/tools/navigate.ts index 3b13c32..9115a20 100644 --- a/src/server/tools/navigate.ts +++ b/src/server/tools/navigate.ts @@ -5,7 +5,7 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import type { SessionManager } from "../../session/manager.js"; -import { gotoWithRetry } from "../../net/navigate.js"; +import { DEFAULT_GOTO, gotoWithRetry } from "../../net/navigate.js"; import { jsonResult } from "../result.js"; import { withSession } from "./with-session.js"; @@ -21,7 +21,7 @@ export function registerNavigateTool(server: McpServer, sessions: SessionManager async (args) => { const a = args as Record; return withSession(sessions, String(a.sessionId), async (s) => { - await gotoWithRetry(s.page, String(a.url), { waitUntil: "domcontentloaded", timeout: 30_000 }); + await gotoWithRetry(s.page, String(a.url), DEFAULT_GOTO); try { await s.page.waitForLoadState("networkidle", { timeout: 10_000 }); } catch { diff --git a/src/server/tools/products.ts b/src/server/tools/products.ts new file mode 100644 index 0000000..5f2fb9d --- /dev/null +++ b/src/server/tools/products.ts @@ -0,0 +1,39 @@ +/** + * `browser_products`: structured per-card product extraction for e-commerce and + * listing pages. Returns one row per product (title linked to its own price), + * solving the "flat prices with no title" gap of plain price scraping. + * @module server/tools/products + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { extractProducts } from "../../extraction/products.js"; +import type { SessionManager } from "../../session/manager.js"; +import { jsonResult } from "../result.js"; +import { withSession } from "./with-session.js"; + +/** Register `browser_products`. */ +export function registerProductsTool(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_products", + { + title: "Extract product cards", + description: + "Extract structured product cards from an e-commerce / search-results page: one {title, price, currency, url?} per card, each price tied to its own title (unlike flat price scraping). Generic — detects repeated card containers by structure, so it works on Digitec, Booking, Amazon… Use it to answer 'which is the cheapest?' (sort by price), compare listings, or build a product table. Pass `containerSelector` to pin the card selector, `limit` to cap rows.", + inputSchema: { + sessionId: z.string(), + limit: z.number().int().positive().optional(), + containerSelector: z.string().optional(), + }, + }, + async (args) => { + const a = args as Record; + return withSession(sessions, String(a.sessionId), async (s) => { + const products = await extractProducts(s.page, { + limit: typeof a.limit === "number" ? a.limit : undefined, + containerSelector: typeof a.containerSelector === "string" ? a.containerSelector : undefined, + }); + return jsonResult({ url: s.page.url(), count: products.length, products }); + }); + }, + ); +} diff --git a/src/server/tools/serp-batch.ts b/src/server/tools/serp-batch.ts index c4f4ece..5256035 100644 --- a/src/server/tools/serp-batch.ts +++ b/src/server/tools/serp-batch.ts @@ -7,6 +7,7 @@ import { z } from "zod"; import { resolveConfig } from "../../agent/config.js"; import { serpBatch } from "../../agent/serp-batch.js"; import { toAgentOptions } from "../map-options.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; import { agentOptionShape } from "../schemas.js"; @@ -28,7 +29,7 @@ export function registerSerpBatchTool(server: McpServer): void { ...agentOptionShape, }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const config = resolveConfig(toAgentOptions(a)); const rows = await serpBatch(config, { @@ -38,6 +39,7 @@ export function registerSerpBatchTool(server: McpServer): void { hl: a.hl as string | undefined, gl: a.gl as string | undefined, delayMs: a.delayMs as number | undefined, + onProgress: progressReporter(extra), }); return jsonResult({ rows }); }, diff --git a/src/server/tools/shots-batch.ts b/src/server/tools/shots-batch.ts index ed6da31..224409a 100644 --- a/src/server/tools/shots-batch.ts +++ b/src/server/tools/shots-batch.ts @@ -11,6 +11,7 @@ import { resolveConfig } from "../../agent/config.js"; import { captureShotsBatch } from "../../agent/shots-batch.js"; import { parseViewports } from "../../engine/viewport.js"; import { toAgentOptions } from "../map-options.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; /** Register `browser_shots_batch`. */ @@ -32,7 +33,7 @@ export function registerShotsBatchTool(server: McpServer): void { proxyUrl: z.string().optional(), }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const config = resolveConfig(toAgentOptions(a)); const urls = Array.isArray(a.urls) ? a.urls.map(String) : []; @@ -43,6 +44,7 @@ export function registerShotsBatchTool(server: McpServer): void { viewports, typeof a.settleMs === "number" ? a.settleMs : undefined, typeof a.concurrency === "number" ? a.concurrency : undefined, + progressReporter(extra), ); return jsonResult({ count: results.length, results }); }, diff --git a/src/server/tools/site-shots.ts b/src/server/tools/site-shots.ts index efd8fd1..a7dd66f 100644 --- a/src/server/tools/site-shots.ts +++ b/src/server/tools/site-shots.ts @@ -11,6 +11,7 @@ import { resolveConfig } from "../../agent/config.js"; import { siteShots } from "../../agent/site-shots.js"; import { parseViewports } from "../../engine/viewport.js"; import { toAgentOptions } from "../map-options.js"; +import { progressReporter } from "../progress.js"; import { jsonResult } from "../result.js"; /** Register `browser_site_shots`. */ @@ -37,11 +38,12 @@ export function registerSiteShotsTool(server: McpServer): void { proxyUrl: z.string().optional(), }, }, - async (args) => { + async (args, extra) => { const a = args as Record; const num = (v: unknown): number | undefined => (typeof v === "number" ? v : undefined); const config = resolveConfig(toAgentOptions(a)); const result = await siteShots(config, String(a.url), { + onProgress: progressReporter(extra), maxPages: num(a.maxPages), maxDepth: num(a.maxDepth), sameOrigin: a.sameOrigin === false ? false : undefined, diff --git a/src/server/tools/tabs.ts b/src/server/tools/tabs.ts new file mode 100644 index 0000000..bce5d59 --- /dev/null +++ b/src/server/tools/tabs.ts @@ -0,0 +1,69 @@ +/** + * Multi-tab management tool (`browser_tabs`) for a live session. + * @module server/tools/tabs + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { z } from "zod"; +import type { SessionManager } from "../../session/manager.js"; +import type { SessionData } from "../../session/session.js"; +import { closeTab, listTabs, openTab, selectTab } from "../../session/tabs.js"; +import { errorResult, jsonResult } from "../result.js"; +import { withSession } from "./with-session.js"; + +const DESCRIPTION = + "Manage the tabs of a live session: list them, open a new tab (optional url), " + + "select one as the active target of every other browser_* tool, or close one. " + + "Use it when a click spawned a popup (OAuth login, target=_blank link): 'list' " + + "to find it, 'select' to drive it, 'close' then 'select' to come back. Always " + + "returns the tab list plus the active index after the action. Closing the last " + + "tab is refused — use browser_close to end the session."; + +type TabAction = "list" | "new" | "select" | "close"; + +/** Apply a mutating tab action; `list` is a no-op (the listing happens after). */ +async function applyAction(s: SessionData, action: TabAction, index: number, url?: string): Promise { + if (action === "new") await openTab(s, url); + else if (action === "select") await selectTab(s, index); + else if (action === "close") await closeTab(s, index); +} + +/** Map tab-domain errors to MCP error results; rethrow everything else. */ +function tabError(err: unknown): CallToolResult { + if (err instanceof RangeError) return errorResult(err.message, "invalid_tab_index"); + if (err instanceof Error && err.message.startsWith("cannot_close_last_tab")) + return errorResult(err.message, "cannot_close_last_tab"); + throw err; +} + +/** Register `browser_tabs`. */ +export function registerTabsTool(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_tabs", + { + title: "Tabs", + description: DESCRIPTION, + inputSchema: { + sessionId: z.string(), + action: z.enum(["list", "new", "select", "close"]), + index: z.number().optional(), + url: z.string().optional(), + }, + }, + async (args) => { + const a = args as Record; + const action = a.action as TabAction; + return withSession(sessions, String(a.sessionId), async (s) => { + if ((action === "select" || action === "close") && typeof a.index !== "number") + return errorResult(`missing_index: action "${action}" requires an index`, "missing_index"); + try { + await applyAction(s, action, Number(a.index ?? 0), a.url ? String(a.url) : undefined); + } catch (err) { + return tabError(err); + } + const tabs = await listTabs(s); + return jsonResult({ tabs, active: tabs.find((t) => t.active)?.index ?? 0 }); + }); + }, + ); +} diff --git a/src/server/tools/visual-diff.ts b/src/server/tools/visual-diff.ts index 0ffe3ea..c1d2fbf 100644 --- a/src/server/tools/visual-diff.ts +++ b/src/server/tools/visual-diff.ts @@ -9,6 +9,7 @@ import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { writeFileBytes } from "../../lib/fs.js"; import { diffPng } from "../../lib/pixel-diff.js"; +import { assertPngPath } from "../../lib/safe-png.js"; import type { SessionManager } from "../../session/manager.js"; import { errorResult, jsonResult } from "../result.js"; import { withSession } from "./with-session.js"; @@ -34,14 +35,27 @@ export function registerVisualDiffTool(server: McpServer, sessions: SessionManag const x = args as Record; const threshold = typeof x.threshold === "number" ? x.threshold : 0.1; if (typeof x.a === "string" && typeof x.b === "string") { - const d = diffPng(readFileSync(x.a), readFileSync(x.b), threshold); - writeFileBytes(`${x.b}.diff.png`, d.diffPng); - return jsonResult({ ...stats(d), diffImage: `${x.b}.diff.png` }); + let a: string; + let b: string; + try { + a = assertPngPath(x.a, "a"); + b = assertPngPath(x.b, "b"); + } catch (e) { + return errorResult(e instanceof Error ? e.message : String(e)); + } + const d = diffPng(readFileSync(a), readFileSync(b), threshold); + writeFileBytes(`${b}.diff.png`, d.diffPng); + return jsonResult({ ...stats(d), diffImage: `${b}.diff.png` }); } if (typeof x.sessionId !== "string" || typeof x.baseline !== "string") { return errorResult("browser_visual_diff needs either `a`+`b` paths, or `sessionId`+`baseline`"); } - const baseline = x.baseline; + let baseline: string; + try { + baseline = assertPngPath(x.baseline, "baseline"); + } catch (e) { + return errorResult(e instanceof Error ? e.message : String(e)); + } return withSession(sessions, x.sessionId, async (s) => { const shot = await s.page.screenshot({ fullPage: x.fullPage === true }); if (!existsSync(baseline)) { diff --git a/src/server/tools/with-session.ts b/src/server/tools/with-session.ts index ffe0ff5..4a2ebe8 100644 --- a/src/server/tools/with-session.ts +++ b/src/server/tools/with-session.ts @@ -15,7 +15,7 @@ const HEALED = "page_crashed: the page was recreated and restored — retry your /** Evict a lost session and return the standard error result. */ async function evictLost(sessions: SessionManager, id: string): Promise { await sessions.close(id).catch(() => {}); - return errorResult(LOST); + return errorResult(LOST, "session_lost"); } /** @@ -57,13 +57,14 @@ export async function withSession( try { session = sessions.get(id); } catch (err) { - if (err instanceof SessionNotFoundError) return errorResult(err.message); + if (err instanceof SessionNotFoundError) return errorResult(err.message, "session_not_found"); throw err; } const blocked = await ensureHealthy(sessions, id, session); if (blocked) return blocked; + sessions.markBusy(id); try { return await fn(session); } catch (err) { @@ -73,9 +74,11 @@ export async function withSession( // discovers the browser is gone, evict rather than mislead into retrying. try { await recoverSession(session); - return errorResult(HEALED); + return errorResult(HEALED, "page_crashed"); } catch { return evictLost(sessions, id); } + } finally { + sessions.markIdle(id); } } diff --git a/src/session/close.ts b/src/session/close.ts new file mode 100644 index 0000000..68b0b4f --- /dev/null +++ b/src/session/close.ts @@ -0,0 +1,43 @@ +/** + * Session teardown: persist storage state, then close context + browser. + * @module session/close + */ +import { dirname } from "node:path"; +import { ensureDir } from "../lib/fs.js"; +import type { SessionData } from "./session.js"; + +/** + * Close a session. For launched browsers, close context + browser. For a CDP + * attach (`connected`), only drop the link — never close the user's browser + * or its default context. + */ +export async function closeSession(session: SessionData): Promise { + if (session.connected) { + try { + await session.browser?.close(); + } catch { + /* ignore: detaching from a user browser */ + } + return; + } + if (session.config.storageStatePath) { + try { + ensureDir(dirname(session.config.storageStatePath)); + await session.context.storageState({ path: session.config.storageStatePath }); + } catch { + /* best-effort: never block teardown */ + } + } + try { + await session.context.close(); + } catch { + /* ignore */ + } + if (session.browser) { + try { + await session.browser.close(); + } catch { + /* ignore */ + } + } +} diff --git a/src/session/dialogs.ts b/src/session/dialogs.ts new file mode 100644 index 0000000..1f41f83 --- /dev/null +++ b/src/session/dialogs.ts @@ -0,0 +1,96 @@ +/** + * Per-session native dialog handling (alert/confirm/prompt/beforeunload): + * a mutable policy applied to upcoming dialogs plus a ring buffer of the last + * observed dialogs. State lives in a WeakMap keyed on the SessionData object, + * so nothing needs to be added to session.ts internals. + * @module session/dialogs + */ +import type { Dialog, Page } from "playwright"; +import type { SessionData } from "./session.js"; + +/** Policy applied to dialogs as they appear. */ +export interface DialogPolicy { + action: "accept" | "dismiss"; + /** Text typed into `prompt` dialogs when accepting. */ + promptText?: string; +} + +/** One observed dialog, recorded in the per-session ring buffer. */ +export interface DialogRecord { + type: string; + message: string; + at: number; + /** How the policy resolved it. */ + handled: "accept" | "dismiss"; +} + +/** Ring buffer capacity (most recent dialogs kept). */ +const MAX_DIALOGS = 20; + +/** Internal per-session dialog state. */ +interface DialogState { + policy: DialogPolicy; + recent: DialogRecord[]; + /** Pages already wired, for idempotent attachment (identity guard). */ + pages: WeakSet; +} + +const states = new WeakMap(); + +/** Get or lazily create the state for a session (default policy: dismiss). */ +function stateFor(session: SessionData): DialogState { + let state = states.get(session); + if (!state) { + state = { policy: { action: "dismiss" }, recent: [], pages: new WeakSet() }; + states.set(session, state); + } + return state; +} + +/** Apply the current policy to one dialog and record it (ring of 20). */ +async function handleDialog(state: DialogState, dialog: Dialog): Promise { + const { action, promptText } = state.policy; + state.recent.push({ type: dialog.type(), message: dialog.message(), at: Date.now(), handled: action }); + if (state.recent.length > MAX_DIALOGS) state.recent.shift(); + // .catch: the dialog may already be handled (e.g. page closed underneath). + if (action === "accept") { + await dialog.accept(dialog.type() === "prompt" ? promptText : undefined).catch(() => {}); + } else { + await dialog.dismiss().catch(() => {}); + } +} + +/** + * Wire the dialog handler onto the session's current page, applying the + * session policy to every dialog. Idempotent per page (identity guard, like + * session/health.ts): re-calling with the same page never doubles handlers, + * while a recovered page gets wired fresh. + * + * @param session - The live session whose `page` to watch. + */ +export function attachDialogs(session: SessionData): void { + const state = stateFor(session); + const page = session.page; + if (state.pages.has(page)) return; + state.pages.add(page); + page.on("dialog", (dialog) => void handleDialog(state, dialog)); +} + +/** + * Set the policy applied to upcoming dialogs. + * + * @param session - The live session. + * @param policy - Action (accept/dismiss) and optional prompt text. + */ +export function setDialogPolicy(session: SessionData, policy: DialogPolicy): void { + stateFor(session).policy = policy; +} + +/** + * The last dialogs observed on this session (oldest first, max 20). + * + * @param session - The live session. + */ +export function recentDialogs(session: SessionData): DialogRecord[] { + return [...stateFor(session).recent]; +} diff --git a/src/session/downloads.ts b/src/session/downloads.ts new file mode 100644 index 0000000..4769ac0 --- /dev/null +++ b/src/session/downloads.ts @@ -0,0 +1,95 @@ +/** + * Per-session download capture: every `download` event is saved under + * `/downloads/` (deduplicated filename) and recorded in a buffer + * keyed on the SessionData object via WeakMap — no session.ts wiring needed. + * Playwright contexts accept downloads by default (`acceptDownloads: true`). + * @module session/downloads + */ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import type { Download, Page } from "playwright"; +import { ensureDir } from "../lib/fs.js"; +import type { SessionData } from "./session.js"; + +/** One captured download. */ +export interface DownloadRecord { + url: string; + suggestedFilename: string; + /** Saved path on disk; empty while saving or when `error` is set. */ + path: string; + at: number; + error?: string; +} + +/** Internal per-session download state. */ +interface DownloadState { + records: DownloadRecord[]; + /** Pages already wired, for idempotent attachment (identity guard). */ + pages: WeakSet; +} + +const states = new WeakMap(); + +/** Get or lazily create the state for a session. */ +function stateFor(session: SessionData): DownloadState { + let state = states.get(session); + if (!state) { + state = { records: [], pages: new WeakSet() }; + states.set(session, state); + } + return state; +} + +/** Non-colliding path in `dir`: name.ext, then name-1.ext, name-2.ext, ... */ +function dedupePath(dir: string, filename: string): string { + const dot = filename.lastIndexOf("."); + const stem = dot > 0 ? filename.slice(0, dot) : filename; + const ext = dot > 0 ? filename.slice(dot) : ""; + let candidate = join(dir, filename); + for (let i = 1; existsSync(candidate); i += 1) candidate = join(dir, `${stem}-${i}${ext}`); + return candidate; +} + +/** Save one download to disk and record the outcome. */ +async function saveDownload(state: DownloadState, dir: string, download: Download): Promise { + const record: DownloadRecord = { + url: download.url(), + suggestedFilename: download.suggestedFilename(), + path: "", + at: Date.now(), + }; + state.records.push(record); + try { + ensureDir(dir); + const path = dedupePath(dir, download.suggestedFilename() || "download"); + await download.saveAs(path); + record.path = path; + } catch (err) { + record.error = err instanceof Error ? err.message : String(err); + } +} + +/** + * Wire the download handler onto the session's current page: each download is + * saved to `/downloads/` (suffixing -1, + * -2 on name collisions) and recorded. Idempotent per page (identity guard). + * + * @param session - The live session whose `page` to watch. + */ +export function attachDownloads(session: SessionData): void { + const state = stateFor(session); + const page = session.page; + if (state.pages.has(page)) return; + state.pages.add(page); + const dir = join(session.config.outputDir, "downloads"); + page.on("download", (download) => void saveDownload(state, dir, download)); +} + +/** + * The downloads captured on this session (oldest first). + * + * @param session - The live session. + */ +export function listDownloads(session: SessionData): DownloadRecord[] { + return [...stateFor(session).records]; +} diff --git a/src/session/manager.ts b/src/session/manager.ts index 41a23ee..c3c3571 100644 --- a/src/session/manager.ts +++ b/src/session/manager.ts @@ -5,7 +5,9 @@ import type { ResolvedConfig } from "../agent/config.js"; import { SessionLimitError, SessionNotFoundError } from "../lib/errors.js"; import { sha1 } from "../lib/fs.js"; -import { closeSession, openSession, type SessionData } from "./session.js"; +import { closeSession } from "./close.js"; +import { openSession, type SessionData } from "./session.js"; +import { TtlGuard } from "./ttl-guard.js"; /** Options for the session manager. */ export interface SessionManagerOptions { @@ -18,7 +20,7 @@ export interface SessionManagerOptions { /** Holds live sessions and closes them on TTL expiry or shutdown. */ export class SessionManager { private readonly sessions = new Map(); - private readonly timers = new Map(); + private readonly guard: TtlGuard; private readonly ttlMs: number; private readonly maxSessions: number; private counter = 0; @@ -26,6 +28,7 @@ export class SessionManager { constructor(opts: SessionManagerOptions = {}) { this.ttlMs = opts.ttlMs ?? 300_000; this.maxSessions = opts.maxSessions ?? 8; + this.guard = new TtlGuard(this.ttlMs, (id) => void this.close(id)); } /** @@ -38,7 +41,7 @@ export class SessionManager { const id = sha1(`${config.outputDir}-${Date.now()}-${this.counter}`).slice(0, 12); const session = await openSession(id, config, this.ttlMs); this.sessions.set(id, session); - this.schedule(id); + this.guard.schedule(id); return session; } @@ -55,27 +58,27 @@ export class SessionManager { const session = this.sessions.get(id); if (!session) return; session.expiresAt = Date.now() + this.ttlMs; - this.schedule(id); + this.guard.schedule(id); } - private schedule(id: string): void { - const existing = this.timers.get(id); - if (existing) clearTimeout(existing); - const timer = setTimeout(() => void this.close(id), this.ttlMs); - timer.unref?.(); - this.timers.set(id, timer); + /** Mark a session as in use by a tool, deferring TTL expiry. */ + markBusy(id: string): void { + if (!this.sessions.has(id)) return; + this.guard.markBusy(id); } - /** Close and remove a session; returns false if it did not exist. */ + /** Mark a session idle again and refresh its TTL. */ + markIdle(id: string): void { + this.guard.markIdle(id); + this.touch(id); + } + + /** Close and remove a session (always, even if busy); false if missing. */ async close(id: string): Promise { const session = this.sessions.get(id); if (!session) return false; this.sessions.delete(id); - const timer = this.timers.get(id); - if (timer) { - clearTimeout(timer); - this.timers.delete(id); - } + this.guard.clear(id); await closeSession(session); return true; } diff --git a/src/session/recover.ts b/src/session/recover.ts index 2d51bcd..539b7ba 100644 --- a/src/session/recover.ts +++ b/src/session/recover.ts @@ -7,6 +7,8 @@ import type { Page } from "playwright"; import { attachListeners } from "../agent/network.js"; import { BrowserLostError } from "../lib/errors.js"; +import { attachDialogs } from "./dialogs.js"; +import { attachDownloads } from "./downloads.js"; import { attachHealth } from "./health.js"; import type { SessionData } from "./session.js"; @@ -35,6 +37,9 @@ export async function recoverSession(session: SessionData): Promise { session.health = "ok"; // Page-only: context/browser listeners persist across recovery (no re-add). attachHealth(session, { pageOnly: true }); + // Idempotent per page; dialog policy and download buffers survive recovery. + attachDialogs(session); + attachDownloads(session); if (session.config.harReplay) { await page diff --git a/src/session/session.ts b/src/session/session.ts index b873c91..bea14dc 100644 --- a/src/session/session.ts +++ b/src/session/session.ts @@ -2,12 +2,12 @@ * A live browser session: a context + page kept alive between MCP calls. * @module session/session */ -import { dirname } from "node:path"; import type { Browser, BrowserContext, Page } from "playwright"; import type { ResolvedConfig } from "../agent/config.js"; import { attachListeners, type NetworkLog } from "../agent/network.js"; import { selectEngineForConfig } from "../engine/registry.js"; -import { ensureDir } from "../lib/fs.js"; +import { attachDialogs } from "./dialogs.js"; +import { attachDownloads } from "./downloads.js"; import { attachHealth } from "./health.js"; /** Liveness of a session's page/browser. */ @@ -59,41 +59,7 @@ export async function openSession( expiresAt: now + ttlMs, }; attachHealth(session); + attachDialogs(session); + attachDownloads(session); return session; } - -/** - * Close a session. For launched browsers, close context + browser. For a CDP - * attach (`connected`), only drop the link — never close the user's browser - * or its default context. - */ -export async function closeSession(session: SessionData): Promise { - if (session.connected) { - try { - await session.browser?.close(); - } catch { - /* ignore: detaching from a user browser */ - } - return; - } - if (session.config.storageStatePath) { - try { - ensureDir(dirname(session.config.storageStatePath)); - await session.context.storageState({ path: session.config.storageStatePath }); - } catch { - /* best-effort: never block teardown */ - } - } - try { - await session.context.close(); - } catch { - /* ignore */ - } - if (session.browser) { - try { - await session.browser.close(); - } catch { - /* ignore */ - } - } -} diff --git a/src/session/tabs-wiring.ts b/src/session/tabs-wiring.ts new file mode 100644 index 0000000..f9f328e --- /dev/null +++ b/src/session/tabs-wiring.ts @@ -0,0 +1,62 @@ +/** + * Per-page wiring registry for multi-tab sessions: network logs and page-level + * health listeners must be attached exactly once per Page — re-selecting a tab + * must not stack handlers (same per-page logs pattern as session/recover.ts). + * @module session/tabs-wiring + */ +import type { Page } from "playwright"; +import { attachListeners, type NetworkLog } from "../agent/network.js"; +import { attachHealth } from "./health.js"; +import type { SessionData } from "./session.js"; + +// Weak keys die with their Page, so closed tabs cost nothing. +const pageLogs = new WeakMap(); +const healthWired = new WeakSet(); + +/** Record the wiring openSession/recoverSession already did on the active page. */ +export function seedActive(session: SessionData): void { + pageLogs.set(session.page, session.logs); + healthWired.add(session.page); +} + +/** + * Attach a network log to a freshly-opened tab BEFORE it navigates, so the + * tab's initial document and subresources are captured (a page opened with a + * URL would otherwise emit every request before `selectTab` wires it). + * Idempotent per page; the log is reused by a later `wireTab`. + */ +export function primeTab(page: Page): NetworkLog { + let logs = pageLogs.get(page); + if (!logs) { + logs = attachListeners(page); + pageLogs.set(page, logs); + } + return logs; +} + +/** Resolve a tab index against the context's pages or throw a RangeError. */ +export function tabAt(session: SessionData, index: number): Page { + const all = session.context.pages(); + const page = all[index]; + if (!page) throw new RangeError(`invalid_tab_index: ${index} (session has ${all.length} tabs, 0-based)`); + return page; +} + +/** + * Wire health (pageOnly) and network listeners on a page's first selection — + * popups arrive unwired — and return its NetworkLog. Idempotent: an already + * wired page just gets its existing log back. Call AFTER `session.page` was + * re-pointed to `page` (attachHealth guards listeners by that identity). + */ +export function wireTab(session: SessionData, page: Page): NetworkLog { + let logs = pageLogs.get(page); + if (!logs) { + logs = attachListeners(page); + pageLogs.set(page, logs); + } + if (!healthWired.has(page)) { + healthWired.add(page); + attachHealth(session, { pageOnly: true }); + } + return logs; +} diff --git a/src/session/tabs.ts b/src/session/tabs.ts new file mode 100644 index 0000000..7fd511d --- /dev/null +++ b/src/session/tabs.ts @@ -0,0 +1,79 @@ +/** + * Multi-tab helpers over a live session: list, select, open and close the + * pages (tabs, OAuth popups) of the session's browser context. + * @module session/tabs + */ +import type { Page } from "playwright"; +import { DEFAULT_GOTO, gotoWithRetry } from "../net/navigate.js"; +import type { SessionData } from "./session.js"; +import { primeTab, seedActive, tabAt, wireTab } from "./tabs-wiring.js"; + +/** One tab of a session, as reported to the agent. */ +export interface TabInfo { + index: number; + url: string; + title: string; + active: boolean; +} + +/** List the context's pages with index, url, title and the active marker. */ +export async function listTabs(session: SessionData): Promise { + return Promise.all( + session.context.pages().map(async (page, index) => ({ + index, + url: page.url(), + title: await page.title().catch(() => ""), + active: page === session.page, + })), + ); +} + +/** + * Make the tab at `index` the session's active page: re-points `session.page`, + * `session.lastUrl` and `session.logs`. Health/network listeners are wired on + * first selection only; each page keeps its own NetworkLog across switches. + */ +export async function selectTab(session: SessionData, index: number): Promise { + seedActive(session); + const page = tabAt(session, index); + session.page = page; + session.lastUrl = page.url(); + session.logs = wireTab(session, page); + await page.bringToFront().catch(() => {}); + return page; +} + +/** + * Open a new tab (optionally navigated to `url` with retry) and select it. + * On navigation failure the orphan page is closed before rethrowing. + */ +export async function openTab(session: SessionData, url?: string): Promise { + const page = await session.context.newPage(); + // Wire the network log before navigating so the tab's first document and + // subresources are captured (listeners attached after goto miss them). + primeTab(page); + if (url) { + try { + await gotoWithRetry(page, url, DEFAULT_GOTO); + } catch (err) { + await page.close().catch(() => {}); + throw err; + } + } + await selectTab(session, session.context.pages().indexOf(page)); + return page; +} + +/** + * Close the tab at `index`. Closing the last tab is refused. When the active + * tab is closed, the first remaining tab is selected BEFORE closing so the + * health listener's identity guard keeps the session healthy. + */ +export async function closeTab(session: SessionData, index: number): Promise { + const all = session.context.pages(); + if (all.length <= 1) + throw new Error("cannot_close_last_tab: a session needs one open tab — use browser_close instead"); + const page = tabAt(session, index); + if (page === session.page) await selectTab(session, all.findIndex((p) => p !== page)); + await page.close(); +} diff --git a/src/session/ttl-guard.ts b/src/session/ttl-guard.ts new file mode 100644 index 0000000..55fcc23 --- /dev/null +++ b/src/session/ttl-guard.ts @@ -0,0 +1,58 @@ +/** + * Per-id TTL timers guarded by a busy counter: expiry is deferred while a + * session is in use by a tool, instead of closing it mid-call. + * @module session/ttl-guard + */ + +/** Schedules per-id expiry timers and defers them while the id is busy. */ +export class TtlGuard { + private readonly timers = new Map(); + private readonly busy = new Map(); + + /** + * @param ttlMs - Idle TTL before `onExpire` fires (ms). + * @param onExpire - Called when an id expires while not busy. + */ + constructor( + private readonly ttlMs: number, + private readonly onExpire: (id: string) => void, + ) {} + + /** (Re)schedule the expiry timer for an id. */ + schedule(id: string): void { + const existing = this.timers.get(id); + if (existing) clearTimeout(existing); + const timer = setTimeout(() => this.expire(id), this.ttlMs); + timer.unref?.(); + this.timers.set(id, timer); + } + + /** Timer callback: reschedule while busy, expire otherwise. */ + private expire(id: string): void { + if ((this.busy.get(id) ?? 0) > 0) { + this.schedule(id); + return; + } + this.onExpire(id); + } + + /** Increment the busy count (a tool is holding the session). */ + markBusy(id: string): void { + this.busy.set(id, (this.busy.get(id) ?? 0) + 1); + } + + /** Decrement the busy count (never below zero). */ + markIdle(id: string): void { + const next = Math.max(0, (this.busy.get(id) ?? 0) - 1); + if (next === 0) this.busy.delete(id); + else this.busy.set(id, next); + } + + /** Drop the timer and busy entry for an id (explicit close). */ + clear(id: string): void { + const timer = this.timers.get(id); + if (timer) clearTimeout(timer); + this.timers.delete(id); + this.busy.delete(id); + } +} diff --git a/tests/integration/mcp.test.ts b/tests/integration/mcp.test.ts index b28bfe1..5a285e0 100644 --- a/tests/integration/mcp.test.ts +++ b/tests/integration/mcp.test.ts @@ -41,10 +41,12 @@ const EXPECTED = [ "browser_wait", "browser_login", "browser_snapshot", + "browser_autoscroll", "browser_act", "browser_collect", "browser_extract", "browser_extract_schema", + "browser_products", "browser_screenshot", "browser_inspect", "browser_visual_diff", @@ -56,6 +58,11 @@ const EXPECTED = [ "browser_metrics", "browser_live_view", "browser_live_view_stop", + "browser_tabs", + "browser_dialog", + "browser_downloads", + "browser_console", + "browser_network", ]; test("MCP exposes the expected tool set with no duplicates", async () => { diff --git a/tests/live/live-booking.ts b/tests/live/live-booking.ts new file mode 100644 index 0000000..7358cce --- /dev/null +++ b/tests/live/live-booking.ts @@ -0,0 +1,87 @@ +/** + * Live test against Booking.com (real anti-bot, consent wall, hotel prices): + * full stealth+consent+currency pipeline via browser_probe on a hotel page, + * then the new tools (network, console, snapshot, extract) on search results. + * Timezone is left to the host (coherent with the real exit IP — forcing a + * mismatching TZ is a strong bot signal). Run: `node --import tsx tests/live/live-booking.ts` + * @module tests/live/live-booking + */ +import { check, connect, type NetRow, payload, state } from "./live-checks.js"; + +type Mcp = Awaited>; +type Challenges = { captcha?: boolean; cloudflare?: boolean; turnstile?: boolean; hcaptcha?: boolean }; + +const PROBE_URL = + "https://www.booking.com/searchresults.html?ss=Paris&checkin=2026-07-15&checkout=2026-07-17&group_adults=2&no_rooms=1&selected_currency=EUR&lang=fr-fr"; +const SEARCH = + "https://www.booking.com/searchresults.html?ss=Amsterdam&checkin=2026-07-15&checkout=2026-07-17&group_adults=2&no_rooms=1&selected_currency=EUR&lang=en-gb"; + +/** A) Full pipeline via browser_probe: stealth must reach a real Booking page. */ +async function probeHotel(client: Mcp): Promise { + const r = payload( + await client.callTool({ + name: "browser_probe", + arguments: { url: PROBE_URL, currency: "EUR", autoConsent: true, humanMode: true, extractPrices: true, detectChallenges: true, waitMs: 4000 }, + }), + ); + const title = String(r.title ?? ""); + const text = String(r.text ?? ""); + check("probe: page Booking réelle atteinte (pas un blocage 403)", title.length > 0 && text.length > 400, + `title="${title.slice(0, 60)}", text=${text.length} chars`); + const ch = (r.challenges ?? {}) as Challenges; + const blocked = Boolean(ch.captcha || ch.cloudflare || ch.turnstile || ch.hcaptcha); + check("probe: pas de challenge bloquant (stealth OK)", !blocked, `challenges=${JSON.stringify(ch)}`); + const offers = r.hotelOffers as { headline?: { amount: number; currency: string }; options?: unknown[] } | null; + const prices = (r.prices as Array<{ currency: string; amount: number }> | undefined) ?? []; + console.log(` ↳ prix=${prices.length}, headline=${JSON.stringify(offers?.headline ?? null)}, options=${offers?.options?.length ?? 0}, consent=${JSON.stringify(r.consent ?? null)}`); + check("probe: extraction de prix sur la page Booking", prices.length >= 1 || !!offers?.headline, + `${prices.length} prix, ex: ${JSON.stringify(prices[0] ?? null)}`); +} + +/** B) New tools live on the search-results page (heavier, WAF-guarded). */ +async function searchResults(client: Mcp): Promise { + const open = payload(await client.callTool({ name: "browser_open", arguments: { currency: "EUR", humanMode: true, blockResources: ["image", "media", "font"] } })); + const sid = String(open.sessionId); + const nav = payload(await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: SEARCH, waitMs: 4000 } })); + check("search: navigation aboutie sur booking.com", String(nav.url).includes("booking.com"), `url=${String(nav.url).slice(0, 60)}`); + + const net = payload(await client.callTool({ name: "browser_network", arguments: { sessionId: sid, urlContains: "booking.com", limit: 80 } })); + const rows = (net.requests as NetRow[] | undefined) ?? []; + // The 80-entry FIFO buffer evicts the initial document on a request-heavy site + // like Booking; assert real captured traffic (200s) instead of the doc itself. + const ok200 = rows.filter((r) => r.status === 200); + check("search: trafic booking.com réel capturé (≥3 réponses 200)", ok200.length >= 3, + `${rows.length} req booking.com, ${ok200.length} en 200, types=${JSON.stringify([...new Set(rows.map((r) => r.resourceType))])}`); + check("search: blocage images actif (aucune image en 200)", !rows.some((r) => r.resourceType === "image" && r.status === 200), + `images 200: ${rows.filter((r) => r.resourceType === "image" && r.status === 200).length}`); + + const ext = payload(await client.callTool({ name: "browser_extract", arguments: { sessionId: sid, kind: "all" } })); + const offers = ext.hotelOffers as { options?: unknown[] } | null; + const prices = (ext.prices as Array<{ currency: string }> | undefined) ?? []; + console.log(` ↳ extract: prix=${prices.length}, offres=${offers?.options?.length ?? 0}`); + check("search: browser_extract répond (structure prices/hotelOffers)", Array.isArray(prices) && offers !== undefined, + `${prices.length} prix, EUR=${prices.filter((p) => p.currency === "EUR").length}`); + + const snap = payload(await client.callTool({ name: "browser_snapshot", arguments: { sessionId: sid } })); + check("search: snapshot d'éléments interactifs", Number(snap.count) > 0, `count=${String(snap.count)}`); + const cons = payload(await client.callTool({ name: "browser_console", arguments: { sessionId: sid } })); + check("search: browser_console requêtable", typeof cons.count === "number", `count=${String(cons.count)}`); + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); +} + +async function main(): Promise { + const client = await connect(); + try { + await probeHotel(client); + await searchResults(client); + } finally { + await client.close(); + } + console.log(state.failures === 0 ? "\nRESULT: Booking — pipeline réel OK" : `\nRESULT: ${state.failures} échec(s) (Booking durcit l'anti-bot selon l'IP)`); + process.exit(state.failures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-check-2.ts b/tests/live/live-check-2.ts new file mode 100644 index 0000000..8e24d75 --- /dev/null +++ b/tests/live/live-check-2.ts @@ -0,0 +1,96 @@ +/** + * Live checks #2 (real browser, real network): named profile save/load cycle, + * blockResources network aborts, and MCP progress notifications. + * Run: `node --import tsx tests/live/live-check-2.ts` + * @module tests/live/live-check-2 + */ +import { existsSync, readFileSync, rmSync } from "node:fs"; +import type { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { CallToolResultSchema } from "@modelcontextprotocol/sdk/types.js"; +import { check, connect, type NetRow, payload, state } from "./live-checks.js"; +import { FAKE_ORIGIN, startCookieProxy } from "./live-proxy.js"; + +const HOME = "/tmp/fuse-live-home"; +const PROFILE_FILE = `${HOME}/profiles/livetest.json`; +// Real https page: Playwright never emits network events for data: pages (pw#7280, #34383). +const IMG_PAGE = "https://www.wikipedia.org/"; + +/** 1) Profile: real save on close, then real load (configured-context) on reopen. */ +async function checkProfile(client: Client): Promise { + const proxy = await startCookieProxy(); + const proxyUrl = `http://127.0.0.1:${proxy.port}`; + const open = payload(await client.callTool({ name: "browser_open", arguments: { profile: "livetest", proxyUrl } })); + const sid = String(open.sessionId); + const nav = payload(await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: "https://example.com" } })); + check("profile: navigate https://example.com", nav.title === "Example Domain", `title=${String(nav.title)}`); + await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: `http://${FAKE_ORIGIN}/` } }); + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); + const saved = existsSync(PROFILE_FILE) ? readFileSync(PROFILE_FILE, "utf8") : ""; + check("profile: livetest.json écrit dans FUSE_BROWSER_HOME", saved.length > 0, PROFILE_FILE); + check('profile: storageState contient "example.com" (cookie + origin)', saved.includes("example.com") && saved.includes('"fuse"'), + saved.replace(/\s+/g, " ").slice(0, 200)); + const re = payload(await client.callTool({ name: "browser_open", arguments: { profile: "livetest", proxyUrl } })); + const sid2 = String(re.sessionId); + check("profile: réouverture du même profil sans erreur", typeof re.sessionId === "string" && sid2.length > 0, `sessionId=${sid2}`); + await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid2, url: `http://${FAKE_ORIGIN}/` } }); + check("profile: cookie rejoué au reload (chemin configured-context)", proxy.lastCookie()?.includes("fuse=live") === true, + `Cookie reçu par le proxy: ${String(proxy.lastCookie())}`); + await client.callTool({ name: "browser_close", arguments: { sessionId: sid2 } }); + await proxy.close(); +} + +/** Open (with/without blockResources), load IMG_PAGE, return its image network rows. */ +async function imageRows(client: Client, blocked: boolean): Promise { + const args = blocked ? { blockResources: ["image"] } : {}; + const open = payload(await client.callTool({ name: "browser_open", arguments: args })); + const sid = String(open.sessionId); + const nav = payload(await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: IMG_PAGE, waitMs: 2500 } })); + check(`blockResources: document non bloqué (${blocked ? "avec" : "sans"} blocage)`, + String(nav.url).startsWith("https://www.wikipedia.org"), `url=${String(nav.url).slice(0, 48)}`); + const net = payload(await client.callTool({ name: "browser_network", arguments: { sessionId: sid, limit: 80 } })); + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); + return ((net.requests as NetRow[] | undefined) ?? []).filter((r) => r.resourceType === "image"); +} + +/** 2) blockResources: images aborted (request seen, no response); counter-proof gets 200s. */ +async function checkBlockResources(client: Client): Promise { + const aborted = await imageRows(client, true); + const allAborted = aborted.length >= 1 && aborted.every((r) => r.status === undefined); + check("blockResources: images abortées (requêtes visibles, aucune réponse 200)", allAborted, + `${aborted.length} image(s), ex: ${JSON.stringify(aborted[0] ?? null)}`); + const free = await imageRows(client, false); + check("blockResources: contre-épreuve sans blocage → au moins une image en 200", + free.some((r) => r.status === 200), `${free.length} image(s)`); +} + +/** 3) Progress notifications received client-side during browser_fetch_batch. */ +async function checkProgress(client: Client): Promise { + const events: Array<{ progress: number; total?: number }> = []; + await client.callTool( + { name: "browser_fetch_batch", arguments: { urls: ["https://example.com", "https://example.org"] } }, + CallToolResultSchema, + { onprogress: (p) => events.push({ progress: p.progress, total: p.total }), resetTimeoutOnProgress: true }, + ); + const ok = events.length >= 1 && events.every((e) => typeof e.progress === "number") && events.some((e) => e.total === 2); + check("progress: notifications onprogress reçues (progress numérique, total=2)", ok, JSON.stringify(events)); +} + +async function main(): Promise { + rmSync(HOME, { recursive: true, force: true }); + const client = await connect({ FUSE_BROWSER_HOME: HOME }); + try { + await checkProfile(client); + await checkBlockResources(client); + await checkProgress(client); + } finally { + await client.close(); + rmSync(HOME, { recursive: true, force: true }); + } + console.log(state.failures === 0 ? "\nRESULT: tout passe en conditions réelles" : `\nRESULT: ${state.failures} échec(s)`); + process.exit(state.failures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-check.ts b/tests/live/live-check.ts new file mode 100644 index 0000000..08203e7 --- /dev/null +++ b/tests/live/live-check.ts @@ -0,0 +1,95 @@ +/** + * Live non-regression check for fuse-browser MCP server (real browser, real sites). + * 1) tools/list contains every pre-change tool + the 5 new ones + * 2) FUSE_CAPS=core filters tool exposure + * 3) New tools exercised live: tabs, dialog, downloads, console, network, + * visual_diff path validation. + * + * Run: `node --import tsx tests/live/live-check.ts` + * @module tests/live/live-check + */ +import { BASELINE, NEW_TOOLS, check, connect, payload, state } from "./live-checks.js"; + +/** Minimal shape of a tab entry returned by browser_tabs. */ +interface TabEntry { + url?: string; +} + +async function main(): Promise { + // 1) Full server: no tool regression + const full = await connect(); + const tools = (await full.listTools()).tools.map((t) => t.name); + const missing = BASELINE.filter((t) => !tools.includes(t)); + check(`baseline intact (${BASELINE.length} tools d'origine)`, missing.length === 0, + missing.length ? `MANQUANTS: ${missing.join(", ")}` : `${tools.length} tools exposés`); + const missingNew = NEW_TOOLS.filter((t) => !tools.includes(t)); + check("5 nouveaux tools présents", missingNew.length === 0, + missingNew.length ? `manquants: ${missingNew.join(", ")}` : NEW_TOOLS.join(", ")); + + // 2) FUSE_CAPS filtering + const coreOnly = await connect({ FUSE_CAPS: "core" }); + const coreTools = (await coreOnly.listTools()).tools.map((t) => t.name); + check("FUSE_CAPS=core filtre les batchs", !coreTools.includes("browser_probe") && coreTools.includes("browser_navigate"), + `${coreTools.length} tools (vs ${tools.length})`); + await coreOnly.close(); + + // 3) Real session on a real site + const open = payload(await full.callTool({ name: "browser_open", arguments: { url: "https://example.com" } })); + const sessionId = open.sessionId as string; + check("browser_open sur example.com", typeof sessionId === "string" && sessionId.length > 0, `sessionId=${sessionId}`); + + // tabs + const tabNew = payload(await full.callTool({ name: "browser_tabs", arguments: { sessionId, action: "new", url: "https://example.org" } })); + const newTabs = tabNew.tabs as TabEntry[] | undefined; + check("browser_tabs new (example.org)", newTabs?.length === 2, `tabs=${newTabs?.length}, active=${tabNew.active}`); + const tabSel = payload(await full.callTool({ name: "browser_tabs", arguments: { sessionId, action: "select", index: 0 } })); + const selTabs = tabSel.tabs as TabEntry[] | undefined; + check("browser_tabs select 0", tabSel.active === 0, JSON.stringify(selTabs?.map((t) => t.url))); + const tabClose = payload(await full.callTool({ name: "browser_tabs", arguments: { sessionId, action: "close", index: 1 } })); + check("browser_tabs close 1", (tabClose.tabs as TabEntry[] | undefined)?.length === 1); + + // dialog: arm accept, auto-fire a confirm() on load + await full.callTool({ name: "browser_dialog", arguments: { sessionId, action: "accept" } }); + const dlgPage = "data:text/html,dlg"; + await full.callTool({ name: "browser_navigate", arguments: { sessionId, url: dlgPage } }); + await new Promise((r) => setTimeout(r, 800)); + const dlg = payload(await full.callTool({ name: "browser_dialog", arguments: { sessionId, action: "accept" } })); + const recent = (dlg.recent ?? []) as Array<{ type: string }>; + check("browser_dialog capture le confirm()", recent.some((d) => d.type === "confirm"), + `recent=${JSON.stringify(recent.slice(0, 2))}`); + + // downloads: auto-click a download anchor + const dlPage = "data:text/html,dl"; + await full.callTool({ name: "browser_navigate", arguments: { sessionId, url: dlPage } }); + await new Promise((r) => setTimeout(r, 1500)); + const dls = payload(await full.callTool({ name: "browser_downloads", arguments: { sessionId } })); + const downloads = dls.downloads as unknown[] | undefined; + check("browser_downloads capture le fichier", ((dls.count as number) ?? 0) >= 1, JSON.stringify(downloads?.[0] ?? null)); + + // console + network on a real site + await full.callTool({ name: "browser_navigate", arguments: { sessionId, url: "https://example.com" } }); + const net = payload(await full.callTool({ name: "browser_network", arguments: { sessionId } })); + const requests = net.requests as unknown[] | undefined; + check("browser_network voit les requêtes", ((net.count as number) ?? 0) >= 1, + `${net.count} requêtes, ex: ${JSON.stringify(requests?.[0] ?? null).slice(0, 140)}`); + const cons = await full.callTool({ name: "browser_console", arguments: { sessionId } }); + check("browser_console répond", !(cons as { isError?: boolean }).isError, `count=${payload(cons).count}`); + + // visual_diff path hardening + const bad = await full.callTool({ name: "browser_visual_diff", arguments: { a: "/etc/passwd", b: "/tmp/x.txt" } }); + check("browser_visual_diff rejette les non-.png", (bad as { isError?: boolean }).isError === true, + (payload(bad)._raw as string | undefined)?.slice(0, 100) ?? JSON.stringify(payload(bad)).slice(0, 100)); + + // structured error code on unknown session + const lost = await full.callTool({ name: "browser_navigate", arguments: { sessionId: "deadbeef0000", url: "https://example.com" } }); + const lostPayload = (lost as { structuredContent?: { code?: string } }).structuredContent; + check("code erreur structuré (session_not_found)", lostPayload?.code === "session_not_found", JSON.stringify(lostPayload)); + + await full.callTool({ name: "browser_close", arguments: { sessionId } }); + await full.close(); + + console.log(state.failures === 0 ? "\nRESULT: ZERO REGRESSION — tout passe en conditions réelles" : `\nRESULT: ${state.failures} échec(s)`); + process.exit(state.failures === 0 ? 0 : 1); +} + +main().catch((err) => { console.error("FATAL:", err); process.exit(1); }); diff --git a/tests/live/live-checks.ts b/tests/live/live-checks.ts new file mode 100644 index 0000000..250a017 --- /dev/null +++ b/tests/live/live-checks.ts @@ -0,0 +1,77 @@ +/** + * Shared helpers for the live non-regression harness: MCP connection, + * payload extraction, and the PASS/FAIL assertion counter. + * @module tests/live/live-checks + */ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; + +/** Absolute project root (cwd for the spawned MCP server). */ +export const ROOT = + "/Users/brunoazoulay/Labo/docker-lab/dev.local/Dev-ai/claude-code/fuse-browser"; + +/** Tools exposed before this changeset — must all survive. */ +export const BASELINE = [ + "browser_probe", "browser_probe_html", "browser_fetch", "browser_fetch_batch", + "browser_crawl", "browser_collect_batch", "browser_shots_batch", "browser_site_shots", + "browser_serp_batch", "browser_open", "browser_status", "browser_close", + "browser_connect", "browser_navigate", "browser_click", "browser_fill", + "browser_scroll", "browser_press", "browser_select", "browser_back", + "browser_forward", "browser_wait", "browser_login", "browser_snapshot", + "browser_act", "browser_collect", "browser_wait_for", "browser_run", + "browser_extract", "browser_extract_schema", "browser_screenshot", + "browser_inspect", "browser_visual_diff", "browser_handoff", + "browser_live_view", "browser_live_view_stop", "browser_metrics", +]; + +/** Tools introduced by this changeset. */ +export const NEW_TOOLS = [ + "browser_tabs", "browser_dialog", "browser_downloads", "browser_console", "browser_network", +]; + +/** Loosely-typed decoded tool payload (server returns JSON of varying shape). */ +export type Payload = Record; + +/** Merged network row returned by `browser_network`. */ +export interface NetRow { + url: string; + status?: number; + resourceType?: string; +} + +/** Mutable assertion counter shared across the run. */ +export const state = { failures: 0 }; + +/** Record a PASS/FAIL line and increment the failure counter on miss. */ +export function check(label: string, ok: boolean, detail?: string): void { + console.log(`${ok ? "PASS" : "FAIL"} ${label}${detail ? ` — ${detail}` : ""}`); + if (!ok) state.failures += 1; +} + +/** Spawn the MCP server over stdio and return a connected client. */ +export async function connect(env?: Record): Promise { + const transport = new StdioClientTransport({ + command: "node", + args: ["--import", "tsx", "src/bin/mcp.ts"], + cwd: ROOT, + env: { ...(process.env as Record), ...env }, + }); + const client = new Client({ name: "live-check", version: "0.0.1" }); + await client.connect(transport); + return client; +} + +/** Extract structured content (or parse the text block) from a tool result. */ +export function payload(res: unknown): Payload { + const r = res as { + content?: Array<{ type: string; text?: string }>; + structuredContent?: Payload; + }; + if (r.structuredContent) return r.structuredContent; + const text = (r.content ?? []).filter((c) => c.type === "text").map((c) => c.text).join("\n"); + try { + return JSON.parse(text) as Payload; + } catch { + return { _raw: text }; + } +} diff --git a/tests/live/live-cli-run.ts b/tests/live/live-cli-run.ts new file mode 100644 index 0000000..c4965a9 --- /dev/null +++ b/tests/live/live-cli-run.ts @@ -0,0 +1,51 @@ +/** + * Shared spawner for the live CLI checks: run the real CLI binary in a child + * process and capture `{ code, stdout, stderr }`. Used by `live-cli.ts`. + * @module tests/live/live-cli-run + */ +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +/** Absolute path to the CLI entry point. */ +const CLI = fileURLToPath(new URL("../../src/bin/cli.ts", import.meta.url)); + +/** Result of one CLI invocation. */ +export interface CliResult { + code: number; + stdout: string; + stderr: string; +} + +/** + * Spawn `node --import tsx src/bin/cli.ts ` and resolve with its output. + * + * @param args - CLI arguments (command, url, flags). + * @returns Captured exit code and streams. + */ +export function runCli(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn("node", ["--import", "tsx", CLI, ...args], { + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (d) => { + stdout += String(d); + }); + child.stderr.on("data", (d) => { + stderr += String(d); + }); + child.on("error", reject); + child.on("close", (code) => resolve({ code: code ?? 0, stdout, stderr })); + }); +} + +/** Parse stdout as JSON, returning `null` on failure (keeps assertions terse). */ +export function parseJson(stdout: string): Record | null { + try { + return JSON.parse(stdout) as Record; + } catch { + return null; + } +} diff --git a/tests/live/live-cli.ts b/tests/live/live-cli.ts new file mode 100644 index 0000000..c71a39f --- /dev/null +++ b/tests/live/live-cli.ts @@ -0,0 +1,72 @@ +/** + * Live non-regression + parity check for the 6 new one-shot CLI page commands. + * Spawns the real CLI binary against real sites and validates JSON output / exit + * codes, plus a regression sweep (15 commands in --help, probe still works). + * + * Run: `node --import tsx tests/live/live-cli.ts` + * @module tests/live/live-cli + */ +import { type CliResult, parseJson, runCli } from "./live-cli-run.js"; + +let failures = 0; + +/** Assert `cond`; print PASS/FAIL with a short detail line. */ +function check(name: string, cond: boolean, detail = ""): void { + if (cond) { + console.log(`PASS ${name}${detail ? ` — ${detail}` : ""}`); + return; + } + failures += 1; + console.log(`FAIL ${name}${detail ? ` — ${detail}` : ""}`); +} + +/** Run a command, parse stdout JSON, and apply `assert` to it. */ +async function probeCmd(args: string[], assert: (r: CliResult, j: Record | null) => void): Promise { + const r = await runCli(args); + assert(r, parseJson(r.stdout)); +} + +async function newCommands(): Promise { + await probeCmd(["products", "https://www.digitec.ch/en/search?q=macbook", "--wait-ms", "5000"], (r, j) => + check("products → non-empty", r.code === 0 && Array.isArray(j?.products) && (j?.count as number) > 0, `count=${j?.count}`)); + await probeCmd(["extract", "https://example.com", "--kind", "text"], (r, j) => + check("extract text → non-empty", r.code === 0 && typeof j?.text === "string" && (j.text as string).length > 0, `${(j?.text as string)?.slice(0, 30)}…`)); + await probeCmd(["snapshot", "https://example.com"], (r, j) => + check("snapshot → count > 0", r.code === 0 && (j?.count as number) > 0, `count=${j?.count}`)); + await probeCmd(["run", "https://example.com", "--steps", '[{"type":"wait","ms":500},{"type":"extract","kind":"text"}]'], (r, j) => + check("run → ok:true", r.code === 0 && j?.ok === true, `steps=${(j?.steps as unknown[])?.length}`)); +} + +async function screenshotAndInspect(): Promise { + const out = "/tmp/cli-shot.png"; + const shot = await runCli(["screenshot", "https://example.com", "--full-page", "--output", out]); + const { existsSync, statSync } = await import("node:fs"); + check("screenshot → file written", shot.code === 0 && existsSync(out) && statSync(out).size > 0, + existsSync(out) ? `${statSync(out).size} bytes` : "missing"); + // inspect needs a ref from the snapshot; example.com's single link is ref "0". + await probeCmd(["inspect", "https://example.com", "--ref", "0"], (r, j) => + check("inspect → style present", r.code === 0 && j?.style != null, `ref=${j?.ref}`)); +} + +async function regression(): Promise { + const help = await runCli(["--help"]); + const cmds = ["probe", "fetch", "fetch-batch", "crawl", "collect-batch", "serp-batch", "shots", "shots-batch", + "site-shots", "run", "products", "extract", "snapshot", "screenshot", "inspect"]; + const listed = cmds.filter((c) => new RegExp(`\\n ${c} `).test(help.stdout)); + check("--help lists 15 commands", listed.length === 15, `listed=${listed.length}: ${cmds.filter((c) => !listed.includes(c)).join(",") || "all"}`); + await probeCmd(["probe", "https://example.com", "--extract-prices"], (r, j) => + check("probe regression", r.code === 0 && typeof j?.url === "string", `url=${j?.url}`)); +} + +async function main(): Promise { + await newCommands(); + await screenshotAndInspect(); + await regression(); + console.log(failures === 0 ? "\nRESULT: all CLI commands PASS (zero regression)" : `\nRESULT: ${failures} failure(s)`); + process.exit(failures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-digitec.ts b/tests/live/live-digitec.ts new file mode 100644 index 0000000..daee9b0 --- /dev/null +++ b/tests/live/live-digitec.ts @@ -0,0 +1,71 @@ +/** + * Live task on the LOCAL (modified-code) MCP server: find the cheapest real + * MacBook laptop on digitec.ch. Loads the search, scrolls to load more cards, + * extracts the page text and pairs each price with the product title on the + * next line (digitec renders category / "CHF" / amount / title as separate + * lines), then keeps only "Apple MacBook" laptops (not third-party accessories). + * Run: `node --import tsx tests/live/live-digitec.ts` + * @module tests/live/live-digitec + */ +import { connect, payload } from "./live-checks.js"; + +const URL = "https://www.digitec.ch/en/search?q=macbook"; + +/** A parsed product: price in CHF + its title. */ +interface Product { + amount: number; + title: string; +} + +/** Parse "1'099.–" / "6.90" / "13.23 currently" → number (CHF). */ +function parseAmount(line: string): number | null { + const m = line.match(/([0-9][0-9'’]*)(?:[.,]([0-9]{2}))?/); + if (!m) return null; + const whole = Number((m[1] ?? "").replace(/['’]/g, "")); + return m[2] ? whole + Number(m[2]) / 100 : whole; +} + +/** Pair each "CHF\n\n" block; keep Apple MacBook laptops. */ +function macbooks(text: string): Product[] { + const lines = text.split("\n").map((l) => l.trim()).filter(Boolean); + const out: Product[] = []; + for (let i = 0; i < lines.length - 2; i += 1) { + if (lines[i] !== "CHF") continue; + const amount = parseAmount(lines[i + 1] ?? ""); + const title = lines[i + 2] ?? ""; + if (amount && /apple\s+macbook/i.test(title)) out.push({ amount, title }); + } + return out; +} + +async function main(): Promise<void> { + const client = await connect(); + const open = payload(await client.callTool({ name: "browser_open", arguments: { currency: "CHF", humanMode: true, blockResources: ["media", "font"] } })); + const sid = String(open.sessionId); + const nav = payload(await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: URL, waitMs: 5000 } })); + console.log(`nav: "${String(nav.title).slice(0, 60)}"`); + await client.callTool({ name: "browser_wait_for", arguments: { sessionId: sid, text: "Apple MacBook", timeoutMs: 15_000 } }).catch(() => {}); + for (let s = 0; s < 4; s += 1) { + await client.callTool({ name: "browser_scroll", arguments: { sessionId: sid, deltaY: 2400 } }); + await client.callTool({ name: "browser_wait", arguments: { sessionId: sid, ms: 900 } }).catch(() => {}); + } + + const txt = payload(await client.callTool({ name: "browser_extract", arguments: { sessionId: sid, kind: "text", format: "text" } })); + const found = macbooks(String(txt.text ?? "")).sort((a, b) => a.amount - b.amount); + const seen = new Set<string>(); + const unique = found.filter((p) => (seen.has(p.title) ? false : seen.add(p.title))); + + console.log(`\nApple MacBooks trouvés: ${unique.length}`); + for (const p of unique.slice(0, 12)) console.log(` CHF ${p.amount.toFixed(2)} | ${p.title.slice(0, 80)}`); + const cheapest = unique[0]; + console.log(cheapest ? `\n>>> LE MOINS CHER: CHF ${cheapest.amount.toFixed(2)} — ${cheapest.title}` : "\n>>> aucun MacBook laptop détecté"); + + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); + await client.close(); + process.exit(cheapest ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-milan.ts b/tests/live/live-milan.ts new file mode 100644 index 0000000..2e52f13 --- /dev/null +++ b/tests/live/live-milan.ts @@ -0,0 +1,62 @@ +/** + * Live task on the LOCAL (modified-code) MCP server: cheapest hotel ROOM in Milan + * on booking.com (not activities/tickets). Sorts by price, scrolls to load cards, + * and extracts only the amounts anchored to "N nights, M adults" — the per-stay + * room totals — then reports the lowest with its hotel-card context. + * Run: `node --import tsx tests/live/live-milan.ts` + * @module tests/live/live-milan + */ +import { connect, payload } from "./live-checks.js"; + +const URL = + "https://www.booking.com/searchresults.html?ss=Milan&checkin=2026-07-15&checkout=2026-07-17&group_adults=2&no_rooms=1&selected_currency=EUR&order=price&lang=en-gb"; + +/** A room offer parsed from the card text. */ +interface Room { + amount: number; + context: string; +} + +/** Extract per-stay room totals: amounts directly tied to "N nights, M adults". */ +function rooms(text: string): Room[] { + const flat = text.replace(/[‎‏ ]/g, " "); + const re = /nights?,\s*\d+\s*adults?\s*€\s*([\d',.]+)/gi; + const out: Room[] = []; + for (const m of flat.matchAll(re)) { + const amount = Number((m[1] ?? "").replace(/['\s]/g, "").replace(/\.(?=\d{3})/g, "")); + const at = m.index ?? 0; + if (amount >= 30) out.push({ amount, context: flat.slice(Math.max(0, at - 220), at).replace(/\n{2,}/g, "\n").trim() }); + } + return out; +} + +async function main(): Promise<void> { + const client = await connect(); + const open = payload(await client.callTool({ name: "browser_open", arguments: { currency: "EUR", humanMode: true, blockResources: ["media", "font", "image"] } })); + const sid = String(open.sessionId); + const nav = payload(await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: URL, waitMs: 6000 } })); + console.log(`nav: "${String(nav.title).slice(0, 70)}"`); + await client.callTool({ name: "browser_wait_for", arguments: { sessionId: sid, text: "adults", timeoutMs: 15_000 } }).catch(() => {}); + for (let s = 0; s < 3; s += 1) { + await client.callTool({ name: "browser_scroll", arguments: { sessionId: sid, deltaY: 2600 } }); + await client.callTool({ name: "browser_wait", arguments: { sessionId: sid, ms: 1000 } }).catch(() => {}); + } + + const txt = payload(await client.callTool({ name: "browser_extract", arguments: { sessionId: sid, kind: "text", format: "text" } })); + const found = rooms(String(txt.text ?? "")).sort((a, b) => a.amount - b.amount); + const seen = new Set<number>(); + const uniq = found.filter((r) => (seen.has(r.amount) ? false : seen.add(r.amount))); + console.log(`\nchambres (prix séjour 2 nuits, 2 adultes): ${uniq.length}`); + for (const r of uniq.slice(0, 10)) console.log(` EUR ${r.amount.toFixed(2)}`); + + const cheapest = uniq[0]; + console.log(cheapest ? `\n>>> CHAMBRE LA MOINS CHÈRE: EUR ${cheapest.amount.toFixed(2)} (15–17 juil, 2 adultes)\n--- carte ---\n${cheapest.context.slice(-320)}` : "\n>>> aucune chambre détectée"); + + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); + await client.close(); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-new-tools.ts b/tests/live/live-new-tools.ts new file mode 100644 index 0000000..73e4509 --- /dev/null +++ b/tests/live/live-new-tools.ts @@ -0,0 +1,56 @@ +/** + * Live validation of the new tools on the LOCAL server: browser_autoscroll + + * browser_products give the cheapest MacBook on digitec WITHOUT client-side + * parsing (the whole point of structured per-card extraction). Also checks the + * tools are exposed and the screenshot:// resource works. + * Run: `node --import tsx tests/live/live-new-tools.ts` + * @module tests/live/live-new-tools + */ +import { check, connect, payload, state } from "./live-checks.js"; + +const URL = "https://www.digitec.ch/en/search?q=macbook"; + +/** A structured product from browser_products. */ +interface Product { + title: string; + price: number; + currency: string; +} + +async function main(): Promise<void> { + const client = await connect(); + const tools = (await client.listTools()).tools.map((t) => t.name); + check("browser_products exposé", tools.includes("browser_products")); + check("browser_autoscroll exposé", tools.includes("browser_autoscroll")); + + const open = payload(await client.callTool({ name: "browser_open", arguments: { currency: "CHF", humanMode: true, blockResources: ["media", "font"] } })); + const sid = String(open.sessionId); + await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: URL, waitMs: 4000 } }); + await client.callTool({ name: "browser_wait_for", arguments: { sessionId: sid, text: "Apple MacBook", timeoutMs: 15_000 } }).catch(() => {}); + + const scroll = payload(await client.callTool({ name: "browser_autoscroll", arguments: { sessionId: sid, maxScrolls: 5, idleRounds: 2 } })); + check("browser_autoscroll a chargé la liste", Number(scroll.rounds) >= 1, `rounds=${String(scroll.rounds)}, height=${String(scroll.height)}`); + + const prod = payload(await client.callTool({ name: "browser_products", arguments: { sessionId: sid, limit: 80 } })); + const all = (prod.products as Product[] | undefined) ?? []; + check("browser_products renvoie des items structurés {title, price}", all.length > 0, `count=${all.length}`); + const macs = all.filter((p) => /apple\s+macbook/i.test(p.title)).sort((a, b) => a.price - b.price); + check("MacBooks isolés et triés par prix", macs.length >= 1, `${macs.length} MacBooks`); + for (const m of macs.slice(0, 6)) console.log(` ${m.currency} ${m.price} | ${m.title.slice(0, 70)}`); + const cheapest = macs[0]; + console.log(cheapest ? `\n>>> via browser_products: LE MOINS CHER = ${cheapest.currency} ${cheapest.price} — ${cheapest.title}` : "\n>>> aucun MacBook"); + + const res = await client.readResource({ uri: `screenshot://${sid}/last` }).catch((e: unknown) => ({ err: String(e) })); + const blob = (res as { contents?: Array<{ blob?: string }> }).contents?.[0]?.blob; + check("resource screenshot:// renvoie une image", (blob?.length ?? 0) > 5000, `blob=${blob?.length ?? 0} chars`); + + await client.callTool({ name: "browser_close", arguments: { sessionId: sid } }); + await client.close(); + console.log(state.failures === 0 ? "\nRESULT: nouveaux tools OK en réel" : `\nRESULT: ${state.failures} échec(s)`); + process.exit(state.failures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/live-proxy.ts b/tests/live/live-proxy.ts new file mode 100644 index 0000000..1909627 --- /dev/null +++ b/tests/live/live-proxy.ts @@ -0,0 +1,73 @@ +/** + * Minimal HTTP forward proxy for the live profile check: serves the fake + * origin `http://live.example.com/` itself (real `Set-Cookie` + localStorage + * page), tunnels CONNECT (https) to the real host, and records the last + * `Cookie` request header it sees — proving a storage-state reload. + * @module tests/live/live-proxy + */ +import { createServer } from "node:http"; +import { connect as netConnect, type Socket } from "node:net"; + +/** Fake origin served by the proxy (real example.com stays CONNECT-tunneled). */ +export const FAKE_ORIGIN = "live.example.com"; + +/** HTML served on the fake origin: sets localStorage so the origin is persisted. */ +const PAGE = '<html><body>live<script>localStorage.setItem("fuse", "live");</script></body></html>'; + +/** Running proxy handle. */ +export interface CookieProxy { + /** Ephemeral listen port on 127.0.0.1. */ + port: number; + /** Last `Cookie` request header received on the fake origin. */ + lastCookie: () => string | undefined; + /** Stop the server. */ + close: () => Promise<void>; +} + +/** + * Start the proxy on an ephemeral 127.0.0.1 port. + * @returns Handle with the port, the last seen Cookie header, and a closer. + */ +export async function startCookieProxy(): Promise<CookieProxy> { + let lastCookie: string | undefined; + const tunnels = new Set<Socket>(); + const server = createServer((req, res) => { + if (String(req.url).includes(FAKE_ORIGIN)) { + lastCookie = req.headers.cookie ?? lastCookie; + res.writeHead(200, { "content-type": "text/html", "set-cookie": "fuse=live; Path=/" }); + res.end(PAGE); + return; + } + res.writeHead(502, { "content-type": "text/plain" }); + res.end("proxy: only the fake origin is served over plain http"); + }); + server.on("connect", (req, clientSocket, head) => { + const [host, port] = String(req.url).split(":"); + const upstream = netConnect(Number(port ?? "443"), host ?? "", () => { + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + upstream.write(head); + upstream.pipe(clientSocket); + clientSocket.pipe(upstream); + }); + tunnels.add(clientSocket).add(upstream); + const drop = (s: Socket) => { + tunnels.delete(s); + s.destroy(); + }; + upstream.on("error", () => drop(clientSocket)).on("close", () => tunnels.delete(upstream)); + clientSocket.on("error", () => drop(upstream)).on("close", () => tunnels.delete(clientSocket)); + }); + await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + const port = typeof addr === "object" && addr !== null ? addr.port : 0; + return { + port, + lastCookie: () => lastCookie, + close: () => + new Promise((resolve) => { + for (const s of tunnels) s.destroy(); + tunnels.clear(); + server.close(() => resolve()); + }), + }; +} diff --git a/tests/live/live-scenario.ts b/tests/live/live-scenario.ts new file mode 100644 index 0000000..5b76324 --- /dev/null +++ b/tests/live/live-scenario.ts @@ -0,0 +1,86 @@ +/** + * Live E2E scenario on real websites: full agent workflow on books.toscrape.com + * (navigate, price extraction, snapshot, tabs, network blocking, screenshot). + * Run: `node --import tsx tests/live/live-scenario.ts` + * @module tests/live/live-scenario + */ +import { check, connect, type NetRow, payload, state } from "./live-checks.js"; + +const SHOP = "https://books.toscrape.com/"; +const WIKI = "https://en.wikipedia.org/wiki/Web_scraping"; + +/** Image content block of an MCP CallToolResult. */ +interface ImageBlock { + type: string; + data?: string; +} + +async function main(): Promise<void> { + const client = await connect(); + const open = payload( + await client.callTool({ name: "browser_open", arguments: { blockResources: ["image", "font"] } }), + ); + const sid = String(open.sessionId); + check("open session (blockResources image+font)", sid.length > 0, `sessionId=${sid}`); + + const nav = payload( + await client.callTool({ name: "browser_navigate", arguments: { sessionId: sid, url: SHOP, waitMs: 1200 } }), + ); + check("navigate books.toscrape.com", String(nav.title).includes("Books to Scrape"), `title=${String(nav.title)}`); + + const extract = payload( + await client.callTool({ name: "browser_extract", arguments: { sessionId: sid, kind: "prices" } }), + ); + // The listing shows 20 books: every <article> card's price must be captured, + // not just the first (mainText now joins all matches). + const prices = (extract.prices as Array<{ currency: string; amount: number }> | undefined) ?? []; + check("extract: prix de toute la grille (≥10 GBP distincts)", + prices.length >= 10 && prices.every((p) => p.currency === "GBP"), + `${prices.length} prix, ex: ${JSON.stringify(prices.slice(0, 3))}`); + + const snap = payload(await client.callTool({ name: "browser_snapshot", arguments: { sessionId: sid } })); + check("snapshot: éléments interactifs", Number(snap.count) > 10, `count=${String(snap.count)}`); + + const tabs = payload( + await client.callTool({ name: "browser_tabs", arguments: { sessionId: sid, action: "new", url: WIKI } }), + ); + const tabList = (tabs.tabs as Array<{ url: string }> | undefined) ?? []; + check("tabs: Wikipedia ouvert en onglet 2", tabList.length === 2 && tabList[1]?.url.includes("wikipedia.org"), + `active=${String(tabs.active)}, urls=${JSON.stringify(tabList.map((t) => t.url.slice(0, 40)))}`); + + const wikiNet = payload( + await client.callTool({ name: "browser_network", arguments: { sessionId: sid, limit: 80 } }), + ); + const rows = (wikiNet.requests as NetRow[] | undefined) ?? []; + const imgOk = rows.filter((r) => r.resourceType === "image" && r.status === 200); + const docOk = rows.some((r) => r.resourceType === "document" && r.status === 200); + check("network: document 200, images abortées (blocage actif)", docOk && imgOk.length === 0, + `${rows.length} requêtes, images en 200: ${imgOk.length}`); + + const back = payload( + await client.callTool({ name: "browser_tabs", arguments: { sessionId: sid, action: "select", index: 0 } }), + ); + check("tabs: retour onglet boutique", back.active === 0); + + const shot = (await client.callTool({ + name: "browser_screenshot", + arguments: { sessionId: sid, fullPage: true }, + })) as { content?: ImageBlock[] }; + const img = (shot.content ?? []).find((c) => c.type === "image"); + check("screenshot fullPage capturé", (img?.data?.length ?? 0) > 10_000, `${img?.data?.length ?? 0} chars base64`); + + const cons = payload(await client.callTool({ name: "browser_console", arguments: { sessionId: sid, level: "error" } })); + check("console: requêtable (erreurs JS)", typeof cons.count === "number", `count=${String(cons.count)}`); + + const closed = payload(await client.callTool({ name: "browser_close", arguments: { sessionId: sid } })); + check("close session", closed.closed === true); + + await client.close(); + console.log(state.failures === 0 ? "\nRESULT: scénario réel complet — tout passe" : `\nRESULT: ${state.failures} échec(s)`); + process.exit(state.failures === 0 ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/stealth-bench.ts b/tests/live/stealth-bench.ts new file mode 100644 index 0000000..24c5dce --- /dev/null +++ b/tests/live/stealth-bench.ts @@ -0,0 +1,84 @@ +/** + * Stealth non-regression benchmark: drives the local MCP server's + * `browser_probe` against public anti-bot detection pages and scores + * the resulting page text for known leak signatures (webdriver, + * HeadlessChrome, failed verdicts, missing plugins/languages). + * + * Honest by design: each signal can only FAIL on a positive detection. + * Targets that cannot be reached are reported as UNREACHABLE and do not + * silently inflate the score. + * + * Run: `node --import tsx tests/live/stealth-bench.ts` + * Exit 0 when global pass ratio >= THRESHOLD, else exit 1. + * @module tests/live/stealth-bench + */ +import { connect, payload } from "./live-checks.js"; +import { TARGETS, type Target } from "./stealth-signals.js"; + +/** Minimum global pass ratio (0..1) required for a green run. */ +const THRESHOLD = 0.8; + +/** Per-target outcome accumulator. */ +interface Outcome { + passed: number; + total: number; + reachable: boolean; +} + +/** Probe one target and print its per-signal verdicts. */ +async function scoreTarget( + client: Awaited<ReturnType<typeof connect>>, + target: Target, +): Promise<Outcome> { + const res = await client.callTool({ + name: "browser_probe", + arguments: { url: target.url, waitMs: target.waitMs, detectChallenges: true }, + }); + const text = String((payload(res).text as string | undefined) ?? "").toLowerCase(); + const reachable = text.length > 200; + console.log(`\n# ${target.name} (${target.url}) [${text.length} chars]`); + if (!reachable) { + console.log(" UNREACHABLE — empty/short text, signals skipped"); + return { passed: 0, total: target.signals.length, reachable: false }; + } + let passed = 0; + for (const sig of target.signals) { + const ok = sig.ok(text); + if (ok) passed += 1; + console.log(` ${ok ? "PASS" : "FAIL"} ${sig.label}`); + } + console.log(` SCORE ${passed}/${target.signals.length}`); + return { passed, total: target.signals.length, reachable: true }; +} + +/** Run the full benchmark and exit with the threshold verdict. */ +async function main(): Promise<void> { + console.log(`Stealth bench — ${TARGETS.length} targets, threshold ${THRESHOLD}`); + const client = await connect(); + let passed = 0; + let total = 0; + let unreachable = 0; + try { + for (const target of TARGETS) { + const o = await scoreTarget(client, target); + passed += o.passed; + total += o.total; + if (!o.reachable) unreachable += 1; + } + } finally { + await client.close(); + } + const ratio = total === 0 ? 0 : passed / total; + console.log( + `\nGLOBAL ${passed}/${total} (${(ratio * 100).toFixed(0)}%)` + + (unreachable ? ` — ${unreachable} target(s) UNREACHABLE` : ""), + ); + const green = ratio >= THRESHOLD && unreachable < TARGETS.length; + console.log(green ? "RESULT: STEALTH OK (non-regression)" : "RESULT: STEALTH REGRESSION"); + process.exit(green ? 0 : 1); +} + +main().catch((err) => { + console.error("FATAL:", err); + process.exit(1); +}); diff --git a/tests/live/stealth-signals.ts b/tests/live/stealth-signals.ts new file mode 100644 index 0000000..0475a9e --- /dev/null +++ b/tests/live/stealth-signals.ts @@ -0,0 +1,89 @@ +/** + * Stealth signal definitions and scoring for the anti-bot benchmark. + * Each target declares text-based PASS/FAIL signals extracted from the + * probe report's `text` field. Signals are intentionally conservative: + * a signal can only FAIL on a positive detection, never on absence of + * the page text (that is reported separately as an UNREACHABLE target). + * @module tests/live/stealth-signals + */ + +/** One detectable stealth signal evaluated against lowercased page text. */ +export interface Signal { + /** Human-readable label printed in the report. */ + label: string; + /** Returns true when the signal PASSES (stealth holds). */ + ok: (text: string) => boolean; +} + +/** A detection target: a public, stable anti-bot test page. */ +export interface Target { + /** Short identifier used in the report. */ + name: string; + /** Absolute URL probed via the MCP server. */ + url: string; + /** Milliseconds to wait after load for client-side tests to settle. */ + waitMs: number; + /** Signals evaluated against the rendered text. */ + signals: Signal[]; +} + +/** True when `needle` is absent from `text` (a clean, non-detected state). */ +const absent = (text: string, needle: string): boolean => !text.includes(needle); + +/** + * Read a "NN% <label>" percentage emitted by creepjs (e.g. "33% headless"). + * Returns the integer percentage, or null when the label is not present. + */ +function creepPct(text: string, label: string): number | null { + const m = new RegExp(`(\\d+)\\s*%\\s*${label}`).exec(text); + return m ? Number(m[1]) : null; +} + +/** PASS when the creepjs percentage exists and stays at/below `max`. */ +function creepBelow(text: string, label: string, max: number): boolean { + const pct = creepPct(text, label); + return pct !== null && pct <= max; +} + +/** + * Stable, public detection targets (verified live, 2026-06). + * sannysoft + creepjs + browserleaks each surface complementary signals. + */ +export const TARGETS: Target[] = [ + { + name: "sannysoft", + url: "https://bot.sannysoft.com/", + waitMs: 6000, + signals: [ + { label: "no HeadlessChrome in UA", ok: (t) => absent(t, "headlesschrome") }, + { label: "webdriver not 'present (failed)'", ok: (t) => absent(t, "present (failed)") }, + { label: "no '(failed)' verdicts", ok: (t) => absent(t, "(failed)") }, + { label: "plugins reported (PluginArray)", ok: (t) => t.includes("pluginarray") || t.includes("plugins") }, + ], + }, + { + // creepjs renders its summary as text: "NN% like headless", "NN% headless", + // "NN% stealth" (lower = better, near-0% ideal; confirmed via decodo + + // creepjs source). We score those percentages directly rather than a + // misleading absence check on the word "headless", always present as a label. + name: "creepjs", + url: "https://abrahamjuliot.github.io/creepjs/", + waitMs: 12000, + signals: [ + { label: "no 'webdriver' leak", ok: (t) => absent(t, "webdriver") }, + { label: "headless score <= 50%", ok: (t) => creepBelow(t, "headless", 50) }, + { label: "'like headless' score <= 60%", ok: (t) => creepBelow(t, "like headless", 60) }, + { label: "summary rendered (has headless score)", ok: (t) => creepPct(t, "headless") !== null }, + ], + }, + { + name: "browserleaks-js", + url: "https://browserleaks.com/javascript", + waitMs: 5000, + signals: [ + { label: "webdriver = false", ok: (t) => absent(t, "webdriver true") && !/webdriver\s+true/.test(t) }, + { label: "no HeadlessChrome token", ok: (t) => absent(t, "headlesschrome") }, + { label: "languages present", ok: (t) => t.includes("language") }, + ], + }, +]; diff --git a/tests/unit/auto-scroll.test.ts b/tests/unit/auto-scroll.test.ts new file mode 100644 index 0000000..3a91740 --- /dev/null +++ b/tests/unit/auto-scroll.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { decideStop } from "../../src/actions/auto-scroll.js"; +import type { ScrollProbe } from "../../src/interfaces/auto-scroll.js"; + +const base = { + idle: 0, + rounds: 1, + idleRounds: 2, + maxScrolls: 20, + minCount: 1, + hasSelector: false, +}; + +const probe = (height: number, count = 0): ScrollProbe => ({ height, count }); + +describe("decideStop", () => { + test("continues and resets idle while height grows", () => { + const out = decideStop({ ...base, prev: probe(1000), curr: probe(1500), idle: 1 }); + expect(out).toEqual({ stop: false, idle: 0 }); + }); + + test("increments idle when height is flat", () => { + const out = decideStop({ ...base, prev: probe(2000), curr: probe(2000), idle: 0 }); + expect(out).toEqual({ stop: false, idle: 1 }); + }); + + test("stops after idleRounds consecutive flat rounds", () => { + const out = decideStop({ ...base, prev: probe(2000), curr: probe(2000), idle: 1 }); + expect(out).toEqual({ stop: true, idle: 2 }); + }); + + test("stops when selector reaches minCount", () => { + const out = decideStop({ + ...base, + hasSelector: true, + minCount: 24, + prev: probe(1000), + curr: probe(1500, 30), + idle: 0, + }); + expect(out.stop).toBe(true); + }); + + test("keeps scrolling when selector under minCount", () => { + const out = decideStop({ + ...base, + hasSelector: true, + minCount: 24, + prev: probe(1000), + curr: probe(1500, 10), + idle: 0, + }); + expect(out.stop).toBe(false); + }); + + test("stops at maxScrolls cap even while still growing", () => { + const out = decideStop({ ...base, rounds: 20, prev: probe(1000), curr: probe(1500), idle: 0 }); + expect(out.stop).toBe(true); + }); + + test("first round (prev null) counts as growth", () => { + const out = decideStop({ ...base, prev: null, curr: probe(800), idle: 0 }); + expect(out).toEqual({ stop: false, idle: 0 }); + }); +}); diff --git a/tests/unit/block.test.ts b/tests/unit/block.test.ts new file mode 100644 index 0000000..4336250 --- /dev/null +++ b/tests/unit/block.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from "bun:test"; +import type { BrowserContext, Route } from "playwright"; +import { applyResourceBlocking } from "../../src/net/block.js"; + +type RouteHandler = (route: Route) => unknown; + +/** Minimal typed BrowserContext mock capturing route registrations. */ +function mockContext(): { context: BrowserContext; patterns: unknown[]; handlers: RouteHandler[] } { + const patterns: unknown[] = []; + const handlers: RouteHandler[] = []; + const context = { + route: async (pattern: unknown, handler: RouteHandler) => { + patterns.push(pattern); + handlers.push(handler); + }, + } as unknown as BrowserContext; + return { context, patterns, handlers }; +} + +/** Minimal typed Route mock recording abort/fallback calls. */ +function mockRoute(resourceType: string): { route: Route; calls: string[] } { + const calls: string[] = []; + const route = { + request: () => ({ resourceType: () => resourceType }), + abort: async () => { + calls.push("abort"); + }, + fallback: async () => { + calls.push("fallback"); + }, + } as unknown as Route; + return { route, calls }; +} + +describe("applyResourceBlocking", () => { + test("aborts blocked types, falls back otherwise", async () => { + const { context, patterns, handlers } = mockContext(); + await applyResourceBlocking(context, ["image", "font"]); + expect(patterns).toEqual(["**/*"]); + const handler = handlers[0] as RouteHandler; + + const img = mockRoute("image"); + await handler(img.route); + expect(img.calls).toEqual(["abort"]); + + const doc = mockRoute("document"); + await handler(doc.route); + expect(doc.calls).toEqual(["fallback"]); + }); + + test("type matching is case-insensitive", async () => { + const { context, handlers } = mockContext(); + await applyResourceBlocking(context, ["Image"]); + const img = mockRoute("image"); + await (handlers[0] as RouteHandler)(img.route); + expect(img.calls).toEqual(["abort"]); + }); + + test("unknown types are ignored; nothing valid installs no route", async () => { + const { context, handlers } = mockContext(); + await applyResourceBlocking(context, ["images", "gif", "bogus"]); + expect(handlers).toHaveLength(0); + }); + + test("unknown types mixed with valid ones do not block extra traffic", async () => { + const { context, handlers } = mockContext(); + await applyResourceBlocking(context, ["bogus", "media"]); + const media = mockRoute("media"); + const xhr = mockRoute("xhr"); + const handler = handlers[0] as RouteHandler; + await handler(media.route); + await handler(xhr.route); + expect(media.calls).toEqual(["abort"]); + expect(xhr.calls).toEqual(["fallback"]); + }); +}); diff --git a/tests/unit/caps.test.ts b/tests/unit/caps.test.ts new file mode 100644 index 0000000..a58e7c2 --- /dev/null +++ b/tests/unit/caps.test.ts @@ -0,0 +1,37 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { CAP_GROUPS, parseCaps } from "../../src/server/caps.js"; + +/** Silence the stderr report for unknown group names during these tests. */ +const errSpy = spyOn(console, "error").mockImplementation(() => {}); + +afterEach(() => errSpy.mockClear()); + +describe("parseCaps", () => { + test("undefined enables every group", () => { + expect(parseCaps(undefined)).toEqual(new Set(CAP_GROUPS)); + }); + + test("empty / blank string enables every group", () => { + expect(parseCaps("")).toEqual(new Set(CAP_GROUPS)); + expect(parseCaps(" , ,")).toEqual(new Set(CAP_GROUPS)); + }); + + test('"core,extract" enables exactly those two groups', () => { + expect(parseCaps("core,extract")).toEqual(new Set(["core", "extract"])); + }); + + test("tolerates case and surrounding whitespace", () => { + expect(parseCaps(" CORE , Extract ")).toEqual(new Set(["core", "extract"])); + }); + + test("ignores unknown names but keeps the known ones", () => { + expect(parseCaps("core,bogus")).toEqual(new Set(["core"])); + expect(errSpy).toHaveBeenCalledTimes(1); + expect(String(errSpy.mock.calls[0]?.[0])).toContain('unknown FUSE_CAPS group "bogus"'); + }); + + test("only unknown names falls back to every group", () => { + expect(parseCaps("bogus,nope")).toEqual(new Set(CAP_GROUPS)); + expect(errSpy).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/unit/dialogs.test.ts b/tests/unit/dialogs.test.ts new file mode 100644 index 0000000..cc23f3e --- /dev/null +++ b/tests/unit/dialogs.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from "bun:test"; +import { attachDialogs, recentDialogs, setDialogPolicy } from "../../src/session/dialogs.js"; +import type { SessionData } from "../../src/session/session.js"; + +const tick = () => new Promise((r) => setTimeout(r, 0)); + +/** Minimal Playwright Dialog stand-in tracking how it was resolved. */ +class FakeDialog { + acceptCalled = false; + acceptText: string | undefined; + dismissCalled = false; + constructor(private kind: string, private msg: string, private failResolve = false) {} + type = (): string => this.kind; + message = (): string => this.msg; + accept = async (text?: string): Promise<void> => { + if (this.failResolve) throw new Error("already handled"); + this.acceptCalled = true; + this.acceptText = text; + }; + dismiss = async (): Promise<void> => { + if (this.failResolve) throw new Error("already handled"); + this.dismissCalled = true; + }; +} + +/** Fake page capturing `dialog` handlers + session stub (state is WeakMap-keyed). */ +function fixture() { + const handlers: Array<(d: FakeDialog) => void> = []; + const page = { + on(event: string, handler: (d: FakeDialog) => void): void { + if (event === "dialog") handlers.push(handler); + }, + }; + const session = { id: "s", page } as unknown as SessionData; + return { session, handlers, emit: (d: FakeDialog) => handlers.forEach((h) => h(d)) }; +} + +describe("dialogs policy", () => { + test("default policy dismisses dialogs and records them", async () => { + const { session, emit } = fixture(); + attachDialogs(session); + const dialog = new FakeDialog("confirm", "sure?"); + emit(dialog); + await tick(); + expect(dialog.dismissCalled).toBe(true); + expect(dialog.acceptCalled).toBe(false); + expect(recentDialogs(session)).toEqual([ + { type: "confirm", message: "sure?", at: expect.any(Number), handled: "dismiss" }, + ]); + }); + + test("accept fills prompts with promptText, others without text", async () => { + const { session, emit } = fixture(); + attachDialogs(session); + setDialogPolicy(session, { action: "accept", promptText: "hello" }); + const prompt = new FakeDialog("prompt", "name?"); + const confirm = new FakeDialog("confirm", "ok?"); + emit(prompt); + emit(confirm); + await tick(); + expect(prompt.acceptCalled).toBe(true); + expect(prompt.acceptText).toBe("hello"); + expect(confirm.acceptCalled).toBe(true); + expect(confirm.acceptText).toBeUndefined(); + expect(recentDialogs(session).map((d) => d.handled)).toEqual(["accept", "accept"]); + }); + + test("a rejected accept is swallowed but still recorded", async () => { + const { session, emit } = fixture(); + attachDialogs(session); + setDialogPolicy(session, { action: "accept" }); + emit(new FakeDialog("alert", "late", true)); + await tick(); + expect(recentDialogs(session)).toHaveLength(1); + }); +}); + +describe("dialogs ring buffer and idempotence", () => { + test("keeps only the 20 most recent dialogs", async () => { + const { session, emit } = fixture(); + attachDialogs(session); + for (let i = 1; i <= 25; i += 1) emit(new FakeDialog("alert", `msg-${i}`)); + await tick(); + const recent = recentDialogs(session); + expect(recent).toHaveLength(20); + expect(recent[0]?.message).toBe("msg-6"); + expect(recent[19]?.message).toBe("msg-25"); + }); + + test("attachDialogs is idempotent per page", () => { + const { session, handlers } = fixture(); + attachDialogs(session); + attachDialogs(session); + expect(handlers).toHaveLength(1); + }); +}); diff --git a/tests/unit/heal-locator.test.ts b/tests/unit/heal-locator.test.ts new file mode 100644 index 0000000..82a8c54 --- /dev/null +++ b/tests/unit/heal-locator.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from "bun:test"; +import type { Locator, Page } from "playwright"; +import { healLocator } from "../../src/actions/heal-locator.js"; +import { REF_ATTRIBUTE } from "../../src/extraction/snapshot.js"; +import type { InteractiveElement } from "../../src/interfaces/extraction.js"; + +/** A locator stub: configurable count + visibility, records its selector. */ +function makeLocator(count: number, visible: boolean): Locator { + const self = { + first: () => self, + count: async () => count, + isVisible: async () => visible, + }; + return self as unknown as Locator; +} + +const MISS = makeLocator(0, false); + +/** Minimal element factory for snapshot-based healing. */ +function el(ref: string, text: string): InteractiveElement { + return { + index: 0, + ref, + tag: "button", + text, + role: null, + id: null, + name: null, + type: null, + href: null, + visible: true, + box: { x: 0, y: 0, width: 1, height: 1 }, + }; +} + +/** Build a typed Page mock with overridable locating methods. */ +function makePage(over: Partial<Record<keyof Page, unknown>>): Page { + const base = { + getByRole: () => MISS, + getByText: () => MISS, + frames: () => [{ isDetached: () => false, locator: () => makeLocator(1, true) }], + }; + return { ...base, ...over } as unknown as Page; +} + +const emptySnapshot = async (): Promise<InteractiveElement[]> => []; + +describe("healLocator", () => { + test("returns null for a blank target without touching the page", async () => { + const page = makePage({}); + expect(await healLocator(page, " ", emptySnapshot)).toBeNull(); + }); + + test("recovers via accessible role+name (button)", async () => { + const hit = makeLocator(1, true); + const page = makePage({ getByRole: (role: string) => (role === "button" ? hit : MISS) }); + expect(await healLocator(page, "Save", emptySnapshot)).toBe(hit); + }); + + test("falls through role to visible text", async () => { + const hit = makeLocator(1, true); + const page = makePage({ getByText: () => hit }); + expect(await healLocator(page, "Continue", emptySnapshot)).toBe(hit); + }); + + test("ignores a role match that is present but not visible", async () => { + const page = makePage({ getByRole: () => makeLocator(1, false) }); + expect(await healLocator(page, "Hidden", emptySnapshot)).toBeNull(); + }); + + test("re-snapshots and re-matches the original label via its ref", async () => { + const frameLoc = makeLocator(1, true); + const page = makePage({ + frames: () => [{ isDetached: () => false, locator: () => frameLoc }], + }); + const snapshot = async () => [el("0", "Other"), el("0", "Add to cart now")]; + const out = await healLocator(page, "add to cart", snapshot); + expect(out).toBe(frameLoc); + }); + + test("returns null when nothing recovers the target", async () => { + const page = makePage({}); + expect(await healLocator(page, "Nope", emptySnapshot)).toBeNull(); + }); + + test("REF_ATTRIBUTE is exported for ref resolution", () => { + expect(typeof REF_ATTRIBUTE).toBe("string"); + }); +}); diff --git a/tests/unit/helpers/fake-tabs.ts b/tests/unit/helpers/fake-tabs.ts new file mode 100644 index 0000000..7fa1468 --- /dev/null +++ b/tests/unit/helpers/fake-tabs.ts @@ -0,0 +1,75 @@ +/** + * Browser-free fakes for tabs unit tests: a context whose pages are plain + * objects exposing only the surface session/tabs touches. + * @module tests/unit/helpers/fake-tabs + */ +import type { Page } from "playwright"; +import { resolveConfig } from "../../../src/agent/config.js"; +import type { SessionData } from "../../../src/session/session.js"; + +/** Listener names registered per fake page (to assert no handler stacking). */ +export const handlersOf = new WeakMap<Page, string[]>(); + +/** A fake context, its page factory, live page list and goto failure switch. */ +export interface FakeCtx { + ctx: { pages: () => Page[]; newPage: () => Promise<Page> }; + addPage: (url: string, failGoto?: boolean) => Page; + pages: Page[]; + /** Make pages spawned by `newPage` reject their `goto` with "boom". */ + setNewPageGotoFails: (fail: boolean) => void; +} + +/** Build a fake BrowserContext over plain in-memory pages. */ +export function makeCtx(): FakeCtx { + const pages: Page[] = []; + const state = { failGoto: false }; + function addPage(url: string, failGoto = false): Page { + let current = url; + const handlers: string[] = []; + const page = { + url: () => current, + title: async () => `t:${current}`, + on: (ev: string) => handlers.push(ev), + bringToFront: async () => {}, + goto: async (u: string) => { + if (failGoto) throw new Error("boom"); + current = u; + return { status: () => 200, headers: () => ({}) }; + }, + close: async () => { + pages.splice(pages.indexOf(page as unknown as Page), 1); + }, + mainFrame: () => null, + } as unknown as Page; + handlersOf.set(page, handlers); + pages.push(page); + return page; + } + const ctx = { pages: () => pages, newPage: async () => addPage("about:blank", state.failGoto) }; + return { ctx, addPage, pages, setNewPageGotoFails: (fail) => (state.failGoto = fail) }; +} + +/** Fake tabs session plus its context handles (mirrors openSession bookkeeping). */ +export interface FakeTabsSession extends Omit<FakeCtx, "ctx"> { + session: SessionData; +} + +/** Build a session over a fake context with one tab on https://a.example/. */ +export function makeSession(): FakeTabsSession { + const { ctx, addPage, pages, setNewPageGotoFails } = makeCtx(); + const page = addPage("https://a.example/"); + const session = { + id: "s1", + context: ctx, + browser: null, + page, + config: resolveConfig({}), + logs: { network: [], console: [] }, + connected: true, + health: "ok", + lastUrl: "https://a.example/", + createdAt: Date.now(), + expiresAt: Date.now() + 60_000, + } as unknown as SessionData; + return { session, addPage, pages, setNewPageGotoFails }; +} diff --git a/tests/unit/logs-tools.test.ts b/tests/unit/logs-tools.test.ts new file mode 100644 index 0000000..b703c67 --- /dev/null +++ b/tests/unit/logs-tools.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, test } from "bun:test"; +import { + filterConsole, + filterNetwork, + mergeNetwork, + type NetworkEntry, +} from "../../src/server/tools/logs-filter.js"; + +const consoleBuf = [ + { type: "log", text: "boot" }, + { type: "error", text: "TypeError: x is undefined" }, + { type: "warning", text: "deprecated API" }, + { type: "error", text: "Failed to fetch" }, + { type: "info", text: "ready" }, +]; + +describe("filterConsole", () => { + test("returns everything (last 50 by default) without a level", () => { + expect(filterConsole(consoleBuf)).toEqual(consoleBuf); + }); + + test("filters by exact level", () => { + const errors = filterConsole(consoleBuf, "error"); + expect(errors).toEqual([ + { type: "error", text: "TypeError: x is undefined" }, + { type: "error", text: "Failed to fetch" }, + ]); + }); + + test("keeps only the last `limit` entries", () => { + const out = filterConsole(consoleBuf, undefined, 2); + expect(out).toEqual([ + { type: "error", text: "Failed to fetch" }, + { type: "info", text: "ready" }, + ]); + }); + + test("defaults to the last 50 on a large buffer", () => { + const big = Array.from({ length: 60 }, (_, i) => ({ type: "log", text: `m${i}` })); + const out = filterConsole(big); + expect(out).toHaveLength(50); + expect(out[0]?.text).toBe("m10"); + expect(out[49]?.text).toBe("m59"); + }); +}); + +const events: Array<Record<string, unknown>> = [ + { type: "request", method: "GET", url: "https://a.com/", resourceType: "document" }, + { type: "request", method: "POST", url: "https://a.com/api", resourceType: "xhr" }, + { type: "response", status: 200, url: "https://a.com/" }, + { type: "response", status: 500, url: "https://a.com/api" }, + { type: "request", method: "GET", url: "https://cdn.b.com/app.js", resourceType: "script" }, +]; + +describe("mergeNetwork", () => { + test("joins request and response by url, keeping method/status/resourceType", () => { + const rows = mergeNetwork(events); + expect(rows).toEqual([ + { url: "https://a.com/", method: "GET", resourceType: "document", status: 200 }, + { url: "https://a.com/api", method: "POST", resourceType: "xhr", status: 500 }, + { url: "https://cdn.b.com/app.js", method: "GET", resourceType: "script" }, + ]); + }); + + test("handles legacy events without resourceType", () => { + const rows = mergeNetwork([{ type: "request", method: "GET", url: "https://x.com/" }]); + expect(rows).toEqual([{ url: "https://x.com/", method: "GET" }]); + }); +}); + +describe("filterNetwork", () => { + const rows: NetworkEntry[] = mergeNetwork(events); + + test("filters by exact status", () => { + const out = filterNetwork(rows, { status: 500 }); + expect(out).toHaveLength(1); + expect(out[0]?.url).toBe("https://a.com/api"); + }); + + test("filters by url substring", () => { + const out = filterNetwork(rows, { urlContains: "cdn.b.com" }); + expect(out).toEqual([{ url: "https://cdn.b.com/app.js", method: "GET", resourceType: "script" }]); + }); + + test("combines filters and keeps the last `limit`", () => { + expect(filterNetwork(rows, { urlContains: "a.com", limit: 1 })).toEqual([ + { url: "https://a.com/api", method: "POST", resourceType: "xhr", status: 500 }, + ]); + expect(filterNetwork(rows, { status: 404 })).toEqual([]); + }); +}); diff --git a/tests/unit/map-options.test.ts b/tests/unit/map-options.test.ts index 091fa35..953d17f 100644 --- a/tests/unit/map-options.test.ts +++ b/tests/unit/map-options.test.ts @@ -36,3 +36,19 @@ describe("toAgentOptions — CDP remote", () => { expect(toAgentOptions({}).respectRobots).toBeUndefined(); }); }); + +describe("toAgentOptions — profile & blockResources", () => { + test("profile maps to its storage-state path when storageStatePath is absent", () => { + const a = toAgentOptions({ profile: "github" }); + expect(a.profile).toBe("github"); + expect(a.storageStatePath).toMatch(/profiles[/\\]github\.json$/); + }); + test("explicit storageStatePath wins over profile", () => { + const a = toAgentOptions({ profile: "github", storageStatePath: "/tmp/state.json" }); + expect(a.storageStatePath).toBe("/tmp/state.json"); + }); + test("maps blockResources (default undefined)", () => { + expect(toAgentOptions({ blockResources: ["image", "font"] }).blockResources).toEqual(["image", "font"]); + expect(toAgentOptions({}).blockResources).toBeUndefined(); + }); +}); diff --git a/tests/unit/prices-context.test.ts b/tests/unit/prices-context.test.ts new file mode 100644 index 0000000..6c50732 --- /dev/null +++ b/tests/unit/prices-context.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "bun:test"; +import { extractPrices } from "../../src/extraction/prices.js"; + +describe("price context label", () => { + test("picks the significant line just before the price", () => { + const prices = extractPrices("2 nights, 2 adults\nCHF 240"); + const chf = prices.find((p) => p.currency === "CHF" && p.amount === 240); + expect(chf?.context).toBe("2 nights, 2 adults"); + }); + + test("falls back to the line after when there is none before", () => { + const prices = extractPrices("€ 149\nDeluxe King Room"); + const eur = prices.find((p) => p.currency === "EUR" && p.amount === 149); + expect(eur?.context).toBe("Deluxe King Room"); + }); + + test("prefers the preceding line over the following one", () => { + const prices = extractPrices("Tickets from\n$ 30\nincludes fees"); + const usd = prices.find((p) => p.currency === "USD" && p.amount === 30); + expect(usd?.context).toBe("Tickets from"); + }); + + test("skips a neighbouring price line when choosing context", () => { + const prices = extractPrices("Suite\nCHF 100\nCHF 80"); + const live = prices.find((p) => p.currency === "CHF" && p.amount === 80); + // The line before 80 is the 100 price line, so context comes from "Suite". + expect(live?.context).toBe("Suite"); + }); + + test("omits context when no significant neighbour exists", () => { + const prices = extractPrices("CHF 50"); + const chf = prices.find((p) => p.currency === "CHF" && p.amount === 50); + expect(chf?.context).toBeUndefined(); + }); + + test("does not break existing currency/amount/line assertions", () => { + const prices = extractPrices("Hotel Name\nCHF 106"); + const chf = prices.find((p) => p.currency === "CHF" && p.amount === 106); + expect(chf?.line).toBe("CHF 106"); + expect(chf?.context).toBe("Hotel Name"); + }); +}); diff --git a/tests/unit/prices.test.ts b/tests/unit/prices.test.ts index 8e9d552..b878108 100644 --- a/tests/unit/prices.test.ts +++ b/tests/unit/prices.test.ts @@ -36,6 +36,35 @@ describe("extractPrices", () => { const prices = extractPrices("Price R 350"); expect(prices.some((p) => p.currency === "ZAR" && p.amount === 350)).toBe(true); }); + + test("stitches a currency split across DOM lines (digitec prefix)", () => { + const prices = extractPrices("CHF\n6.90"); + expect(prices.some((p) => p.currency === "CHF" && p.amount === 6.9)).toBe(true); + }); + + test("stitches a currency on the line after the amount (suffix)", () => { + const prices = extractPrices("6.90\nCHF"); + expect(prices.some((p) => p.currency === "CHF" && p.amount === 6.9)).toBe(true); + }); + + test("matches currency as a trailing suffix on the same line", () => { + const euro = extractPrices("10 €"); + expect(euro.some((p) => p.currency === "EUR" && p.amount === 10)).toBe(true); + const krona = extractPrices("350 kr"); + expect(krona.some((p) => p.amount === 350 && ["NOK", "SEK", "DKK"].includes(p.currency))).toBe(true); + }); + + test("handles non-breaking space between currency and amount", () => { + const prices = extractPrices("CHF\u00A06.90"); + expect(prices.some((p) => p.currency === "CHF" && p.amount === 6.9)).toBe(true); + }); + + test("keeps strikethrough and live price as two distinct amounts", () => { + const prices = extractPrices("CHF 100\nCHF 80"); + const chf = new Set(prices.filter((p) => p.currency === "CHF").map((p) => p.amount)); + expect(chf.has(100)).toBe(true); + expect(chf.has(80)).toBe(true); + }); }); describe("normaliseAmount", () => { @@ -45,4 +74,9 @@ describe("normaliseAmount", () => { expect(normaliseAmount("1,234.56")).toBe(1234.56); expect(normaliseAmount("129")).toBe(129); }); + + test("locale-aware decimal separator (CH and EU formats)", () => { + expect(normaliseAmount("1'234.56")).toBe(1234.56); + expect(normaliseAmount("1.234,56")).toBe(1234.56); + }); }); diff --git a/tests/unit/probe-queue.test.ts b/tests/unit/probe-queue.test.ts index f82b67c..3898965 100644 --- a/tests/unit/probe-queue.test.ts +++ b/tests/unit/probe-queue.test.ts @@ -52,7 +52,7 @@ describe("probe queue", () => { const runs = d.map((def) => withQueue(CFG, async () => def.promise)); await Promise.resolve(); await expect(withQueue(CFG, async () => "overflow")).rejects.toBeInstanceOf(QueueFullError); - d.forEach((x) => x.resolve()); + for (const x of d) x.resolve(); await Promise.all(runs); }); diff --git a/tests/unit/products.test.ts b/tests/unit/products.test.ts new file mode 100644 index 0000000..0128d03 --- /dev/null +++ b/tests/unit/products.test.ts @@ -0,0 +1,80 @@ +/** + * Unit tests for the pure product-card heuristic, run against a linkedom + * Document (no real browser). Covers grouping, price/currency parsing, title + * selection, url capture, containerSelector mode and limit. + * @module tests/unit/products + */ +import { describe, expect, test } from "bun:test"; +import { parseHTML } from "linkedom"; +import { collectProducts } from "../../src/extraction/products-dom.js"; +import type { DomDocument } from "../../src/interfaces/dom.js"; + +/** Build a linkedom Document (structurally a DomDocument) from a body fragment. */ +function doc(body: string): DomDocument { + return parseHTML(`<html><body>${body}</body></html>`).document as unknown as DomDocument; +} + +/** A grid of `n` cards, each an <article class="card"> with link + price. */ +function grid(items: Array<{ title: string; price: string; href?: string }>): string { + return `<main>${items + .map( + (it) => + `<article class="card"><a href="${it.href ?? "#"}"><h3>${it.title}</h3></a><span class="price">${it.price}</span></article>`, + ) + .join("")}</main>`; +} + +describe("collectProducts — repeated-card auto-detection", () => { + test("detects ≥3 repeated cards and links each price to its title", () => { + const html = grid([ + { title: "Laptop A", price: "CHF 999.-", href: "/a" }, + { title: "Laptop B", price: "CHF 1'299.90", href: "/b" }, + { title: "Laptop C", price: "CHF 750", href: "/c" }, + ]); + const products = collectProducts(doc(html)); + expect(products).toHaveLength(3); + const a = products.find((p) => p.title === "Laptop A"); + expect(a).toEqual({ title: "Laptop A", price: 999, currency: "CHF", url: "/a" }); + const b = products.find((p) => p.title === "Laptop B"); + expect(b?.price).toBe(1299.9); + }); + + test("parses EUR, USD, GBP suffix and prefix forms", () => { + const products = collectProducts( + doc(grid([ + { title: "Euro Item", price: "1.234,50 €" }, + { title: "Dollar Item", price: "$19.99" }, + { title: "Pound Item", price: "£5" }, + ])), + ); + const by = (t: string) => products.find((p) => p.title === t); + expect(by("Euro Item")).toMatchObject({ currency: "EUR", price: 1234.5 }); + expect(by("Dollar Item")).toMatchObject({ currency: "USD", price: 19.99 }); + expect(by("Pound Item")).toMatchObject({ currency: "GBP", price: 5 }); + }); + + test("ignores a lone priced node that does not repeat ≥3 times", () => { + const html = `<main><div class="solo"><span>EUR 42</span></div></main>`; + expect(collectProducts(doc(html))).toHaveLength(0); + }); + + test("drops repeated cards that carry no price", () => { + const html = `<main>${["X", "Y", "Z"] + .map((t) => `<article class="card"><h3>${t}</h3></article>`) + .join("")}</main>`; + expect(collectProducts(doc(html))).toHaveLength(0); + }); +}); + +describe("collectProducts — options", () => { + test("containerSelector pins the card element directly", () => { + const html = `<ul><li class="row"><a href="/p1">Phone</a> USD 599</li></ul>`; + const products = collectProducts(doc(html), { containerSelector: "li.row" }); + expect(products).toEqual([{ title: "Phone", price: 599, currency: "USD", url: "/p1" }]); + }); + + test("limit caps the number of returned cards", () => { + const items = Array.from({ length: 5 }, (_, i) => ({ title: `P${i}`, price: "CHF 10" })); + expect(collectProducts(doc(grid(items)), { limit: 2 })).toHaveLength(2); + }); +}); diff --git a/tests/unit/profiles.test.ts b/tests/unit/profiles.test.ts new file mode 100644 index 0000000..44cb0cf --- /dev/null +++ b/tests/unit/profiles.test.ts @@ -0,0 +1,35 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { profileStoragePath } from "../../src/identity/profiles.js"; + +const KEY = "FUSE_BROWSER_HOME"; +const saved = process.env[KEY]; +afterEach(() => { + if (saved === undefined) delete process.env[KEY]; + else process.env[KEY] = saved; +}); + +describe("profileStoragePath — valid names", () => { + test("resolves under FUSE_BROWSER_HOME/profiles/<name>.json", () => { + process.env[KEY] = "/tmp/fuse-home"; + expect(profileStoragePath("github")).toBe(join("/tmp/fuse-home", "profiles", "github.json")); + expect(profileStoragePath("My_Org-2")).toBe(join("/tmp/fuse-home", "profiles", "My_Org-2.json")); + }); + test("defaults to ~/.fuse-browser when FUSE_BROWSER_HOME is unset", () => { + delete process.env[KEY]; + expect(profileStoragePath("a")).toBe(join(homedir(), ".fuse-browser", "profiles", "a.json")); + }); + test("accepts the 41-char maximum", () => { + expect(() => profileStoragePath(`a${"b".repeat(40)}`)).not.toThrow(); + }); +}); + +describe("profileStoragePath — invalid names", () => { + const bad = ["", "-lead", "_lead", "has space", "a/b", "a\\b", "../etc", "dot.name", "x".repeat(42)]; + for (const name of bad) { + test(`rejects ${JSON.stringify(name)}`, () => { + expect(() => profileStoragePath(name)).toThrow(/Invalid profile name/); + }); + } +}); diff --git a/tests/unit/progress.test.ts b/tests/unit/progress.test.ts new file mode 100644 index 0000000..39abfc7 --- /dev/null +++ b/tests/unit/progress.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { type ProgressFn, progressReporter, type ToolExtra } from "../../src/server/progress.js"; + +/** Build a fake `extra` capturing sent notifications (optionally rejecting). */ +function fakeExtra(progressToken?: string | number, reject = false) { + const sent: unknown[] = []; + const extra = { + _meta: progressToken === undefined ? undefined : { progressToken }, + sendNotification: (n: unknown) => { + sent.push(n); + return reject ? Promise.reject(new Error("transport down")) : Promise.resolve(); + }, + } as unknown as ToolExtra; + return { extra, sent }; +} + +describe("progressReporter", () => { + test("is a silent no-op when the client sent no progressToken", () => { + const { extra, sent } = fakeExtra(undefined); + const report: ProgressFn = progressReporter(extra); + expect(() => report(4, 12, "page 4")).not.toThrow(); + expect(sent).toHaveLength(0); + }); + + test("no-op too when _meta exists but has no progressToken", () => { + const { extra, sent } = fakeExtra(undefined); + (extra as { _meta?: object })._meta = {}; + progressReporter(extra)(1, 2); + expect(sent).toHaveLength(0); + }); + + test("emits a notifications/progress per call with token, progress, total, message", () => { + const { extra, sent } = fakeExtra("tok-1"); + const report = progressReporter(extra); + report(4, 12, "https://a.example"); + report(5, 12); + expect(sent).toHaveLength(2); + expect(sent[0]).toEqual({ + method: "notifications/progress", + params: { progressToken: "tok-1", progress: 4, total: 12, message: "https://a.example" }, + }); + expect(sent[1]).toEqual({ + method: "notifications/progress", + params: { progressToken: "tok-1", progress: 5, total: 12 }, + }); + }); + + test("supports numeric progress tokens (JSON-RPC ids)", () => { + const { extra, sent } = fakeExtra(7); + progressReporter(extra)(1, 3, "q1"); + expect(sent[0]).toMatchObject({ params: { progressToken: 7, progress: 1, total: 3 } }); + }); + + test("never throws nor leaves an unhandled rejection when sendNotification rejects", async () => { + const { extra, sent } = fakeExtra("tok-2", true); + const report = progressReporter(extra); + expect(() => report(1, 2, "boom-item")).not.toThrow(); + await Bun.sleep(0); // flush microtasks: the rejection must be swallowed + expect(sent).toHaveLength(1); + }); +}); diff --git a/tests/unit/proxy-pool.test.ts b/tests/unit/proxy-pool.test.ts index 2193824..4b1dd75 100644 --- a/tests/unit/proxy-pool.test.ts +++ b/tests/unit/proxy-pool.test.ts @@ -20,7 +20,7 @@ describe("ProxyPool", () => { }); test("returns null when all proxies are cooling", () => { - let t = 0; + const t = 0; const p = new ProxyPool(["a"], 1000, () => t); expect(p.acquire()).toBe("a"); p.reportBlocked("a"); diff --git a/tests/unit/resource-screenshot.test.ts b/tests/unit/resource-screenshot.test.ts new file mode 100644 index 0000000..6158f95 --- /dev/null +++ b/tests/unit/resource-screenshot.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { SessionNotFoundError } from "../../src/lib/errors.js"; +import type { SessionManager } from "../../src/session/manager.js"; +import { + captureSessionScreenshot, + isSessionMissing, +} from "../../src/server/resource-screenshot.js"; + +/** Minimal SessionManager stub: `get` either returns a fake page or throws. */ +function stubSessions(shot: Buffer | Error): SessionManager { + const page = { screenshot: async () => shot }; + return { + get: (id: string) => { + if (shot instanceof Error) throw shot; + return { id, page }; + }, + } as unknown as SessionManager; +} + +describe("captureSessionScreenshot", () => { + test("returns a base64 JPEG blob echoing the URI", async () => { + const sessions = stubSessions(Buffer.from("JPEGBYTES")); + const out = await captureSessionScreenshot(sessions, "abc", "screenshot://abc/last"); + expect(out.contents).toEqual([ + { + uri: "screenshot://abc/last", + mimeType: "image/jpeg", + blob: Buffer.from("JPEGBYTES").toString("base64"), + }, + ]); + }); + + test("propagates SessionNotFoundError when the session is gone", async () => { + const sessions = stubSessions(new SessionNotFoundError("abc")); + const promise = captureSessionScreenshot(sessions, "abc", "screenshot://abc/last"); + await expect(promise).rejects.toBeInstanceOf(SessionNotFoundError); + }); +}); + +describe("isSessionMissing", () => { + test("true only for SessionNotFoundError", () => { + expect(isSessionMissing(new SessionNotFoundError("x"))).toBe(true); + expect(isSessionMissing(new Error("other"))).toBe(false); + expect(isSessionMissing("nope")).toBe(false); + }); +}); diff --git a/tests/unit/result.test.ts b/tests/unit/result.test.ts index 6f5d2de..fd1a815 100644 --- a/tests/unit/result.test.ts +++ b/tests/unit/result.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { imageJsonResult, jsonResult } from "../../src/server/result.js"; +import { errorResult, imageJsonResult, jsonResult } from "../../src/server/result.js"; describe("imageJsonResult", () => { test("returns an image block + a JSON text block + structuredContent", () => { @@ -24,3 +24,19 @@ describe("imageJsonResult", () => { expect(r.content.every((c) => c.type === "text")).toBe(true); }); }); + +describe("errorResult", () => { + test("without code: text + isError, no structuredContent (back-compat)", () => { + const r = errorResult("boom"); + expect(r.isError).toBe(true); + expect(r.content).toEqual([{ type: "text", text: "boom" }]); + expect(r.structuredContent).toBeUndefined(); + }); + + test("with code: adds structuredContent { code, message }", () => { + const r = errorResult("session gone", "session_not_found"); + expect(r.isError).toBe(true); + expect(r.content).toEqual([{ type: "text", text: "session gone" }]); + expect(r.structuredContent).toEqual({ code: "session_not_found", message: "session gone" }); + }); +}); diff --git a/tests/unit/safe-png.test.ts b/tests/unit/safe-png.test.ts new file mode 100644 index 0000000..97ea9cd --- /dev/null +++ b/tests/unit/safe-png.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, test } from "bun:test"; +import { isAbsolute, resolve } from "node:path"; +import { assertPngPath } from "../../src/lib/safe-png.js"; + +describe("assertPngPath", () => { + test("resolves a relative .png path to an absolute one", () => { + const out = assertPngPath("shots/page.png", "a"); + expect(out).toBe(resolve("shots/page.png")); + expect(isAbsolute(out)).toBe(true); + }); + + test("keeps an absolute .png path (normalized)", () => { + expect(assertPngPath("/tmp/x//shots/../base.png", "baseline")).toBe(resolve("/tmp/x/base.png")); + }); + + test("accepts uppercase .PNG extension", () => { + expect(assertPngPath("/tmp/SHOT.PNG", "b")).toBe(resolve("/tmp/SHOT.PNG")); + }); + + test("rejects an empty string", () => { + expect(() => assertPngPath("", "a")).toThrow(/a: path must be a non-empty string/); + }); + + test("rejects a non-.png extension", () => { + expect(() => assertPngPath("/tmp/file.txt", "b")).toThrow(/b: path must end with \.png/); + }); + + test("rejects a path containing a NUL character", () => { + expect(() => assertPngPath("/tmp/evil\0.png", "baseline")).toThrow(/baseline: path must not contain NUL/); + }); +}); diff --git a/tests/unit/session-manager.test.ts b/tests/unit/session-manager.test.ts index 42ef121..d93fefa 100644 --- a/tests/unit/session-manager.test.ts +++ b/tests/unit/session-manager.test.ts @@ -2,6 +2,34 @@ import { describe, expect, test } from "bun:test"; import { resolveConfig } from "../../src/agent/config.js"; import { SessionLimitError } from "../../src/lib/errors.js"; import { SessionManager } from "../../src/session/manager.js"; +import type { SessionData } from "../../src/session/session.js"; +import type { TtlGuard } from "../../src/session/ttl-guard.js"; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** Connected fake: closeSession() only tries browser?.close() — no Chromium needed. */ +function fakeSession(id: string): SessionData { + return { + id, + context: null, + browser: null, + page: null, + config: resolveConfig({}), + logs: null, + connected: true, + health: "ok", + lastUrl: "", + createdAt: Date.now(), + expiresAt: Date.now(), + } as unknown as SessionData; +} + +/** Inject a session without launching a browser (mirrors open() bookkeeping). */ +function inject(mgr: SessionManager, session: SessionData): void { + const internals = mgr as unknown as { sessions: Map<string, SessionData>; guard: TtlGuard }; + internals.sessions.set(session.id, session); + internals.guard.schedule(session.id); +} // We only exercise the synchronous cap guard, which throws BEFORE any browser // launch — so no real Chromium is needed here. @@ -11,3 +39,32 @@ describe("SessionManager cap", () => { await expect(mgr.open(resolveConfig({}))).rejects.toBeInstanceOf(SessionLimitError); }); }); + +describe("SessionManager busy TTL guard", () => { + test("busy session survives TTL expiry, then closes once idle", async () => { + const mgr = new SessionManager({ ttlMs: 50 }); + inject(mgr, fakeSession("busy")); + mgr.markBusy("busy"); + await sleep(160); // > 3x TTL: timer fired but must reschedule, not close + expect(mgr.list().map((s) => s.id)).toContain("busy"); + mgr.markIdle("busy"); // refreshes TTL; next expiry may close + await sleep(160); + expect(mgr.list()).toHaveLength(0); + }); + + test("explicit close() works even while busy", async () => { + const mgr = new SessionManager({ ttlMs: 60_000 }); + inject(mgr, fakeSession("held")); + mgr.markBusy("held"); + await expect(mgr.close("held")).resolves.toBe(true); + expect(mgr.list()).toHaveLength(0); + }); + + test("markBusy/markIdle on an unknown id is a no-op", () => { + const mgr = new SessionManager({ ttlMs: 50 }); + expect(() => { + mgr.markBusy("ghost"); + mgr.markIdle("ghost"); + }).not.toThrow(); + }); +}); diff --git a/tests/unit/tabs.test.ts b/tests/unit/tabs.test.ts new file mode 100644 index 0000000..c1936f6 --- /dev/null +++ b/tests/unit/tabs.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from "bun:test"; +import type { Page } from "playwright"; +import { closeTab, listTabs, openTab, selectTab } from "../../src/session/tabs.js"; +import { handlersOf, makeSession } from "./helpers/fake-tabs.js"; + +describe("listTabs", () => { + test("reports index, url, async title and the active marker", async () => { + const { session, addPage } = makeSession(); + addPage("https://b.example/"); + const tabs = await listTabs(session); + expect(tabs).toEqual([ + { index: 0, url: "https://a.example/", title: "t:https://a.example/", active: true }, + { index: 1, url: "https://b.example/", title: "t:https://b.example/", active: false }, + ]); + }); +}); + +describe("selectTab", () => { + test("re-points page/lastUrl/logs and wires the popup once", async () => { + const { session, addPage } = makeSession(); + const popup = addPage("https://oauth.example/"); + const initialLogs = session.logs; + await selectTab(session, 1); + expect(session.page).toBe(popup); + expect(session.lastUrl).toBe("https://oauth.example/"); + expect(session.logs).not.toBe(initialLogs); + const popupLogs = session.logs; + const wired = handlersOf.get(popup) ?? []; + expect(wired).toContain("request"); // network listeners + expect(wired).toContain("crash"); // pageOnly health listeners + const count = wired.length; + await selectTab(session, 0); // back to the original tab… + expect(session.logs).toBe(initialLogs); // …restores its own log + await selectTab(session, 1); // re-select: no handler stacking + expect(session.logs).toBe(popupLogs); + expect((handlersOf.get(popup) ?? []).length).toBe(count); + }); + + test("throws RangeError on an out-of-range index", async () => { + const { session } = makeSession(); + await expect(selectTab(session, 3)).rejects.toThrow(/invalid_tab_index: 3/); + }); +}); + +describe("openTab", () => { + test("opens, navigates and selects the new tab", async () => { + const { session, pages } = makeSession(); + await openTab(session, "https://c.example/"); + expect(pages).toHaveLength(2); + expect(session.page).toBe(pages[1] as Page); + expect(session.page.url()).toBe("https://c.example/"); + }); + + test("closes the orphan page when navigation fails", async () => { + const { session, pages, setNewPageGotoFails } = makeSession(); + setNewPageGotoFails(true); + const first = session.page; + await expect(openTab(session, "https://down.example/")).rejects.toThrow("boom"); + expect(pages).toHaveLength(1); // orphan closed + expect(session.page).toBe(first); // active tab untouched + }); +}); + +describe("closeTab", () => { + test("refuses to close the last tab", async () => { + const { session } = makeSession(); + await expect(closeTab(session, 0)).rejects.toThrow(/cannot_close_last_tab/); + }); + + test("closing the active tab falls back to the first remaining one", async () => { + const { session, addPage, pages } = makeSession(); + addPage("https://b.example/"); + await selectTab(session, 1); + await closeTab(session, 1); + expect(pages).toHaveLength(1); + expect(session.page.url()).toBe("https://a.example/"); + expect(session.health).toBe("ok"); + }); + + test("closing an inactive tab keeps the active one", async () => { + const { session, addPage, pages } = makeSession(); + addPage("https://b.example/"); + await closeTab(session, 1); + expect(pages).toHaveLength(1); + expect(session.page.url()).toBe("https://a.example/"); + }); +});