diff --git a/docs/src/api/class-browsercontext.md b/docs/src/api/class-browsercontext.md index 03589a5f4c8f8..e4f2379cd9621 100644 --- a/docs/src/api/class-browsercontext.md +++ b/docs/src/api/class-browsercontext.md @@ -1307,6 +1307,12 @@ When set to `minimal`, only record information necessary for routing from HAR. T Optional setting to control resource content management. If `attach` is specified, resources are persisted as separate files or entries in the ZIP archive. If `embed` is specified, content is stored inline the HAR file. +### option: BrowserContext.routeFromHAR.interceptAPIRequests +* since: v1.62 +- `interceptAPIRequests` <[boolean]> + +If set to `true`, requests made via [APIRequestContext] (such as [`property: BrowserContext.request`] or [`property: Page.request`]) are also served from the HAR file. By default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for backward compatibility. + ## async method: BrowserContext.routeWebSocket * since: v1.48 diff --git a/packages/isomorphic/protocolMetainfo.ts b/packages/isomorphic/protocolMetainfo.ts index 37fe231666294..cf93e3f62f25d 100644 --- a/packages/isomorphic/protocolMetainfo.ts +++ b/packages/isomorphic/protocolMetainfo.ts @@ -89,6 +89,8 @@ export const methodMetainfo = new Map([ ['BrowserContext.setGeolocation', { title: 'Set geolocation', group: 'configuration', }], ['BrowserContext.setHTTPCredentials', { title: 'Set HTTP credentials', group: 'configuration', }], ['BrowserContext.setNetworkInterceptionPatterns', { title: 'Route requests', group: 'route', }], + ['BrowserContext.routeAPIRequestsFromHar', { internal: true, }], + ['BrowserContext.unrouteAPIRequestsFromHar', { internal: true, }], ['BrowserContext.setWebSocketInterceptionPatterns', { title: 'Route WebSockets', group: 'route', }], ['BrowserContext.setOffline', { title: 'Set offline mode', }], ['BrowserContext.storageState', { title: 'Get storage state', group: 'configuration', }], diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index b2150842591d0..15f2c2da4e153 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -9608,6 +9608,15 @@ export interface BrowserContext { * @param options */ routeFromHAR(har: string, options?: { + /** + * If set to `true`, requests made via [APIRequestContext](https://playwright.dev/docs/api/class-apirequestcontext) + * (such as [browserContext.request](https://playwright.dev/docs/api/class-browsercontext#browser-context-request) or + * [page.request](https://playwright.dev/docs/api/class-page#page-request)) are also served from the HAR file. By + * default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for + * backward compatibility. + */ + interceptAPIRequests?: boolean; + /** * - If set to 'abort' any request not found in the HAR file will be aborted. * - If set to 'fallback' falls through to the next route handler in the handler chain. diff --git a/packages/playwright-core/src/client/browserContext.ts b/packages/playwright-core/src/client/browserContext.ts index 71f744aa642a9..90e61cc3331bf 100644 --- a/packages/playwright-core/src/client/browserContext.ts +++ b/packages/playwright-core/src/client/browserContext.ts @@ -387,7 +387,7 @@ export class BrowserContext extends ChannelOwner await this._updateWebSocketInterceptionPatterns({ title: 'Route WebSockets' }); } - async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full' } = {}): Promise { + async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback', update?: boolean, updateContent?: 'attach' | 'embed', updateMode?: 'minimal' | 'full', interceptAPIRequests?: boolean } = {}): Promise { const localUtils = this._connection.localUtils(); if (!localUtils) throw new Error('Route from har is not supported in thin clients'); @@ -398,6 +398,8 @@ export class BrowserContext extends ChannelOwner const harRouter = await HarRouter.create(localUtils, har, options.notFound || 'abort', { urlMatch: options.url }); this._harRouters.push(harRouter); await harRouter.addContextRoute(this); + if (options.interceptAPIRequests) + await harRouter.addAPIRequestRoute(this); } private _disposeHarRouters() { diff --git a/packages/playwright-core/src/client/channels.d.ts b/packages/playwright-core/src/client/channels.d.ts index c66c5f2b5f249..a13f05f405c67 100644 --- a/packages/playwright-core/src/client/channels.d.ts +++ b/packages/playwright-core/src/client/channels.d.ts @@ -1344,6 +1344,8 @@ export interface BrowserContextChannel extends BrowserContextEventTarget, Channe setGeolocation(params: BrowserContextSetGeolocationParams): Promise; setHTTPCredentials(params: BrowserContextSetHTTPCredentialsParams): Promise; setNetworkInterceptionPatterns(params: BrowserContextSetNetworkInterceptionPatternsParams): Promise; + routeAPIRequestsFromHar(params: BrowserContextRouteAPIRequestsFromHarParams): Promise; + unrouteAPIRequestsFromHar(params: BrowserContextUnrouteAPIRequestsFromHarParams): Promise; setWebSocketInterceptionPatterns(params: BrowserContextSetWebSocketInterceptionPatternsParams): Promise; setOffline(params: BrowserContextSetOfflineParams): Promise; storageState(params: BrowserContextStorageStateParams): Promise; @@ -1577,6 +1579,28 @@ export type BrowserContextSetNetworkInterceptionPatternsOptions = { }; export type BrowserContextSetNetworkInterceptionPatternsResult = void; +export type BrowserContextRouteAPIRequestsFromHarParams = { + harId: string, + urlGlob?: string, + urlRegexSource?: string, + urlRegexFlags?: string, + notFound: 'abort' | 'fallback', +}; +export type BrowserContextRouteAPIRequestsFromHarOptions = { + urlGlob?: string, + urlRegexSource?: string, + urlRegexFlags?: string, +}; +export type BrowserContextRouteAPIRequestsFromHarResult = { + registrationId: string, +}; +export type BrowserContextUnrouteAPIRequestsFromHarParams = { + registrationId: string, +}; +export type BrowserContextUnrouteAPIRequestsFromHarOptions = { + +}; +export type BrowserContextUnrouteAPIRequestsFromHarResult = void; export type BrowserContextSetWebSocketInterceptionPatternsParams = { patterns: { glob?: string, diff --git a/packages/playwright-core/src/client/harRouter.ts b/packages/playwright-core/src/client/harRouter.ts index e4350ad5e3f8c..740122cd3e7ea 100644 --- a/packages/playwright-core/src/client/harRouter.ts +++ b/packages/playwright-core/src/client/harRouter.ts @@ -15,6 +15,7 @@ */ import { debugLogger } from '@utils/debugLogger'; +import { isRegExp, isString } from '@isomorphic/rtti'; import type { BrowserContext } from './browserContext'; import type { LocalUtils } from './localUtils'; @@ -29,6 +30,7 @@ export class HarRouter { private _harId: string; private _notFoundAction: HarNotFoundAction; private _options: { urlMatch?: URLMatch; baseURL?: string; }; + private _apiRequestRegistrations: { context: BrowserContext, registrationId: string }[] = []; static async create(localUtils: LocalUtils, file: string, notFoundAction: HarNotFoundAction, options: { urlMatch?: URLMatch }): Promise { const { harId, error } = await localUtils.harOpen({ file }); @@ -117,11 +119,26 @@ export class HarRouter { await page.route(this._options.urlMatch || '**/*', route => this._handle(route)); } + async addAPIRequestRoute(context: BrowserContext) { + const urlMatch = this._options.urlMatch; + const { registrationId } = await context._channel.routeAPIRequestsFromHar({ + harId: this._harId, + urlGlob: isString(urlMatch) ? urlMatch : undefined, + urlRegexSource: isRegExp(urlMatch) ? urlMatch.source : undefined, + urlRegexFlags: isRegExp(urlMatch) ? urlMatch.flags : undefined, + notFound: this._notFoundAction, + }); + this._apiRequestRegistrations.push({ context, registrationId }); + } + async [Symbol.asyncDispose]() { await this.dispose(); } dispose() { this._localUtils.harClose({ harId: this._harId }).catch(() => {}); + for (const { context, registrationId } of this._apiRequestRegistrations) + context._channel.unrouteAPIRequestsFromHar({ registrationId }).catch(() => {}); + this._apiRequestRegistrations = []; } } diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 2e037ab2b58e4..7a9640c1df7c6 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -832,6 +832,20 @@ scheme.BrowserContextSetNetworkInterceptionPatternsParams = tObject({ })), }); scheme.BrowserContextSetNetworkInterceptionPatternsResult = tOptional(tObject({})); +scheme.BrowserContextRouteAPIRequestsFromHarParams = tObject({ + harId: tString, + urlGlob: tOptional(tString), + urlRegexSource: tOptional(tString), + urlRegexFlags: tOptional(tString), + notFound: tEnum(['abort', 'fallback']), +}); +scheme.BrowserContextRouteAPIRequestsFromHarResult = tObject({ + registrationId: tString, +}); +scheme.BrowserContextUnrouteAPIRequestsFromHarParams = tObject({ + registrationId: tString, +}); +scheme.BrowserContextUnrouteAPIRequestsFromHarResult = tOptional(tObject({})); scheme.BrowserContextSetWebSocketInterceptionPatternsParams = tObject({ patterns: tArray(tObject({ glob: tOptional(tString), diff --git a/packages/playwright-core/src/server/browserContext.ts b/packages/playwright-core/src/server/browserContext.ts index e0dcbb214a42a..ce354c523e19c 100644 --- a/packages/playwright-core/src/server/browserContext.ts +++ b/packages/playwright-core/src/server/browserContext.ts @@ -39,11 +39,13 @@ import type { Browser, BrowserOptions } from './browser'; import type { ConsoleMessage } from './console'; import type { Download } from './download'; import type * as frames from './frames'; +import type { HarBackend } from './harBackend'; import type { PageError } from './page'; import type { Progress } from './progress'; import type { ClientCertificatesProxy } from './socksClientCertificatesInterceptor'; import type { SerializedStorage } from '@injected/storageScript'; import type * as types from './types'; +import type { URLMatch } from '@isomorphic/urlMatch'; import type * as channels from './channels'; const BrowserContextEvent = { @@ -120,6 +122,7 @@ export abstract class BrowserContext extends Sdk private _playwrightBindingExposed?: Promise; readonly dialogManager: DialogManager; private _consoleApiExposed = false; + private _harForAPIRequests: HarForAPIRequestsRegistration[] = []; constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) { super(browser, 'browser-context'); @@ -749,8 +752,37 @@ export abstract class BrowserContext extends Sdk async notifyRoutesInFlightAboutRemovedHandler(handler: network.RouteHandler): Promise { await Promise.all([...this._routesInFlight].map(route => route.removeHandler(handler))); } + + routeAPIRequestsFromHar(options: { harBackend: HarBackend, urlMatch: URLMatch | undefined, notFound: 'abort' | 'fallback', baseURL: string | undefined }): { dispose: () => void } { + const registration: HarForAPIRequestsRegistration = { + harBackend: options.harBackend, + urlMatch: options.urlMatch, + notFound: options.notFound, + baseURL: options.baseURL, + }; + // Give priority to the newest registration, mirroring BrowserContext.route/Page.route. + this._harForAPIRequests.unshift(registration); + return { + dispose: () => { + const index = this._harForAPIRequests.indexOf(registration); + if (index !== -1) + this._harForAPIRequests.splice(index, 1); + }, + }; + } + + harForAPIRequests(): readonly HarForAPIRequestsRegistration[] { + return this._harForAPIRequests; + } } +export type HarForAPIRequestsRegistration = { + harBackend: HarBackend; + urlMatch: URLMatch | undefined; + notFound: 'abort' | 'fallback'; + baseURL: string | undefined; +}; + export function validateBrowserContextOptions(options: types.BrowserContextOptions, browserOptions: BrowserOptions) { if (options.noDefaultViewport && options.deviceScaleFactor !== undefined) throw new Error(`"deviceScaleFactor" option is not supported with null "viewport"`); diff --git a/packages/playwright-core/src/server/channels.d.ts b/packages/playwright-core/src/server/channels.d.ts index 153920cac69b2..9ce0ed2cd52cb 100644 --- a/packages/playwright-core/src/server/channels.d.ts +++ b/packages/playwright-core/src/server/channels.d.ts @@ -1347,6 +1347,8 @@ export interface BrowserContextChannel extends BrowserContextEventTarget, Channe setGeolocation(params: BrowserContextSetGeolocationParams, progress: Progress): Promise; setHTTPCredentials(params: BrowserContextSetHTTPCredentialsParams, progress: Progress): Promise; setNetworkInterceptionPatterns(params: BrowserContextSetNetworkInterceptionPatternsParams, progress: Progress): Promise; + routeAPIRequestsFromHar(params: BrowserContextRouteAPIRequestsFromHarParams, progress: Progress): Promise; + unrouteAPIRequestsFromHar(params: BrowserContextUnrouteAPIRequestsFromHarParams, progress: Progress): Promise; setWebSocketInterceptionPatterns(params: BrowserContextSetWebSocketInterceptionPatternsParams, progress: Progress): Promise; setOffline(params: BrowserContextSetOfflineParams, progress: Progress): Promise; storageState(params: BrowserContextStorageStateParams, progress: Progress): Promise; @@ -1580,6 +1582,28 @@ export type BrowserContextSetNetworkInterceptionPatternsOptions = { }; export type BrowserContextSetNetworkInterceptionPatternsResult = void; +export type BrowserContextRouteAPIRequestsFromHarParams = { + harId: string, + urlGlob?: string, + urlRegexSource?: string, + urlRegexFlags?: string, + notFound: 'abort' | 'fallback', +}; +export type BrowserContextRouteAPIRequestsFromHarOptions = { + urlGlob?: string, + urlRegexSource?: string, + urlRegexFlags?: string, +}; +export type BrowserContextRouteAPIRequestsFromHarResult = { + registrationId: string, +}; +export type BrowserContextUnrouteAPIRequestsFromHarParams = { + registrationId: string, +}; +export type BrowserContextUnrouteAPIRequestsFromHarOptions = { + +}; +export type BrowserContextUnrouteAPIRequestsFromHarResult = void; export type BrowserContextSetWebSocketInterceptionPatternsParams = { patterns: { glob?: string, diff --git a/packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts b/packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts index fb57858e7148d..0ead57553dbab 100644 --- a/packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts @@ -46,10 +46,13 @@ import type { Request, Response, RouteHandler } from '../network'; import type { InitScript, Page, PageError } from '../page'; import type { Disposable } from '../disposable'; import type { DispatcherScope } from './dispatcher'; +import type { LocalUtilsDispatcher } from './localUtilsDispatcher'; import type * as channels from '../channels'; import type { Progress } from '../progress'; import type { URLMatch } from '@isomorphic/urlMatch'; +type HarForAPIRequestsDisposable = Disposable & { registrationId: string }; + export class BrowserContextDispatcher extends Dispatcher implements channels.BrowserContextChannel { _type_BrowserContext = true; private _context: BrowserContext; @@ -335,6 +338,39 @@ export class BrowserContextDispatcher extends Dispatcher { + // Reuse the HarBackend that was already opened via localUtils.harOpen for the page-side + // route, rather than opening a second backend for the same HAR file. The backend is owned + // by LocalUtils and closed via harClose, so this registration must not dispose it. + const localUtils = [...this.connection._dispatcherByGuid.values()].find(d => d._type === 'LocalUtils') as LocalUtilsDispatcher | undefined; + const harBackend = localUtils?.harBackendForId(params.harId); + if (!harBackend) + throw new Error('Internal error: har was not opened'); + const urlMatch: URLMatch | undefined = + params.urlRegexSource !== undefined && params.urlRegexFlags !== undefined ? new RegExp(params.urlRegexSource, params.urlRegexFlags) : + params.urlGlob !== undefined ? params.urlGlob : undefined; + const registrationId = createGuid(); + const registration = this._context.routeAPIRequestsFromHar({ + harBackend, + urlMatch, + notFound: params.notFound, + baseURL: this._context._options.baseURL, + }); + this._disposables.push({ + registrationId, + dispose: async () => registration.dispose(), + } as HarForAPIRequestsDisposable); + return { registrationId }; + } + + async unrouteAPIRequestsFromHar(params: channels.BrowserContextUnrouteAPIRequestsFromHarParams, progress: Progress): Promise { + const index = this._disposables.findIndex(d => (d as HarForAPIRequestsDisposable).registrationId === params.registrationId); + if (index === -1) + return; + const [disposable] = this._disposables.splice(index, 1); + await progress.race(disposable.dispose()); + } + async storageState(params: channels.BrowserContextStorageStateParams, progress: Progress): Promise { return await this._context.storageState(progress, params.indexedDB, params.credentials); } diff --git a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts index 33268afada409..eb44eed4540d9 100644 --- a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts @@ -67,6 +67,10 @@ export class LocalUtilsDispatcher extends Dispatcher { return await localUtils.harUnzip(progress, params); } diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index f048bd2a3659b..5c2080b9c415a 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -23,7 +23,7 @@ import * as zlib from 'zlib'; import { createGuid } from '@utils/crypto'; import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent, timingForSocket } from '@utils/happyEyeballs'; import { assert } from '@isomorphic/assert'; -import { constructURLBasedOnBaseURL } from '@isomorphic/urlMatch'; +import { constructURLBasedOnBaseURL, urlMatches } from '@isomorphic/urlMatch'; import { eventsHelper } from '@utils/eventsHelper'; import { monotonicTime } from '@isomorphic/time'; import { createProxyAgent } from '@utils/network'; @@ -150,6 +150,10 @@ export abstract class APIRequestContext extends SdkObject { abstract addCookies(cookies: channels.NetworkCookie[]): Promise; abstract cookies(progress: Progress, url: URL): Promise; + protected async _lookupInHar(progress: Progress, url: URL, method: string, headers: HeadersObject, postData: Buffer | undefined, maxRedirects: number): Promise { + return undefined; + } + protected _disposeImpl() { this._disposed = true; APIRequestContext.allInstances.delete(this); @@ -225,7 +229,9 @@ export abstract class APIRequestContext extends SdkObject { const postData = serializePostData(params, headers); if (postData) setHeader(headers, 'content-length', String(postData.byteLength)); - const { body, log, response } = await this._sendRequestWithRetries(progress, requestUrl, options, postData, params.maxRetries); + const { body, log, response } = + (await this._lookupInHar(progress, requestUrl, method, headers, postData, maxRedirects)) || + (await this._sendRequestWithRetries(progress, requestUrl, options, postData, params.maxRetries)); const failOnStatusCode = params.failOnStatusCode !== undefined ? params.failOnStatusCode : !!defaults.failOnStatusCode; if (failOnStatusCode && (response.status < 200 || response.status >= 400)) { let responseText = ''; @@ -242,7 +248,7 @@ export abstract class APIRequestContext extends SdkObject { return { ...response, fetchUid }; } - private _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): channels.NetworkCookie[] { + _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): channels.NetworkCookie[] { if (!setCookie) return []; const url = new URL(responseUrl); @@ -269,7 +275,7 @@ export abstract class APIRequestContext extends SdkObject { return cookies; } - private async _updateRequestCookieHeader(progress: Progress, url: URL, headers: HeadersObject) { + async _updateRequestCookieHeader(progress: Progress, url: URL, headers: HeadersObject) { if (getHeader(headers, 'cookie') !== undefined) return; const contextCookies = await this.cookies(progress, url); @@ -682,6 +688,109 @@ export class BrowserContextAPIRequestContext extends APIRequestContext { override async storageState(progress: Progress, indexedDB?: boolean): Promise { return this._context.storageState(progress, indexedDB); } + + protected override async _lookupInHar(progress: Progress, url: URL, method: string, headers: HeadersObject, postData: Buffer | undefined, maxRedirects: number): Promise { + const registrations = this._context.harForAPIRequests(); + if (!registrations.length) + return undefined; + + const log: string[] = []; + const fetchLog = (message: string) => { + log.push(message); + progress.log(message); + }; + + await this._updateRequestCookieHeader(progress, url, headers); + + const urlString = url.toString(); + fetchLog(`→ ${method} ${urlString}`); + const headersArray: HeadersArray = Object.entries(headers).map(([name, value]) => ({ name, value })); + for (const registration of registrations) { + if (!urlMatches(registration.baseURL, urlString, registration.urlMatch)) + continue; + const lookupResult = await progress.race(registration.harBackend.lookup(urlString, method, headersArray, postData, false, { apiRequestOnly: true, maxRedirects })); + if (lookupResult.action === 'error') { + fetchLog(`HAR: ${lookupResult.message ?? 'lookup failed'}`); + continue; + } + if (lookupResult.action === 'noentry') { + if (registration.notFound === 'abort') + throw new Error(`Request "${method} ${urlString}" was not found in the HAR file`); + continue; + } + + const finalUrl = lookupResult.url ?? urlString; + const status = lookupResult.status ?? 0; + const statusText = lookupResult.statusText ?? ''; + const responseHeaders = lookupResult.headers ?? []; + const body = lookupResult.body ?? Buffer.from(''); + fetchLog(`← ${status} ${statusText} (from HAR)`); + for (const { name, value } of responseHeaders) + fetchLog(` ${name}: ${value}`); + + // Emit Request/RequestFinished here (not before the loop) so that we do not double-emit + // when there is no match and we fall through to the live request path, which emits them too. + const requestCookies = getHeader(headers, 'cookie')?.split(';').map(p => { + const indexOfEquals = p.indexOf('='); + const name = indexOfEquals !== -1 ? p.substring(0, indexOfEquals).trim() : p.trim(); + const value = indexOfEquals !== -1 ? p.substring(indexOfEquals + 1).trim() : ''; + return { name, value }; + }) || []; + const requestEvent: APIRequestEvent = { + url, + method, + headers, + cookies: requestCookies, + postData, + }; + this.emit(APIRequestContext.Events.Request, requestEvent); + + const setCookie = responseHeaders.filter(h => h.name.toLowerCase() === 'set-cookie').map(h => h.value); + const cookies = this._parseSetCookieHeader(finalUrl, setCookie); + if (cookies.length) { + try { + await progress.race(this.addCookies(cookies)); + } catch (e) { + // Cookie value is limited by 4096 characters in the browsers. If setCookies failed, + // we try setting each cookie individually just in case only some of them are bad. + await progress.race(Promise.all(cookies.map(c => this.addCookies([c]).catch(() => {})))); + } + } + + const serverAddr = lookupResult.serverIPAddress !== undefined && lookupResult.serverPort !== undefined ? + { ipAddress: lookupResult.serverIPAddress, port: lookupResult.serverPort } : undefined; + + const requestFinishedEvent: APIRequestFinishedEvent = { + requestEvent, + httpVersion: lookupResult.httpVersion ?? 'HTTP/1.1', + statusCode: status, + statusMessage: statusText, + headers: toHeadersObject(responseHeaders), + rawHeaders: responseHeaders.flatMap(({ name, value }) => [name, value]), + cookies, + body, + timings: lookupResult.timings ?? { send: -1, wait: -1, receive: -1 }, + serverIPAddress: lookupResult.serverIPAddress, + serverPort: lookupResult.serverPort, + securityDetails: lookupResult.securityDetails, + }; + this.emit(APIRequestContext.Events.RequestFinished, requestFinishedEvent); + + return { + body, + log, + response: { + url: finalUrl, + status, + statusText, + headers: responseHeaders, + securityDetails: lookupResult.securityDetails, + serverAddr, + }, + }; + } + return undefined; + } } @@ -751,6 +860,25 @@ function toHeadersArray(rawHeaders: string[]): types.HeadersArray { return result; } +function toHeadersObject(headers: types.HeadersArray): http.IncomingHttpHeaders { + const result: http.IncomingHttpHeaders = {}; + for (const { name, value } of headers) { + const key = name.toLowerCase(); + // set-cookie is the only multi-valued header Node exposes as an array. + if (key === 'set-cookie') { + const existing = result['set-cookie']; + if (existing) + existing.push(value); + else + result['set-cookie'] = [value]; + } else { + const existing = result[key]; + result[key] = existing ? `${existing}, ${value}` : value; + } + } + return result; +} + const redirectStatus = [301, 302, 303, 307, 308]; function parseCookie(header: string): channels.NetworkCookie | null { diff --git a/packages/playwright-core/src/server/harBackend.ts b/packages/playwright-core/src/server/harBackend.ts index bff11879e112b..93fb7febfec18 100644 --- a/packages/playwright-core/src/server/harBackend.ts +++ b/packages/playwright-core/src/server/harBackend.ts @@ -26,6 +26,8 @@ import type * as har from '@trace/har'; const redirectStatus = [301, 302, 303, 307, 308]; +class MaxRedirectsExceededError extends Error {} + export class HarBackend { readonly id: string; private _harFile: har.HARFile; @@ -39,18 +41,30 @@ export class HarBackend { this._zipFile = zipFile; } - async lookup(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, isNavigationRequest: boolean): Promise<{ + async lookup(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, isNavigationRequest: boolean, options: { apiRequestOnly?: boolean, maxRedirects?: number } = {}): Promise<{ action: 'error' | 'redirect' | 'fulfill' | 'noentry', message?: string, redirectURL?: string, + url?: string, status?: number, + statusText?: string, + httpVersion?: string, headers?: HeadersArray, - body?: Buffer + cookies?: har.Cookie[], + body?: Buffer, + timings?: har.Timings, + serverIPAddress?: string, + serverPort?: number, + securityDetails?: har.SecurityDetails, }> { let entry; try { - entry = await this._harFindResponse(url, method, headers, postData); + entry = await this._harFindResponse(url, method, headers, postData, options); } catch (e) { + // A redirect-limit overflow is a hard error the caller must surface (mirrors the live + // request path), so let it propagate instead of degrading to a soft 'error' result. + if (e instanceof MaxRedirectsExceededError) + throw e; return { action: 'error', message: 'HAR error: ' + e.message }; } @@ -66,9 +80,17 @@ export class HarBackend { const buffer = await this._loadContent(response.content); return { action: 'fulfill', + url: entry.request.url, status: response.status, + statusText: response.statusText, + httpVersion: response.httpVersion, headers: response.headers, + cookies: response.cookies, body: buffer, + timings: entry.timings, + serverIPAddress: entry.serverIPAddress, + serverPort: entry._serverPort, + securityDetails: entry._securityDetails, }; } catch (e) { return { action: 'error', message: e.message }; @@ -93,14 +115,17 @@ export class HarBackend { return buffer; } - private async _harFindResponse(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined): Promise { + private async _harFindResponse(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, options: { apiRequestOnly?: boolean, maxRedirects?: number } = {}): Promise { const harLog = this._harFile.log; const visited = new Set(); + let maxRedirects = options.maxRedirects; while (true) { const entries: har.Entry[] = []; for (const candidate of harLog.entries) { if (candidate.request.url !== url || candidate.request.method !== method) continue; + if (options.apiRequestOnly && !candidate._apiRequest) + continue; if (method === 'POST' && postData && candidate.request.postData) { const buffer = await this._loadContent(candidate.request.postData); if (!buffer.equals(postData)) { @@ -142,6 +167,16 @@ export class HarBackend { // Follow redirects. const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location'); if (redirectStatus.includes(entry.response.status) && locationHeader) { + if (maxRedirects !== undefined) { + // Mirror the live request path (see APIRequestContext._sendRequest): a negative + // limit means "do not follow redirects" and returns the 3xx response as-is, while + // a zero limit means the redirect budget has been exhausted. + if (maxRedirects < 0) + return entry; + if (maxRedirects === 0) + throw new MaxRedirectsExceededError('Max redirect count exceeded'); + --maxRedirects; + } const locationURL = new URL(locationHeader.value, url); url = locationURL.toString(); if ((entry.response.status === 301 || entry.response.status === 302) && method === 'POST' || diff --git a/packages/playwright-core/src/server/localUtils.ts b/packages/playwright-core/src/server/localUtils.ts index c2b36758a6a94..f82dc1f83bd47 100644 --- a/packages/playwright-core/src/server/localUtils.ts +++ b/packages/playwright-core/src/server/localUtils.ts @@ -148,9 +148,16 @@ async function deleteStackSession(progress: Progress, stackSessions: Map, params: channels.LocalUtilsHarOpenParams): Promise { - let harBackend: HarBackend; - if (params.file.endsWith('.zip')) { - const zipFile = new ZipFile(params.file); + const result = await openHarBackend(progress, params.file); + if ('error' in result) + return { error: result.error }; + harBackends.set(result.harBackend.id, result.harBackend); + return { harId: result.harBackend.id }; +} + +export async function openHarBackend(progress: Progress, file: string): Promise<{ harBackend: HarBackend } | { error: string }> { + if (file.endsWith('.zip')) { + const zipFile = new ZipFile(file); try { const entryNames = await progress.race(zipFile.entries()); const harEntryName = entryNames.find(e => e.endsWith('.har')); @@ -158,17 +165,14 @@ export async function harOpen(progress: Progress, harBackends: Map, params: channels.LocalUtilsHarLookupParams): Promise { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index b2150842591d0..15f2c2da4e153 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -9608,6 +9608,15 @@ export interface BrowserContext { * @param options */ routeFromHAR(har: string, options?: { + /** + * If set to `true`, requests made via [APIRequestContext](https://playwright.dev/docs/api/class-apirequestcontext) + * (such as [browserContext.request](https://playwright.dev/docs/api/class-browsercontext#browser-context-request) or + * [page.request](https://playwright.dev/docs/api/class-page#page-request)) are also served from the HAR file. By + * default these requests are sent to the network, matching the behavior prior to v1.62. Defaults to `false` for + * backward compatibility. + */ + interceptAPIRequests?: boolean; + /** * - If set to 'abort' any request not found in the HAR file will be aborted. * - If set to 'fallback' falls through to the next route handler in the handler chain. diff --git a/packages/protocol/spec/browserContext.yml b/packages/protocol/spec/browserContext.yml index 10c582f970055..753e9d6a2c225 100644 --- a/packages/protocol/spec/browserContext.yml +++ b/packages/protocol/spec/browserContext.yml @@ -155,6 +155,26 @@ BrowserContext: regexFlags: string? urlPattern: URLPattern? + routeAPIRequestsFromHar: + internal: true + parameters: + harId: string + urlGlob: string? + urlRegexSource: string? + urlRegexFlags: string? + notFound: + type: enum + literals: + - abort + - fallback + returns: + registrationId: string + + unrouteAPIRequestsFromHar: + internal: true + parameters: + registrationId: string + setWebSocketInterceptionPatterns: title: Route WebSockets group: route diff --git a/tests/library/browsercontext-har.spec.ts b/tests/library/browsercontext-har.spec.ts index 98c3e1fe04feb..211a172a79b88 100644 --- a/tests/library/browsercontext-har.spec.ts +++ b/tests/library/browsercontext-har.spec.ts @@ -610,3 +610,341 @@ it('should ignore aborted requests', async ({ contextFactory, server }) => { expect(result).toBe('timeout'); } }); + +it.describe('interceptAPIRequests', () => { + it('should fulfill APIRequestContext requests from HAR', { + annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22869' } + }, async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ hello: 'live' })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + const recorded = await page1.request.get(server.PREFIX + '/api/data'); + expect(await recorded.json()).toEqual({ hello: 'live' }); + await context1.close(); + + // Now stop serving on the network side - the request must come from the HAR. + server.setRoute('/api/data', (req, res) => res.end('NOT_FROM_HAR')); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + const replayed = await page2.request.get(server.PREFIX + '/api/data'); + expect(await replayed.json()).toEqual({ hello: 'live' }); + await context2.close(); + }); + + it('should not intercept APIRequestContext requests by default (backward compat)', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ hello: 'live' })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/data'); + await context1.close(); + + // Without the option, the live network is hit. + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ hello: 'fresh' })); + }); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { notFound: 'fallback' }); + const page2 = await context2.newPage(); + const replayed = await page2.request.get(server.PREFIX + '/api/data'); + expect(await replayed.json()).toEqual({ hello: 'fresh' }); + }); + + it('should fall back to the network when interceptAPIRequests + notFound:fallback', async ({ contextFactory, server }, testInfo) => { + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await context1.close(); + + server.setRoute('/api/missing', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ source: 'network' })); + }); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true, notFound: 'fallback' }); + const page2 = await context2.newPage(); + const response = await page2.request.get(server.PREFIX + '/api/missing'); + expect(await response.json()).toEqual({ source: 'network' }); + }); + + it('should abort unmatched APIRequestContext requests when interceptAPIRequests + notFound:abort', async ({ contextFactory, server }, testInfo) => { + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await context1.close(); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true /* default notFound: abort */ }); + const page2 = await context2.newPage(); + const error = await page2.request.get(server.PREFIX + '/api/missing').catch(e => e); + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('was not found in the HAR file'); + }); + + it('should respect url filter for APIRequestContext requests', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ source: 'hario' })); + }); + server.setRoute('/other', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ source: 'live' })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/data'); + await page1.request.get(server.PREFIX + '/other'); + await context1.close(); + + // Re-route /api/data so that only the HAR can produce 'hario'. + server.setRoute('/api/data', (req, res) => res.end('NOT_FROM_HAR')); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true, url: '**/api/**', notFound: 'fallback' }); + const page2 = await context2.newPage(); + const fromHar = await page2.request.get(server.PREFIX + '/api/data'); + expect(await fromHar.json()).toEqual({ source: 'hario' }); + // /other does not match the url filter, so it hits the network. + const fromNetwork = await page2.request.get(server.PREFIX + '/other'); + expect(await fromNetwork.json()).toEqual({ source: 'live' }); + }); + + it('should match APIRequestContext POST requests by body', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/echo', (req, res) => { + const chunks: Buffer[] = []; + req.on('data', c => chunks.push(c)); + req.on('end', () => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ echoed: Buffer.concat(chunks).toString() })); + }); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.post(server.PREFIX + '/echo', { data: 'one' }); + await page1.request.post(server.PREFIX + '/echo', { data: 'two' }); + await context1.close(); + + server.setRoute('/echo', (req, res) => res.end('NOT_FROM_HAR')); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + const r1 = await page2.request.post(server.PREFIX + '/echo', { data: 'one' }); + const r2 = await page2.request.post(server.PREFIX + '/echo', { data: 'two' }); + expect(await r1.json()).toEqual({ echoed: 'one' }); + expect(await r2.json()).toEqual({ echoed: 'two' }); + }); + + it('should stop intercepting APIRequestContext requests after unrouteAll', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ source: 'hario' })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/data'); + await context1.close(); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + // First call: served from HAR. + const first = await page2.request.get(server.PREFIX + '/api/data'); + expect(await first.json()).toEqual({ source: 'hario' }); + + await context2.unrouteAll(); + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ source: 'live' })); + }); + + // After unrouteAll: the registration is gone, the live network is hit. + const second = await page2.request.get(server.PREFIX + '/api/data'); + expect(await second.json()).toEqual({ source: 'live' }); + }); + + it('should only match _apiRequest entries when intercepting APIRequestContext', { + annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22869' } + }, async ({ contextFactory, server }, testInfo) => { + // The HAR will contain TWO entries for the same URL: one from a browser fetch and one from + // page.request. interceptAPIRequests must serve only the API-request entry. + server.setRoute('/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + const fromApi = req.headers['x-from'] === 'api'; + res.end(JSON.stringify({ source: fromApi ? 'recorded-api' : 'recorded-browser' })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + // Browser-side fetch — recorded WITHOUT _apiRequest. + await page1.evaluate(url => fetch(url, { headers: { 'x-from': 'browser' } }).then(r => r.json()), server.PREFIX + '/data'); + // API-request — recorded WITH _apiRequest:true. + await page1.request.get(server.PREFIX + '/data', { headers: { 'x-from': 'api' } }); + await context1.close(); + + // Sanity: the HAR must contain at least one _apiRequest entry. + const harText = fs.readFileSync(harPath, 'utf-8'); + expect(harText).toContain('"_apiRequest":true'); + + // Make the live network unreachable for this URL — if interception works correctly + // we never hit the network anyway. + server.setRoute('/data', (req, res) => res.end('NOT_FROM_HAR')); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + const apiResponse = await page2.request.get(server.PREFIX + '/data', { headers: { 'x-from': 'api' } }); + // Must be the recorded API entry, not the recorded browser entry. + expect(await apiResponse.json()).toEqual({ source: 'recorded-api' }); + }); + + it('should apply set-cookie side-effects from intercepted APIRequestContext requests', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/login', (req, res) => { + res.setHeader('Set-Cookie', 'session=har-token; Path=/'); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ ok: true })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/login'); + await context1.close(); + + server.setRoute('/api/login', (req, res) => res.end('NOT_FROM_HAR')); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + await page2.request.get(server.PREFIX + '/api/login'); + // The set-cookie from the HAR response must be applied to the browser context. + const cookies = await context2.cookies(server.PREFIX); + expect(cookies.find(c => c.name === 'session')?.value).toBe('har-token'); + }); + + it('should populate statusText and serverAddr for intercepted APIRequestContext requests', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.statusCode = 201; + res.statusMessage = 'Created'; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ ok: true })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/data'); + await context1.close(); + + server.setRoute('/api/data', (req, res) => res.end('NOT_FROM_HAR')); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + const response = await page2.request.get(server.PREFIX + '/api/data'); + expect(response.status()).toBe(201); + expect(response.statusText()).toBe('Created'); + }); + + it('should re-record intercepted APIRequestContext requests into a new HAR', async ({ contextFactory, server }, testInfo) => { + server.setRoute('/api/data', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ hello: 'live' })); + }); + + // Record the first HAR with the API request. + const harPath1 = testInfo.outputPath('api1.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath1, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/data'); + await context1.close(); + + // Replay from the first HAR while recording into a second HAR. The Request/RequestFinished + // events emitted from the HAR-replay path must cause the API request to be captured again. + server.setRoute('/api/data', (req, res) => res.end('NOT_FROM_HAR')); + const harPath2 = testInfo.outputPath('api2.har'); + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath1, { interceptAPIRequests: true }); + await context2.routeFromHAR(harPath2, { update: true }); + const page2 = await context2.newPage(); + await page2.goto(server.EMPTY_PAGE); + const replayed = await page2.request.get(server.PREFIX + '/api/data'); + expect(await replayed.json()).toEqual({ hello: 'live' }); + await context2.close(); + + const harText = fs.readFileSync(harPath2, 'utf-8'); + expect(harText).toContain('"_apiRequest":true'); + expect(harText).toContain('/api/data'); + }); + + it('should throw when intercepted APIRequestContext request exceeds maxRedirects', async ({ contextFactory, server }, testInfo) => { + const redirect = '/api/step1'; + server.setRoute('/api/start', (req, res) => { + res.statusCode = 302; + res.setHeader('Location', server.PREFIX + redirect); + res.end(); + }); + server.setRoute('/api/step1', (req, res) => { + res.statusCode = 302; + res.setHeader('Location', server.PREFIX + '/api/step2'); + res.end(); + }); + server.setRoute('/api/step2', (req, res) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ done: true })); + }); + + const harPath = testInfo.outputPath('api.har'); + const context1 = await contextFactory(); + await context1.routeFromHAR(harPath, { update: true }); + const page1 = await context1.newPage(); + await page1.goto(server.EMPTY_PAGE); + await page1.request.get(server.PREFIX + '/api/start'); + await context1.close(); + + const context2 = await contextFactory(); + await context2.routeFromHAR(harPath, { interceptAPIRequests: true }); + const page2 = await context2.newPage(); + const error = await page2.request.get(server.PREFIX + '/api/start', { maxRedirects: 1 }).catch(e => e); + expect(error).toBeInstanceOf(Error); + expect(error.message).toContain('Max redirect count exceeded'); + }); +});