chore(bidi): add support for fetching request bodies

Author: hbenl

packages/playwright-core/src/client/fetch.ts

@@ -239,7 +239,7 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
         }
       }
       if (postDataBuffer === undefined && jsonData === undefined && formData === undefined && multipartData === undefined)
-        postDataBuffer = options.request?.postDataBuffer() || undefined;
+        postDataBuffer = (await options.request?.bodyBuffer()) || undefined;
       const fixtures = {
         __testHookLookup: (options as any).__testHookLookup
       };

packages/playwright-core/src/client/harRouter.ts

@@ -50,7 +50,7 @@ export class HarRouter {
       url: request.url(),
       method: request.method(),
       headers: (await request.headersArray()),
-      postData: request.postDataBuffer() || undefined,
+      postData: (await request.bodyBuffer()) || undefined,
       isNavigationRequest: request.isNavigationRequest()
     });
 

packages/playwright-core/src/server/bidi/bidiBrowser.ts

@@ -74,7 +74,7 @@ export class BidiBrowser extends Browser {
     });
 
     await browser._browserSession.send('network.addDataCollector', {
-      dataTypes: [bidi.Network.DataType.Response],
+      dataTypes: [bidi.Network.DataType.Request, bidi.Network.DataType.Response],
       maxEncodedDataSize: 20_000_000, // same default as in CDP: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/inspector/inspector_network_agent.cc;l=134;drc=4128411589187a396829a827f59a655bed876aa7
     });
 

packages/playwright-core/src/server/bidi/bidiNetworkManager.ts

@@ -79,7 +79,8 @@ export class BidiNetworkManager {
         route = new BidiRouteImpl(this._session, param.request.request);
       }
     }
-    const request = new BidiRequest(frame, redirectedFrom, param, route);
+    const getRequestBody = param.request.bodySize ? () => getNetworkData(this._session, param.request, bidi.Network.DataType.Request) : null;
+    const request = new BidiRequest(frame, redirectedFrom, param, getRequestBody, route);
     this._requests.set(request._id, request);
     this._page.frameManager.requestStarted(request.request, route);
   }
@@ -88,11 +89,7 @@ export class BidiNetworkManager {
     const request = this._requests.get(params.request.request);
     if (!request)
       return;
-    const getResponseBody = async () => {
-      const { bytes } = await this._session.send('network.getData', { request: params.request.request, dataType: bidi.Network.DataType.Response });
-      const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
-      return Buffer.from(bytes.value, encoding);
-    };
+    const getResponseBody = () => getNetworkData(this._session, params.request, bidi.Network.DataType.Response);
     const timings = params.request.timings;
     const startTime = timings.requestTime;
     function relativeToStart(time: number): number {
@@ -236,14 +233,12 @@ class BidiRequest {
   // store the first and only Route in the chain (if any).
   _originalRequestRoute: BidiRouteImpl | undefined;
 
-  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, route: BidiRouteImpl | undefined) {
+  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, getRequestBody: (() => Promise<Buffer>) | null, route: BidiRouteImpl | undefined) {
     this._id = payload.request.request;
     if (redirectedFrom)
       redirectedFrom._redirectedTo = this;
-    // TODO: missing in the spec?
-    const postDataBuffer = null;
     this.request = new network.Request(frame._page.browserContext, frame, null, redirectedFrom ? redirectedFrom.request : null, payload.navigation ?? undefined, payload.request.url,
-        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, postDataBuffer, fromBidiHeaders(payload.request.headers));
+        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, getRequestBody, fromBidiHeaders(payload.request.headers));
     // "raw" headers are the same as "provisional" headers in Bidi.
     this.request.setRawRequestHeaders(null);
     this.request._setBodySize(payload.request.bodySize || 0);
@@ -390,3 +385,9 @@ function resourceTypeFromBidi(requestDestination: string, requestInitiatorType:
     default: return 'other';
   }
 }
+
+async function getNetworkData(session: BidiSession, request: bidi.Network.RequestData, dataType: bidi.Network.DataType) {
+  const { bytes } = await session.send('network.getData', { request: request.request, dataType });
+  const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
+  return Buffer.from(bytes.value, encoding);
+}

packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts

@@ -1383,6 +1383,7 @@ export namespace Network {
 }
 export namespace Network {
   export const enum DataType {
+    Request = 'request',
     Response = 'response',
   }
 }

packages/playwright-core/src/server/dispatchers/networkDispatchers.ts

@@ -67,7 +67,7 @@ export class RequestDispatcher extends Dispatcher<Request, channels.RequestChann
   }
 
   async body(params: channels.RequestBodyParams, progress: Progress): Promise<channels.RequestBodyResult> {
-    const postData = this._object.postDataBuffer();
+    const postData = await this._object.body();
     return { body: postData === null ? undefined : postData };
   }
 

packages/playwright-core/src/server/network.ts

@@ -104,7 +104,7 @@ export class Request extends SdkObject {
   private _url: string;
   private _resourceType: string;
   private _method: string;
-  private _postData: Buffer | null;
+  private _postData: Buffer | null | (() => Promise<Buffer>);
   readonly _headers: HeadersArray;
   private _headersMap = new Map<string, string>();
   readonly _frame: frames.Frame | null = null;
@@ -122,7 +122,7 @@ export class Request extends SdkObject {
   };
 
   constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined,
-    url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray) {
+    url: string, resourceType: string, method: string, postData: Buffer | null | (() => Promise<Buffer>), headers: HeadersArray) {
     super(frame || context, 'request');
     assert(!url.startsWith('data:'), 'Data urls should not fire requests');
     this._context = context;
@@ -173,8 +173,22 @@ export class Request extends SdkObject {
     return this._overrides?.method || this._method;
   }
 
+  async body(): Promise<Buffer | null> {
+    if (this._overrides?.postData)
+      return this._overrides?.postData;
+    if (typeof this._postData === 'function')
+      return await this._postData();
+    else
+      return this._postData;
+  }
+
   postDataBuffer(): Buffer | null {
-    return this._overrides?.postData || this._postData;
+    if (this._overrides?.postData)
+      return this._overrides?.postData;
+    if (typeof this._postData === 'function')
+      return null;
+    else
+      return this._postData;
   }
 
   headers(): HeadersArray {

packages/playwright-ct-core/src/router.ts

@@ -104,7 +104,7 @@ export class Router {
       for (const { name, value } of headersArray)
         headers.append(name, value);
 
-      const buffer = request.postDataBuffer();
+      const buffer = await request.bodyBuffer();
       const body = buffer?.byteLength ? new Int8Array(buffer.buffer as ArrayBuffer, buffer.byteOffset, buffer.length) : undefined;
 
       const newRequest = new Request(request.url(), {

tests/library/browsercontext-har.spec.ts

@@ -342,7 +342,7 @@ it('should record overridden requests to har', async ({ contextFactory, server }
   await page1.route('**/echo_redir', async route => {
     await route.fallback({
       url: server.PREFIX + '/echo',
-      postData: +route.request().postData() + 10,
+      postData: +(await route.request().body()) + 10,
     });
   });
   expect(await page1.evaluate(fetchFunction, { path: '/echo_redir', body: '1' })).toBe('11');

tests/library/browsercontext-route.spec.ts

@@ -21,13 +21,13 @@ import type { Route } from '@playwright/test';
 it('should intercept', async ({ browser, server }) => {
   const context = await browser.newContext();
   let intercepted = false;
-  await context.route('**/empty.html', route => {
+  await context.route('**/empty.html', async route => {
     intercepted = true;
     const request = route.request();
     expect(request.url()).toContain('empty.html');
     expect(request.headers()['user-agent']).toBeTruthy();
     expect(request.method()).toBe('GET');
-    expect(request.postData()).toBe(null);
+    expect(await request.body()).toBe(null);
     expect(request.isNavigationRequest()).toBe(true);
     expect(request.resourceType()).toBe('document');
     expect(request.frame() === page.mainFrame()).toBe(true);