feat: pause on --debug

packages/playwright/src/common/ipc.ts

@@ -84,11 +84,15 @@ export type AttachmentPayload = {
   stepId?: string;
 };
 
+export type TestErrorPayload = {
+  testId: string;
+  errors: TestInfoErrorImpl[];
+};
+
 export type TestInfoErrorImpl = TestInfoError;
 
 export type TestPausedPayload = {
   testId: string;
-  errors: TestInfoErrorImpl[];
 };
 
 export type CustomMessageRequestPayload = {
@@ -105,7 +109,6 @@ export type TestEndPayload = {
   testId: string;
   duration: number;
   status: TestStatus;
-  errors: TestInfoErrorImpl[];
   hasNonRetriableError: boolean;
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -96,7 +96,8 @@ export type JsonTestResultEnd = {
   id: string;
   duration: number;
   status: reporterTypes.TestStatus;
-  errors: reporterTypes.TestError[];
+  /** No longer emitted, but kept for backwards compatibility */
+  errors?: reporterTypes.TestError[];
   /** No longer emitted, but kept for backwards compatibility */
   attachments?: JsonAttachment[];
   annotations?: TestAnnotation[];
@@ -132,7 +133,7 @@ export type JsonFullResult = {
 };
 
 export type JsonEvent = JsonOnConfigureEvent | JsonOnBlobReportMetadataEvent | JsonOnEndEvent | JsonOnExitEvent | JsonOnProjectEvent | JsonOnBeginEvent | JsonOnTestBeginEvent
-  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
+  | JsonOnTestEndEvent | JsonOnStepBeginEvent | JsonOnStepEndEvent | JsonOnAttachEvent | JsonOnTestErrorsEvent | JsonOnErrorEvent | JsonOnStdIOEvent;
 
 export type JsonOnConfigureEvent = {
   method: 'onConfigure';
@@ -198,6 +199,15 @@ export type JsonOnAttachEvent = {
   params: JsonTestResultOnAttach;
 };
 
+export type JsonOnTestErrorsEvent = {
+  method: 'onTestErrors';
+  params: {
+    testId: string;
+    resultId: string;
+    errors: reporterTypes.TestError[];
+  }
+};
+
 export type JsonOnErrorEvent = {
   method: 'onError';
   params: {
@@ -294,6 +304,10 @@ export class TeleReporterReceiver {
       this._onAttach(params.testId, params.resultId, params.attachments);
       return;
     }
+    if (method === 'onTestErrors') {
+      this._onTestErrors(params.testId, params.resultId, params.errors);
+      return;
+    }
     if (method === 'onStepEnd') {
       this._onStepEnd(params.testId, params.resultId, params.step);
       return;
@@ -353,8 +367,11 @@ export class TeleReporterReceiver {
     const result = test.results.find(r => r._id === payload.id)!;
     result.duration = payload.duration;
     result.status = payload.status;
-    result.errors = payload.errors;
-    result.error = result.errors?.[0];
+    // Errors are only present here from legacy blobs. These override all _onTestErrors events
+    if (!!payload.errors) {
+      result.errors = payload.errors;
+      result.error = result.errors[0];
+    }
     // Attachments are only present here from legacy blobs. These override all _onAttach events
     if (!!payload.attachments)
       result.attachments = this._parseAttachments(payload.attachments);
@@ -404,6 +421,13 @@ export class TeleReporterReceiver {
     })));
   }
 
+  private _onTestErrors(testId: string, resultId: string, errors: reporterTypes.TestError[]) {
+    const test = this._tests.get(testId)!;
+    const result = test.results.find(r => r._id === resultId)!;
+    result.errors.push(...errors);
+    result.error = result.errors[0];
+  }
+
   private _onError(error: reporterTypes.TestError) {
     this._reporter.onError?.(error);
   }

packages/playwright/src/reporters/internalReporter.ts

@@ -94,6 +94,7 @@ export class InternalReporter implements ReporterV2 {
   }
 
   onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
+    this._addSnippetToTestErrors(test, result);
     this._reporter.onStepBegin?.(test, result, step);
   }
 

packages/playwright/src/reporters/teleEmitter.ts

@@ -34,6 +34,7 @@ export class TeleReporterEmitter implements ReporterV2 {
   private _messageSink: (message: teleReceiver.JsonEvent) => void;
   private _rootDir!: string;
   private _emitterOptions: TeleReporterEmitterOptions;
+  private _resultKnownErrorsCounts = new Map<string, number>();
   private _resultKnownAttachmentCounts = new Map<string, number>();
   // In case there is blob reporter and UI mode, make sure one does override
   // the id assigned by the other.
@@ -78,6 +79,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       timeout: test.timeout,
       annotations: []
     };
+    this._sendNewErrors(result, test.id);
     this._sendNewAttachments(result, test.id);
     this._messageSink({
       method: 'onTestEnd',
@@ -87,17 +89,20 @@ export class TeleReporterEmitter implements ReporterV2 {
       }
     });
 
-    this._resultKnownAttachmentCounts.delete((result as any)[this._idSymbol]);
+    const resultId = (result as any)[this._idSymbol] as string;
+    this._resultKnownAttachmentCounts.delete(resultId);
+    this._resultKnownErrorsCounts.delete(resultId);
   }
 
   onStepBegin(test: reporterTypes.TestCase, result: reporterTypes.TestResult, step: reporterTypes.TestStep): void {
     (step as any)[this._idSymbol] = createGuid();
+    this._sendNewErrors(result, test.id);
     this._messageSink({
       method: 'onStepBegin',
       params: {
         testId: test.id,
         resultId: (result as any)[this._idSymbol],
-        step: this._serializeStepStart(step)
+        step: this._serializeStepStart(step),
       }
     });
   }
@@ -248,11 +253,26 @@ export class TeleReporterEmitter implements ReporterV2 {
       id: (result as any)[this._idSymbol],
       duration: result.duration,
       status: result.status,
-      errors: result.errors,
       annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
     };
   }
 
+  private _sendNewErrors(result: reporterTypes.TestResult, testId: string) {
+    const resultId = (result as any)[this._idSymbol] as string;
+    const knownErrorCount = this._resultKnownErrorsCounts.get(resultId) ?? 0;
+    if (result.errors.length > knownErrorCount) {
+      this._messageSink({
+        method: 'onTestErrors',
+        params: {
+          testId,
+          resultId: (result as any)[this._idSymbol],
+          errors: result.errors.slice(knownErrorCount),
+        }
+      });
+    }
+    this._resultKnownErrorsCounts.set(resultId, result.errors.length);
+  }
+
   private _sendNewAttachments(result: reporterTypes.TestResult, testId: string) {
     const resultId = (result as any)[this._idSymbol] as string;
     // Track whether this step (or something else since the last step) has added attachments and send them

packages/playwright/src/runner/dispatcher.ts

@@ -28,7 +28,7 @@ import type { ProcessExitData } from './processHost';
 import type { TestGroup } from './testGroups';
 import type { TestError, TestResult, TestStep } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
-import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
+import type { AttachmentPayload, DonePayload, RunPayload, SerializedConfig, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestBeginPayload, TestEndPayload, TestErrorPayload, TestOutputPayload, TestPausedPayload } from '../common/ipc';
 import type { Suite } from '../common/test';
 import type { TestCase } from '../common/test';
 import type { ReporterV2 } from '../reporters/reporterV2';
@@ -322,7 +322,6 @@ class JobDispatcher {
       // Do not show more than one error to avoid confusion, but report
       // as interrupted to indicate that we did actually start the test.
       params.status = 'interrupted';
-      params.errors = [];
     }
     const data = this._dataByTestId.get(params.testId);
     if (!data) {
@@ -333,8 +332,6 @@ class JobDispatcher {
     this._remainingByTestId.delete(params.testId);
     const { result, test } = data;
     result.duration = params.duration;
-    result.errors = params.errors;
-    result.error = result.errors[0];
     result.status = params.status;
     result.annotations = params.annotations;
     test.annotations = [...params.annotations]; // last test result wins
@@ -429,6 +426,17 @@ class JobDispatcher {
     }
   }
 
+  private _onErrors(params: TestErrorPayload) {
+    const data = this._dataByTestId.get(params.testId)!;
+    if (!data)
+      return;
+    const { result } = data;
+    for (const error of params.errors)
+      addLocationAndSnippetToError(this._config.config, error);
+    result.errors.push(...params.errors);
+    result.error = result.errors[0];
+  }
+
   private _failTestWithErrors(test: TestCase, errors: TestError[]) {
     const runData = this._dataByTestId.get(test.id);
     // There might be a single test that has started but has not finished yet.
@@ -439,7 +447,7 @@ class JobDispatcher {
       result = test._appendTestResult();
       this._reporter.onTestBegin?.(test, result);
     }
-    result.errors = [...errors];
+    result.errors.push(...errors);
     result.error = result.errors[0];
     result.status = errors.length ? 'failed' : 'skipped';
     this._reportTestEnd(test, result);
@@ -578,6 +586,7 @@ class JobDispatcher {
       eventsHelper.addEventListener(worker, 'stepBegin', this._onStepBegin.bind(this)),
       eventsHelper.addEventListener(worker, 'stepEnd', this._onStepEnd.bind(this)),
       eventsHelper.addEventListener(worker, 'attach', this._onAttach.bind(this)),
+      eventsHelper.addEventListener(worker, 'errors', this._onErrors.bind(this)),
       eventsHelper.addEventListener(worker, 'testPaused', this._onTestPaused.bind(this, worker)),
       eventsHelper.addEventListener(worker, 'done', this._onDone.bind(this)),
       eventsHelper.addEventListener(worker, 'exit', this.onExit.bind(this)),
@@ -600,9 +609,11 @@ class JobDispatcher {
       }
     };
 
-    for (const error of params.errors)
-      addLocationAndSnippetToError(this._config.config, error);
-    this._failureTracker.onTestPaused?.({ ...params, sendMessage });
+    const data = this._dataByTestId.get(params.testId);
+    if (!data)
+      return;
+
+    this._failureTracker.onTestPaused?.({ errors: data.result.errors, sendMessage });
   }
 
   skipWholeJob(): boolean {

packages/playwright/src/runner/testRunner.ts

@@ -470,7 +470,9 @@ export async function runAllTestsWithConfig(config: FullConfigInternal): Promise
     createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }),
     ...createRunTestsTasks(config),
   ];
-  const status = await runTasks(new TestRun(config, reporter), tasks, config.config.globalTimeout);
+
+  const testRun = new TestRun(config, reporter, { pauseAtEnd: config.configCLIOverrides.debug, pauseOnError: config.configCLIOverrides.debug });
+  const status = await runTasks(testRun, tasks, config.config.globalTimeout);
 
   // Calling process.exit() might truncate large stdout/stderr output.
   // See https://github.com/nodejs/node/issues/6456.

packages/playwright/src/transform/babelBundle.ts

@@ -24,5 +24,5 @@ export type BabelTransformFunction = (code: string, filename: string, isModule:
 export const babelTransform: BabelTransformFunction = require('./babelBundleImpl').babelTransform;
 export type BabelParseFunction = (code: string, filename: string, isModule: boolean) => ParseResult;
 export const babelParse: BabelParseFunction = require('./babelBundleImpl').babelParse;
-export type { NodePath, PluginObj, types as T } from '../../bundles/babel/node_modules/@types/babel__core';
+export type { NodePath, PluginObj, types as T, ParseResult } from '../../bundles/babel/node_modules/@types/babel__core';
 export type { BabelAPI } from '../../bundles/babel/node_modules/@types/babel__helper-plugin-utils';

packages/playwright/src/transform/babelHighlightUtils.ts

@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import { traverse, babelParse, ParseResult, T, types as t } from './babelBundle';
+import type { Location } from '../../types/testReporter';
+
+const astCache = new Map<string, { text: string, ast?: ParseResult }>();
+
+export function pruneAstCaches(fsPathsToRetain: string[]) {
+  const retain = new Set(fsPathsToRetain);
+  for (const key of astCache.keys()) {
+    if (!retain.has(key))
+      astCache.delete(key);
+  }
+}
+
+function getAst(text: string, fsPath: string) {
+  const cached = astCache.get(fsPath);
+  let ast = cached?.ast;
+  if (!cached || cached.text !== text) {
+    try {
+      ast = babelParse(text, path.basename(fsPath), false);
+      astCache.set(fsPath, { text, ast });
+    } catch (e) {
+      astCache.set(fsPath, { text, ast: undefined });
+    }
+  }
+  return ast;
+}
+
+function containsPosition(location: T.SourceLocation, position: Location): boolean {
+  if (position.line < location.start.line || position.line > location.end.line)
+    return false;
+  if (position.line === location.start.line && position.column < location.start.column)
+    return false;
+  if (position.line === location.end.line && position.column > location.end.column)
+    return false;
+  return true;
+}
+
+export function findTestEndPosition(text: string, location: Location): Location | undefined {
+  const ast = getAst(text, location.file);
+  if (!ast)
+    return;
+  let result: Location | undefined;
+  traverse(ast, {
+    enter(path) {
+      if (t.isCallExpression(path.node) && path.node.loc && containsPosition(path.node.loc, location)) {
+        const callNode = path.node;
+        const funcNode = callNode.arguments[callNode.arguments.length - 1];
+        if (callNode.arguments.length >= 2 && t.isFunction(funcNode) && funcNode.body.loc)
+          result = { file: location.file, ...funcNode.body.loc.end };
+      }
+    }
+  });
+  return result;
+}