Visual Regression with Playwright (#9007)

* adding playwright

* playwright-vrt

* full page screenshot

* full page screenshots. TOC handling

* screenshotting more URLs

* Delete tests/tests-examples/demo-todo-app.spec.ts

* getting npm scripts working for playwright vrt

* VRT working

* adding a readme for tests

* markdown fix

* other readme fix

* Clarify usage expectations

* Fix 404 for release notes category listing

* copy edits

---------

Co-authored-by: Rachel Whitton <[email protected]>

tests/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+/test-results/
+/playwright-report/
+/playwright-tests/visuals.spec.ts-snapshots/
+/blob-report/
+/playwright/.cache/

tests/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "tests",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@playwright/test": "^1.40.1",
+    "@types/node": "^20.10.5"
+  },
+  "scripts": {
+    "end-to-end:update-live-snapshots": "npx playwright test --update-snapshots",
+    "end-to-end:pull-request-preview": "PLAYWRIGHT_BASE_URL='https://pr-'${GITHUB_PR_NUMBER}'-documentation.appa.pantheon.site' npx playwright test",
+    "end-to-end:local": "PLAYWRIGHT_BASE_URL='http://localhost:8000/' npx playwright test",
+    "end-to-end:compare-pr-to-live": "npm run end-to-end:update-live-snapshots && npm run end-to-end:pull-request-preview",
+    "end-to-end:compare-local-to-live": "npm run end-to-end:update-live-snapshots && npm run end-to-end:local"
+  }
+}

tests/playwright-tests/visuals.spec.ts

@@ -0,0 +1,66 @@
+import { test, expect } from '@playwright/test';
+
+test('home page', async ({ page }) => {
+  await page.goto('/');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('homepage.png', { fullPage: true });
+});
+
+
+test('Get Started - Landing Page', async ({ page }) => {
+  await page.goto('/get-started/');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('get-started.png', { fullPage: true });
+});
+
+
+test('Release Notes - Category Listing', async ({ page }) => {
+  await page.goto('/release-notes/?category=policy');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('release-notes--category.png', { fullPage: true });
+});
+
+test('Release Notes - Single Entry', async ({ page }) => {
+  await page.goto('/release-notes/2024/05/new-relic-agent-update-released');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('release-notes--single-entry.png', { fullPage: true });
+});
+
+test('Terminus Manual - Single Page', async ({ page }) => {
+  await page.goto('/terminus/install');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('terminus-install.png', { fullPage: true });
+});
+
+test('Terminus Manual - Single Command', async ({ page }) => {
+  await page.goto('/terminus/commands/completion');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('terminus-single-command.png', { fullPage: true });
+});
+
+test('Certification Landing Page', async ({ page }) => {
+  await page.goto('/certification');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('certification-landing-page.png', { fullPage: true });
+});
+
+test('Glossary', async ({ page }) => {
+  await page.goto('/glossary');
+  await page.getByText('Accept Cookies').click();
+  await expect(page).toHaveScreenshot('glossary.png', { fullPage: true });
+});
+
+
+test('certification---study-guide', async ({ page }) => {
+  await page.goto('/certification/study-guide');
+  await page.getByText('Accept Cookies').click();
+  await expect(page.locator('#toc')).toBeVisible();
+  await expect(page).toHaveScreenshot('certification---study-guide.png', { fullPage: true } );
+});
+
+test('drupal-cron', async ({ page }) => {
+  await page.goto('/drupal-cron');
+  await page.getByText('Accept Cookies').click();
+  await expect(page.locator('#toc')).toBeVisible();
+  await expect(page).toHaveScreenshot('drupal-cron.png', { fullPage: true });
+});

tests/playwright.config.ts

@@ -0,0 +1,45 @@
+import { defineConfig, devices } from '@playwright/test';
+
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './playwright-tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    video: 'on',
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // TODO, update this url to handle building on the main branch.
+    baseURL: process.env.PLAYWRIGHT_BASE_URL || "https://docs.pantheon.io",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+  ],
+});

tests/readme.md

@@ -0,0 +1,26 @@
+# Tests for Pantheon Documentation
+
+This directory presently contains only a small amount of Playwright tests meant for locally checking for visual regression. In the future we intend to expand this to include more comprehensive tests and to consolidate pre-existing stale tests like link checking.
+
+## Checking for visual regression
+
+For any changes to the site framework (e.g., stylesheets, dependency updates, theme files, components, javascript, etc.) we want to ensure that the changes do not introduce **unwanted** visual regressions. We can do this by comparing screenshots of the live site to screenshots of a pull request preview or a local development site. Running this suite against a pull request preview depends on the environment variable `GITHUB_PR_NUMBER` being set to the URL of the preview.
+
+To set this variable for the current shell session, run `export GITHUB_PR_NUMBER=<PR_NUMBER>`, where `<PR_NUMBER>` is the number of the pull request.
+
+The following commands are available with `npm run`:
+
+* `end-to-end:compare-pr-to-live`: This command will compare the PR preview to the live site. It is a wrapper around two other commands(`end-to-end:update-live-snapshots` and `end-to-end:pull-request-preview`) that you may wish to run separately for faster execution.
+* `end-to-end:update-live-snapshots`: This command runs the tests against the live site and updates the screenshot snapshots.
+* `end-to-end:pull-request-preview`: This command runs the tests against the PR preview site.
+* `end-to-end:local`: This command runs the tests against a local development site. It requires the Docs site to be running locally via `npm run develop` in the root directory.
+* `end-to-end:compare-local-to-live`: Wraps both `end-to-end:update-live-snapshots` and `end-to-end:local` to compare the local site to the live site.
+
+Use the above commands in a local clone of this project, after navigating to the `tests` subdirectory, for example:
+
+```
+╭─rachelwhitton in [~/documentation/tests] on git:playwright-vrt ✔︎
+╰─➤ $ export GITHUB_PR_NUMBER=9007 && npm run end-to-end:compare-pr-to-live
+```
+
+A new tab will open automatically in your browser with the test results, which are to be reviewed and summarized in a Pull Request review comment for the given change ([example](https://github.com/pantheon-systems/documentation/pull/9036#pullrequestreview-2108883082)). Some failures have been observed as false negatives and can be safely disregarded, for example embedded videos on landing pages.