UI: visual tests better (#956)

* UI: visuail tests initial

* add Dashboards screenshots

* addes snapshots images

* increased maxDiffPixels threshold

* remove logging

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,4 @@
+[core]
+    remote = gs
+['remote "gs"']
+    url = gs://evidently-service-visual-testing

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/workflows/ui.yml

@@ -75,9 +75,15 @@ jobs:
           architecture: "x64"
           cache: "pip"
 
-      - name: Install python dependencies
+      - name: Install Evidently
         run: pip install -e .
 
+      - name: Install dvc
+        run: pip install dvc[gs]
+
+      - name: Pull data
+        run: dvc pull
+
       - uses: pnpm/action-setup@v2
         with:
           version: 8
@@ -98,11 +104,11 @@ jobs:
         run: pnpm dlx [email protected] install --with-deps
 
       - name: Run UI
-        run: evidently ui --port 8000 --demo-projects bikes,reviews --workspace test-workspace &
+        run: evidently ui --port 8000 --workspace workspace-for-visual-testing &
 
       - name: Wait UI to be ready to test
         working-directory: ui/service
-        run: pnpm wait-on tcp:127.0.0.1:8000 -t 300000
+        run: pnpm wait-on tcp:127.0.0.1:8000 -t 10000
 
       - name: Run Service Playwright tests
         working-directory: ui/service

.gitignore

@@ -20,3 +20,4 @@ __pycache__
 # see .devcontainer/devcontainer.json
 test-workspace
 workspace
+workspace-for-visual-testing

ui/service/tests/smoke.spec.ts

@@ -107,11 +107,11 @@ test('Filter Reports and Test Suites by tags', async ({ page }) => {
 
     const rowsNumberAfterFiltration = await page.getByRole('row').count()
 
-    console.log(
-      'rowsNumberBeforeFiltration, rowsNumberAfterFiltration',
-      rowsNumberBeforeFiltration,
-      rowsNumberAfterFiltration
-    )
+    // console.log(
+    //   'rowsNumberBeforeFiltration, rowsNumberAfterFiltration',
+    //   rowsNumberBeforeFiltration,
+    //   rowsNumberAfterFiltration
+    // )
 
     expect(rowsNumberBeforeFiltration).toBeGreaterThan(rowsNumberAfterFiltration)
   }

ui/service/tests/visual.spec.ts

@@ -9,6 +9,7 @@ const goToFirstSnapshotAndExpanSomeWidgets = async ({
   isTestSuite: boolean
   projectName: string
 }) => {
+  await page.goto('/')
   await page.getByRole('link', { name: projectName }).click()
   await page.getByRole('tab', { name: isTestSuite ? 'Test Suites' : 'Reports' }).click()
   await page.getByRole('button', { name: 'View' }).first().click()
@@ -44,48 +45,102 @@ const VisualTestSnapshot = async ({
   projectName: string
   isTestSuite: boolean
 }) => {
-  await page.goto('/')
   await goToFirstSnapshotAndExpanSomeWidgets({
     page,
     projectName,
     isTestSuite
   })
 
-  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 1800 })
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 150 })
+}
+
+const goToSnapshotsList = async ({
+  projectName,
+  page,
+  isTestSuite
+}: {
+  page: Page
+  isTestSuite: boolean
+  projectName: string
+}) => {
+  await page.goto('/')
+  await page.getByRole('link', { name: projectName }).click()
+  await page.getByRole('tab', { name: isTestSuite ? 'Test Suites' : 'Reports' }).click()
+  await page.waitForLoadState('networkidle')
+}
+
+const VisualTestSnapshotsList = async ({
+  page,
+  projectName,
+  isTestSuite
+}: {
+  page: Page
+  projectName: string
+  isTestSuite: boolean
+}) => {
+  await goToSnapshotsList({
+    page,
+    projectName,
+    isTestSuite
+  })
+
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 150 })
 }
 
 const VisualTestDashboard = async ({ page, projectName }: { page: Page; projectName: string }) => {
   await page.goto('/')
   await page.getByRole('link', { name: projectName }).click()
   await page.waitForLoadState('networkidle')
-  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 800 })
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 150 })
 }
 
 const BikesDemoProjectName = 'Demo project - Bikes'
 const ReviewsDemoProjectName = 'Demo project - Reviews'
 
 /////////////////////
-///   Dashboards
+///   Home
 /////////////////////
-test(`${BikesDemoProjectName}: Dashboard`, async ({ page }) => {
-  await VisualTestDashboard({ page, projectName: BikesDemoProjectName })
-})
 
-test(`${ReviewsDemoProjectName}: Dashboard`, async ({ page }) => {
-  await VisualTestDashboard({ page, projectName: ReviewsDemoProjectName })
+test(`Home`, async ({ page }) => {
+  await page.goto('/')
+  await page.waitForLoadState('networkidle')
+  await expect(page).toHaveScreenshot({ fullPage: true, maxDiffPixels: 150 })
 })
 
+/////////////////////
+///  Shapshots List
+/////////////////////
+
+for (const project of [BikesDemoProjectName]) {
+  test(`Reports List: ${project}`, async ({ page }) => {
+    await VisualTestSnapshotsList({ page, projectName: project, isTestSuite: false })
+  })
+
+  test(`Test Suites List: ${project}`, async ({ page }) => {
+    await VisualTestSnapshotsList({ page, projectName: project, isTestSuite: true })
+  })
+}
+
+/////////////////////
+///   Dashboards
+/////////////////////
+
+for (const project of [BikesDemoProjectName, ReviewsDemoProjectName]) {
+  test(`Dashboard: ${project}`, async ({ page }) => {
+    await VisualTestDashboard({ page, projectName: project })
+  })
+}
+
 /////////////////////
 ///   Snapshots
 /////////////////////
-test(`${BikesDemoProjectName}: Report`, async ({ page }) => {
-  await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: false })
-})
 
-test(`${BikesDemoProjectName}: Test Suite`, async ({ page }) => {
-  await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: true })
-})
+for (const project of [BikesDemoProjectName, ReviewsDemoProjectName]) {
+  test(`Report: ${project}`, async ({ page }) => {
+    await VisualTestSnapshot({ page, projectName: project, isTestSuite: false })
+  })
+}
 
-test(`${ReviewsDemoProjectName}: Report`, async ({ page }) => {
-  await VisualTestSnapshot({ page, projectName: ReviewsDemoProjectName, isTestSuite: false })
+test(`Test Suite: ${BikesDemoProjectName}`, async ({ page }) => {
+  await VisualTestSnapshot({ page, projectName: BikesDemoProjectName, isTestSuite: true })
 })

workspace-for-visual-testing.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: 9136a5067f431ded98869250d0476667.dir
+  size: 9283262
+  nfiles: 97
+  hash: md5
+  path: workspace-for-visual-testing