Rename turbopack-tests-manifest to turbopack-dev-tests-manifest (verc…

…el#63409)

## What?

- Renames the Turbopack tests manifest to reflect that it only holds
development tests.
- Creates initial plumbing for Turbopack build tests manifest (currently
empty)
- Added running tests in the Turbopack builds test manifest on PRs
- Implements uploading the Turbopack builds manifest to areweturboyet

What this doesn't implement:
- Updating the Turbopack builds manifest

Open questions:
- Since the manifest is empty there are no test results, I had to add
handling for that in `run-tests.js`:
https://github.com/vercel/next.js/pull/63409/files#diff-269567847b110d8ecaaade3ab592df207bba02c7b446db23d72f18ff0e61d335R359
but not sure if that exit case was added for a specific special reason.

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->


Closes NEXT-2837

.github/CODEOWNERS

@@ -41,5 +41,5 @@ Cargo.toml                                                       @timneutkens @i
 Cargo.lock                                                       @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
 /.cargo/config.toml                                              @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
 /.config/nextest.toml                                            @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
-/test/build-turbopack-tests-manifest.js                          @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
-/test/turbopack-tests-manifest.json                              @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
+/test/build-turbopack-dev-tests-manifest.js                          @timneutkens @ijjk @shuding @huozhi @vercel/turbopack
+/test/turbopack-dev-tests-manifest.json                              @timneutkens @ijjk @shuding @huozhi @vercel/turbopack

.github/actions/upload-turboyet-data/src/main.js

@@ -3,22 +3,64 @@ const path = require('path')
 
 const { createClient } = require('@vercel/kv')
 
-async function main() {
-  try {
-    const file = path.join(process.cwd(), 'test/turbopack-tests-manifest.json')
+async function collectResults(manifestFile) {
+  const file = path.join(process.cwd(), manifestFile)
+
+  let passingTests = ''
+  let failingTests = ''
+  let passCount = 0
+  let failCount = 0
+
+  const contents = await fs.readFile(file, 'utf-8')
+  const results = JSON.parse(contents)
+
+  const currentDate = new Date()
+  const isoString = currentDate.toISOString()
+  const timestamp = isoString.slice(0, 19).replace('T', ' ')
+
+  if (results.version === 2) {
+    for (const [testFileName, result] of Object.entries(results.suites)) {
+      let suitePassCount = 0
+      let suiteFailCount = 0
+
+      suitePassCount += result.passed.length
+      suiteFailCount += result.failed.length
+
+      if (suitePassCount > 0) {
+        passingTests += `${testFileName}\n`
+      }
+
+      if (suiteFailCount > 0) {
+        failingTests += `${testFileName}\n`
+      }
+
+      for (const passed of result.passed) {
+        const passedName = passed.replaceAll('`', '\\`')
+        passingTests += `* ${passedName}\n`
+      }
+
+      for (const passed of result.failed) {
+        const failedName = passed.replaceAll('`', '\\`')
+        failingTests += `* ${failedName}\n`
+      }
 
-    let passingTests = ''
-    let failingTests = ''
-    let passCount = 0
-    let failCount = 0
+      passCount += suitePassCount
+      failCount += suiteFailCount
 
-    const contents = await fs.readFile(file, 'utf-8')
-    const results = JSON.parse(contents)
+      if (suitePassCount > 0) {
+        passingTests += `\n`
+      }
 
-    const currentDate = new Date()
-    const isoString = currentDate.toISOString()
-    const timestamp = isoString.slice(0, 19).replace('T', ' ')
+      if (suiteFailCount > 0) {
+        failingTests += `\n`
+      }
+    }
 
+    const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${
+      passCount + failCount
+    }`
+    return { testRun, passingTests, failingTests }
+  } else {
     for (const [testFileName, result] of Object.entries(results)) {
       let suitePassCount = 0
       let suiteFailCount = 0
@@ -55,26 +97,45 @@ async function main() {
         failingTests += `\n`
       }
     }
+    const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${
+      passCount + failCount
+    }`
+
+    return { testRun, passingTests, failingTests }
+  }
+}
+
+async function main() {
+  try {
+    const developmentResult = await collectResults(
+      'test/turbopack-dev-tests-manifest.json'
+    )
+
+    const productionResult = await collectResults(
+      'test/turbopack-build-tests-manifest.json'
+    )
 
     const kv = createClient({
       url: process.env.TURBOYET_KV_REST_API_URL,
       token: process.env.TURBOYET_KV_REST_API_TOKEN,
     })
 
-    const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${
-      passCount + failCount
-    }`
+    console.log('TEST RESULT DEVELOPMENT')
+    console.log(developmentResult.testRun)
 
-    console.log('TEST RESULT')
-    console.log(testRun)
+    console.log('TEST RESULT PRODUCTION')
+    console.log(productionResult.testRun)
 
-    await kv.rpush('test-runs', testRun)
+    await kv.rpush('test-runs', developmentResult.testRun)
+    await kv.rpush('test-runs-production', productionResult.testRun)
     console.log('SUCCESSFULLY SAVED RUNS')
 
-    await kv.set('passing-tests', passingTests)
+    await kv.set('passing-tests', developmentResult.passingTests)
+    await kv.set('passing-tests-production', productionResult.passingTests)
     console.log('SUCCESSFULLY SAVED PASSING')
 
-    await kv.set('failing-tests', failingTests)
+    await kv.set('failing-tests', developmentResult.failingTests)
+    await kv.set('failing-tests-production', productionResult.failingTests)
     console.log('SUCCESSFULLY SAVED FAILING')
   } catch (error) {
     console.log(error)

.github/workflows/build_and_test.yml

@@ -163,12 +163,12 @@ jobs:
         group: [1/5, 2/5, 3/5, 4/5, 5/5]
     uses: ./.github/workflows/build_reusable.yml
     with:
-      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY}
+      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-dev-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY}
       stepName: 'test-turbopack-dev-${{ matrix.group }}'
     secrets: inherit
 
   test-turbopack-integration:
-    name: test turbopack integration
+    name: test turbopack development integration
     needs: ['changes', 'build-next']
     if: ${{ needs.changes.outputs.docs-only == 'false' }}
 
@@ -179,10 +179,42 @@ jobs:
     uses: ./.github/workflows/build_reusable.yml
     with:
       nodeVersion: 18.17.0
-      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration
+      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-dev-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration
       stepName: 'test-turbopack-integration-${{ matrix.group }}'
     secrets: inherit
 
+  test-turbopack-production:
+    name: test turbopack production
+    needs: ['changes', 'build-next']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1/5, 2/5, 3/5, 4/5, 5/5]
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      nodeVersion: 18.17.0
+      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-build-tests-manifest.json" TURBOPACK=1 TURBOPACK_BUILD=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type start
+      stepName: 'test-turbopack-production-${{ matrix.group }}'
+    secrets: inherit
+
+  test-turbopack-production-integration:
+    name: test turbopack production integration
+    needs: ['changes', 'build-next']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1/5, 2/5, 3/5, 4/5, 5/5]
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      nodeVersion: 18.17.0
+      afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-build-tests-manifest.json" TURBOPACK=1 TURBOPACK_BUILD=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration
+      stepName: 'test-turbopack-production-integration-${{ matrix.group }}'
+    secrets: inherit
+
   test-next-swc-wasm:
     name: test next-swc wasm
     needs: ['changes', 'build-next']
@@ -339,7 +371,7 @@ jobs:
       stepName: 'test-ppr-prod-${{ matrix.group }}'
     secrets: inherit
 
-  report-test-results:
+  report-test-results-to-datadog:
     needs:
       [
         'changes',
@@ -352,6 +384,8 @@ jobs:
         'test-ppr-integration',
         'test-turbopack-dev',
         'test-turbopack-integration',
+        'test-turbopack-production',
+        'test-turbopack-production-integration',
       ]
     if: ${{ always() && needs.changes.outputs.docs-only == 'false' && !github.event.pull_request.head.repo.fork }}
 

.github/workflows/turbopack-update-tests-manifest.yml

@@ -38,6 +38,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN_PULL_REQUESTS }}
           BRANCH_NAME: turbopack-manifest
-          SCRIPT: test/build-turbopack-tests-manifest.js
+          SCRIPT: test/build-turbopack-dev-tests-manifest.js
           PR_TITLE: Update Turbopack test manifest
           PR_BODY: This auto-generated PR updates the integration test manifest used when testing Turbopack.

.github/workflows/turbopack-upload-tests-manifest.yml

@@ -29,7 +29,7 @@ jobs:
 
       # Always run build manifest script to get the latest value
       - run: |
-          node ./test/build-turbopack-tests-manifest.js
+          node ./test/build-turbopack-dev-tests-manifest.js
         env:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN_PULL_REQUESTS }}
 

run-tests.js

@@ -356,9 +356,12 @@ async function main() {
     }
   }
 
+  if (!tests) {
+    tests = []
+  }
+
   if (tests.length === 0) {
     console.log('No tests found for', options.type, 'exiting..')
-    return cleanUpAndExit(1)
   }
 
   console.log(`${GROUP}Running tests:

test/build-turbopack-tests-manifest.js → test/build-turbopack-dev-tests-manifest.js

@@ -13,7 +13,7 @@ async function format(text) {
 
 const override = process.argv.includes('--override')
 
-const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json`
+const PASSING_JSON_PATH = `${__dirname}/turbopack-dev-tests-manifest.json`
 const WORKING_PATH = '/root/actions-runner/_work/next.js/next.js/'
 
 const INITIALIZING_TEST_CASES = [

test/turbopack-build-tests-manifest.json

@@ -0,0 +1,8 @@
+{
+  "version": 2,
+  "suites": {},
+  "rules": {
+    "include": [],
+    "exclude": []
+  }
+}