Add tarball deployments for each build (vercel#62969)

This adds automatic tarball deployments for each build so that we don't
have to trigger a canary release whenever we want to create a full
tarball with all `next-swc` builds.

Example tarball
https://github.com/vercel/next.js/actions/runs/8239762627/job/22534810077#step:9:34

Closes NEXT-2735

.github/workflows/build_and_deploy.yml

@@ -392,6 +392,52 @@ jobs:
           name: wasm-binaries-${{matrix.target}}
           path: packages/next-swc/crates/wasm/pkg-*
 
+  deployTarball:
+    if: ${{ needs.build.outputs.isRelease != 'true' }}
+    name: Deploy tarball
+    runs-on: ubuntu-latest
+    needs:
+      - build
+      - build-wasm
+      - build-native
+    env:
+      VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
+      VERCEL_TEST_TEAM: vtest314-next-e2e-tests
+    steps:
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+      - run: corepack enable
+
+      # https://github.com/actions/virtual-environments/issues/1187
+      - name: tune linux network
+        run: sudo ethtool -K eth0 tx off rx off
+
+      - uses: actions/cache@v4
+        timeout-minutes: 5
+        id: restore-build
+        with:
+          path: ./*
+          key: ${{ github.sha }}-${{ github.run_number }}
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: next-swc-binaries-*
+          merge-multiple: true
+          path: packages/next-swc/native
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wasm-binaries-*
+          merge-multiple: true
+          path: packages/next-swc/crates/wasm
+
+      - run: npm i -g vercel@latest
+
+      - run: node ./scripts/deploy-tarball.js
+
   publishRelease:
     if: ${{ needs.build.outputs.isRelease == 'true' }}
     name: Potentially publish release

scripts/deploy-tarball.js

@@ -0,0 +1,155 @@
+// @ts-check
+const path = require('path')
+const execa = require('execa')
+const fs = require('fs/promises')
+
+const cwd = process.cwd()
+
+async function main() {
+  const deployDir = path.join(cwd, 'files')
+  const publicDir = path.join(deployDir, 'public')
+  await fs.mkdir(publicDir, { recursive: true })
+  await fs.writeFile(
+    path.join(deployDir, 'package.json'),
+    JSON.stringify({
+      name: 'files',
+      dependencies: {},
+      scripts: {
+        build: 'node inject-deploy-url.js',
+      },
+    })
+  )
+  await fs.copyFile(
+    path.join(cwd, 'scripts/inject-deploy-url.js'),
+    path.join(deployDir, 'inject-deploy-url.js')
+  )
+
+  let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
+  let platforms = (await fs.readdir(nativePackagesDir)).filter(
+    (name) => !name.startsWith('.')
+  )
+
+  const optionalDeps = {}
+  const { version } = JSON.parse(
+    await fs.readFile(path.join(cwd, 'lerna.json'), 'utf8')
+  )
+
+  await Promise.all(
+    platforms.map(async (platform) => {
+      let binaryName = `next-swc.${platform}.node`
+      await fs.cp(
+        path.join(cwd, 'packages/next-swc/native', binaryName),
+        path.join(nativePackagesDir, platform, binaryName)
+      )
+      let pkg = JSON.parse(
+        await fs.readFile(
+          path.join(nativePackagesDir, platform, 'package.json'),
+          'utf8'
+        )
+      )
+      pkg.version = version
+      await fs.writeFile(
+        path.join(nativePackagesDir, platform, 'package.json'),
+        JSON.stringify(pkg, null, 2)
+      )
+      const { stdout } = await execa(`npm`, [
+        `pack`,
+        `${path.join(nativePackagesDir, platform)}`,
+      ])
+      process.stdout.write(stdout)
+      const tarballName = stdout.split('\n').pop()?.trim() || ''
+      await fs.rename(
+        path.join(cwd, tarballName),
+        path.join(publicDir, tarballName)
+      )
+      optionalDeps[pkg.name] = `https://DEPLOY_URL/${tarballName}`
+    })
+  )
+
+  const nextPkgJsonPath = path.join(cwd, 'packages/next/package.json')
+  const nextPkg = JSON.parse(await fs.readFile(nextPkgJsonPath, 'utf8'))
+
+  nextPkg.optionalDependencies = optionalDeps
+
+  await fs.writeFile(nextPkgJsonPath, JSON.stringify(nextPkg, null, 2))
+
+  const { stdout: nextPackStdout } = await execa(`npm`, [
+    `pack`,
+    `${path.join(cwd, 'packages/next')}`,
+  ])
+  process.stdout.write(nextPackStdout)
+  const nextTarballName = nextPackStdout.split('\n').pop()?.trim() || ''
+  await fs.rename(
+    path.join(cwd, nextTarballName),
+    path.join(publicDir, nextTarballName)
+  )
+
+  await fs.writeFile(
+    path.join(deployDir, 'vercel.json'),
+    JSON.stringify(
+      {
+        version: 2,
+        rewrites: [
+          {
+            source: '/next.tgz',
+            destination: `/${nextTarballName}`,
+          },
+        ],
+      },
+      null,
+      2
+    )
+  )
+  const vercelConfigDir = path.join(cwd, '.vercel')
+
+  if (process.env.VERCEL_TEST_TOKEN) {
+    await fs.mkdir(vercelConfigDir)
+    await fs.writeFile(
+      path.join(vercelConfigDir, 'auth.json'),
+      JSON.stringify({
+        token: process.env.VERCEL_TEST_TOKEN,
+      })
+    )
+    await fs.writeFile(
+      path.join(vercelConfigDir, 'config.json'),
+      JSON.stringify({})
+    )
+    console.log('wrote config to', vercelConfigDir)
+  }
+
+  const child = execa(
+    'vercel',
+    [
+      '--scope',
+      process.env.VERCEL_TEST_TEAM || '',
+      '--global-config',
+      vercelConfigDir,
+      '-y',
+    ],
+    {
+      cwd: deployDir,
+    }
+  )
+  let deployOutput = ''
+  const handleData = (type) => (chunk) => {
+    process[type].write(chunk)
+
+    // only want stdout since that's where deployment URL
+    // is sent to
+    if (type === 'stdout') {
+      deployOutput += chunk.toString()
+    }
+  }
+  child.stdout?.on('data', handleData('stdout'))
+  child.stderr?.on('data', handleData('stderr'))
+
+  await child
+
+  const deployUrl = deployOutput.trim()
+  console.log(`\n\nNext.js tarball: ${deployUrl.trim()}/next.tgz`)
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

scripts/inject-deploy-url.js

@@ -0,0 +1,75 @@
+const path = require('path')
+const { spawn } = require('child_process')
+const fs = require('fs/promises')
+
+const cwd = process.cwd()
+
+async function main() {
+  const tarballs = await fs.readdir(path.join(cwd, 'public'))
+  const nextTarball = tarballs.find((item) => !item.includes('-swc'))
+
+  await fs.rename(
+    path.join(cwd, 'public', nextTarball),
+    path.join(cwd, nextTarball)
+  )
+
+  await new Promise((resolve, reject) => {
+    const child = spawn('tar', ['-xf', nextTarball], {
+      stdio: 'inherit',
+      shell: true,
+      cwd,
+    })
+
+    child.on('exit', (code) => {
+      if (code) {
+        return reject(`Failed with code ${code}`)
+      }
+      resolve()
+    })
+  })
+
+  const unpackedPackageJson = path.join(cwd, 'package/package.json')
+  const parsedPackageJson = JSON.parse(
+    await fs.readFile(unpackedPackageJson, 'utf8')
+  )
+  const { optionalDependencies } = parsedPackageJson
+
+  for (const key of Object.keys(optionalDependencies)) {
+    optionalDependencies[key] = optionalDependencies[key].replace(
+      'DEPLOY_URL',
+      process.env.VERCEL_URL
+    )
+  }
+
+  await fs.writeFile(
+    unpackedPackageJson,
+    JSON.stringify(parsedPackageJson, null, 2)
+  )
+
+  await fs.unlink(nextTarball)
+
+  await new Promise((resolve, reject) => {
+    const child = spawn('tar', ['-czf', nextTarball, 'package'], {
+      stdio: 'inherit',
+      shell: true,
+      cwd,
+    })
+
+    child.on('exit', (code) => {
+      if (code) {
+        return reject(`Failed with code ${code}`)
+      }
+      resolve()
+    })
+  })
+
+  await fs.rename(
+    path.join(cwd, nextTarball),
+    path.join(cwd, 'public', nextTarball)
+  )
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})