Add PPR evals and fix env loading for agent-eval 0.9.5

.gitignore

@@ -50,6 +50,7 @@ examples/**/out/*
 
 # env
 .env*.local
+evals/.env
 
 pr-stats.md
 test-timings.json

evals/evals/agent-041-optimize-ppr-shell/EVAL.ts

@@ -0,0 +1,61 @@
+/**
+ * Optimize PPR Shell
+ *
+ * Tests whether the agent decomposes a monolithic loading.tsx (which creates
+ * a single implicit Suspense boundary around the entire page) into granular
+ * Suspense boundaries — one per dashboard section — so each section can
+ * stream independently and the PPR shell contains more static content.
+ *
+ * Tricky because the starting code uses Next.js's loading.tsx convention,
+ * which is an implicit Suspense boundary. Agents need to recognize that
+ * loading.tsx creates an all-or-nothing loading state, and that optimizing
+ * the PPR shell requires replacing it with per-section Suspense boundaries
+ * so each section can stream independently.
+ */
+
+import { expect, test } from 'vitest'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+const appDir = join(process.cwd(), 'app')
+
+function readFile(name: string): string {
+  return readFileSync(join(appDir, name), 'utf-8')
+}
+
+test('Page has at least 3 Suspense boundaries', () => {
+  const page = readFile('page.tsx')
+
+  const suspenseCount = (page.match(/<Suspense[\s>]/g) || []).length
+  expect(suspenseCount).toBeGreaterThanOrEqual(3)
+})
+
+test('Each dashboard section has its own Suspense boundary in page.tsx', () => {
+  const page = readFile('page.tsx')
+
+  // Split page into Suspense blocks: text between each <Suspense and </Suspense>
+  const suspenseBlocks = page.split(/<Suspense[\s>]/).slice(1)
+
+  const components = ['CardStats', 'RevenueChart', 'LatestInvoices']
+  for (const component of components) {
+    const inOwnBlock = suspenseBlocks.some(
+      (block) => block.includes(component) && block.includes('</Suspense>')
+    )
+    expect(inOwnBlock, `${component} should be inside its own <Suspense>`).toBe(
+      true
+    )
+  }
+})
+
+test('Page does not await all data before rendering', () => {
+  const page = readFile('page.tsx')
+
+  // The page should not call getDashboardData() or fetch() at the top level.
+  // A simple check: the page shouldn't contain the original monolithic fetch.
+  expect(page).not.toMatch(/await\s+getDashboardData\s*\(/)
+
+  // The page component itself should not be async (data fetching moves into children)
+  // OR if it is async, it should not await a data fetch before returning JSX.
+  // We check the simpler signal: getDashboardData should not be called in page.tsx at all.
+  expect(page).not.toMatch(/getDashboardData\s*\(/)
+})

evals/evals/agent-041-optimize-ppr-shell/PROMPT.md

@@ -0,0 +1 @@
+Optimize the partial pre-rendering shell for this app.

evals/evals/agent-041-optimize-ppr-shell/app/CardStats.tsx

@@ -0,0 +1,20 @@
+export function CardStats({
+  totalRevenue,
+  totalInvoices,
+}: {
+  totalRevenue: number
+  totalInvoices: number
+}) {
+  return (
+    <div className="grid grid-cols-2 gap-4">
+      <div className="card">
+        <h2>Total Revenue</h2>
+        <p>${totalRevenue.toLocaleString()}</p>
+      </div>
+      <div className="card">
+        <h2>Total Invoices</h2>
+        <p>{totalInvoices}</p>
+      </div>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/LatestInvoices.tsx

@@ -0,0 +1,18 @@
+export function LatestInvoices({
+  invoices,
+}: {
+  invoices: { id: string; name: string; amount: number }[]
+}) {
+  return (
+    <div className="invoices">
+      <h2>Latest Invoices</h2>
+      <ul>
+        {invoices.map((invoice) => (
+          <li key={invoice.id}>
+            {invoice.name} - ${invoice.amount.toLocaleString()}
+          </li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/RevenueChart.tsx

@@ -0,0 +1,18 @@
+export function RevenueChart({
+  revenue,
+}: {
+  revenue: { month: string; amount: number }[]
+}) {
+  return (
+    <div className="chart">
+      <h2>Revenue</h2>
+      <ul>
+        {revenue.map((item) => (
+          <li key={item.month}>
+            {item.month}: ${item.amount.toLocaleString()}
+          </li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/layout.tsx

@@ -0,0 +1,18 @@
+import type { Metadata } from 'next'
+
+export const metadata: Metadata = {
+  title: 'Create Next App',
+  description: 'Generated by create next app',
+}
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/loading.tsx

@@ -0,0 +1,8 @@
+export default function Loading() {
+  return (
+    <div className="loading">
+      <div className="spinner" />
+      <p>Loading dashboard...</p>
+    </div>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/app/page.tsx

@@ -0,0 +1,24 @@
+import { RevenueChart } from './RevenueChart'
+import { LatestInvoices } from './LatestInvoices'
+import { CardStats } from './CardStats'
+
+async function getDashboardData() {
+  const res = await fetch('https://api.example.com/dashboard')
+  return res.json()
+}
+
+export default async function Page() {
+  const data = await getDashboardData()
+
+  return (
+    <main>
+      <h1>Dashboard</h1>
+      <CardStats
+        totalRevenue={data.totalRevenue}
+        totalInvoices={data.totalInvoices}
+      />
+      <RevenueChart revenue={data.revenue} />
+      <LatestInvoices invoices={data.invoices} />
+    </main>
+  )
+}

evals/evals/agent-041-optimize-ppr-shell/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from 'next'
+
+const nextConfig: NextConfig = {
+  cacheComponents: true,
+}
+
+export default nextConfig

evals/evals/agent-041-optimize-ppr-shell/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^16",
+    "react": "19.1.0",
+    "react-dom": "19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^5",
+    "vitest": "^3.1.3",
+    "@vitejs/plugin-react": "^4.4.1",
+    "vite-tsconfig-paths": "^5.1.4"
+  }
+}

evals/evals/agent-041-optimize-ppr-shell/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

evals/evals/agent-042-enable-ppr/EVAL.ts

@@ -0,0 +1,31 @@
+/**
+ * Enable PPR
+ *
+ * Tests whether the agent knows that partial pre-rendering is enabled via
+ * `cacheComponents: true` in next.config.ts, NOT the old
+ * `experimental: { ppr: true }` flag.
+ *
+ * Tricky because most training data and older docs reference the experimental
+ * flag. The current way to enable PPR in Next.js 16 is `cacheComponents: true`.
+ */
+
+import { expect, test } from 'vitest'
+import { readFileSync } from 'fs'
+import { join } from 'path'
+
+test('Enables PPR via cacheComponents in next.config', () => {
+  const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8')
+
+  expect(config).toMatch(/cacheComponents\s*:\s*true/)
+})
+
+test('Does not use the old experimental.ppr flag', () => {
+  const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8')
+
+  // Strip comments to avoid false positives from explanatory comments
+  const stripped = config
+    .replace(/\/\*[\s\S]*?\*\//g, '')
+    .replace(/\/\/.*$/gm, '')
+
+  expect(stripped).not.toMatch(/ppr\s*:\s*true/)
+})

evals/evals/agent-042-enable-ppr/PROMPT.md

@@ -0,0 +1 @@
+Enable partial pre-rendering for this app.

evals/evals/agent-042-enable-ppr/app/ProductList.tsx

@@ -0,0 +1,18 @@
+async function getProducts() {
+  const res = await fetch('https://api.example.com/products')
+  return res.json()
+}
+
+export async function ProductList() {
+  const products = await getProducts()
+
+  return (
+    <ul>
+      {products.map((p: { id: string; name: string; price: number }) => (
+        <li key={p.id}>
+          {p.name} - ${p.price}
+        </li>
+      ))}
+    </ul>
+  )
+}

evals/evals/agent-042-enable-ppr/app/Recommendations.tsx

@@ -0,0 +1,19 @@
+async function getRecommendations() {
+  const res = await fetch('https://api.example.com/recommendations')
+  return res.json()
+}
+
+export async function Recommendations() {
+  const items = await getRecommendations()
+
+  return (
+    <div>
+      <h2>Recommended for you</h2>
+      <ul>
+        {items.map((item: { id: string; name: string }) => (
+          <li key={item.id}>{item.name}</li>
+        ))}
+      </ul>
+    </div>
+  )
+}

evals/evals/agent-042-enable-ppr/app/layout.tsx

@@ -0,0 +1,18 @@
+import type { Metadata } from 'next'
+
+export const metadata: Metadata = {
+  title: 'Create Next App',
+  description: 'Generated by create next app',
+}
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode
+}>) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

evals/evals/agent-042-enable-ppr/app/page.tsx

@@ -0,0 +1,17 @@
+import { Suspense } from 'react'
+import { ProductList } from './ProductList'
+import { Recommendations } from './Recommendations'
+
+export default function Page() {
+  return (
+    <main>
+      <h1>Store</h1>
+      <Suspense fallback={<p>Loading products...</p>}>
+        <ProductList />
+      </Suspense>
+      <Suspense fallback={<p>Loading recommendations...</p>}>
+        <Recommendations />
+      </Suspense>
+    </main>
+  )
+}

evals/evals/agent-042-enable-ppr/next.config.ts

@@ -0,0 +1,5 @@
+import type { NextConfig } from 'next'
+
+const nextConfig: NextConfig = {}
+
+export default nextConfig

evals/evals/agent-042-enable-ppr/package.json

@@ -0,0 +1,23 @@
+{
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "^16",
+    "react": "19.1.0",
+    "react-dom": "19.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "typescript": "^5",
+    "vitest": "^3.1.3",
+    "@vitejs/plugin-react": "^4.4.1",
+    "vite-tsconfig-paths": "^5.1.4"
+  }
+}