split scenarios into scenarios and addons

.github/workflows/compare.yml

@@ -22,19 +22,19 @@ jobs:
         current:
           - "${{ github.event.inputs.current }}"
         case:
-          - minimal
           - common-libs
           - esbuild-three
         scenario:
-          - development-default-build
-          - development-cached-build
-          - development-cached-pnp-build
-          - development-cached-rebuild
-          - production-default-build
-          - production-cached-build
-          - production-source-map-build
-          - production-source-map-cached-build
-    runs-on: windows-latest
+          - development-build
+          - development-build+persistent-cache
+          - development-build+pnp
+          - development-rebuild
+          - production-build
+          - production-build+no-minimize
+          - production-build+no-minimize+no-concatenation
+          - production-build+persistent-cache
+          - production-build+source-map+persistent-cache
+    runs-on: linux-latest
     steps:
       - run: echo ${{ github.event.inputs.base }} vs ${{ github.event.inputs.current }}
       - uses: actions/checkout@v2

addons/babel-env.js

@@ -0,0 +1,29 @@
+export const packageJson = (json) => {
+	json.browserslist = "> 0.25%, not dead";
+	Object.assign(json.dependencies, {
+		"@babel/core": "^7.12.16",
+		"@babel/preset-env": "^7.12.16",
+		"babel-loader": "^8.2.2",
+		"core-js": "^3.8.3",
+	});
+	return json;
+};
+
+export const config = (content) => `${content}
+
+module.exports.module = module.exports.module || {};
+module.exports.module.rules = module.exports.module.rules || [];
+module.exports.module.rules.push({
+	test: /\.js$/,
+	use: {
+		loader: "babel-loader",
+		options: {
+			sourceType: "unambiguous",
+			presets: [["@babel/env", {
+				useBuiltIns: "usage",
+				corejs: 3
+			}]]
+		}
+	}
+});
+`;

addons/no-concatenation.js

@@ -0,0 +1 @@
+export const args = ["--no-optimization-concatenate-modules"];

addons/no-minimize.js

@@ -0,0 +1 @@
+export const args = ["--no-optimization-minimize"];

addons/persistent-cache.js

@@ -0,0 +1 @@
+export const args = ["--cache-type", "filesystem"];

addons/pnp.js

@@ -0,0 +1 @@
+export const installOptions = { pnp: true };

addons/source-map.js

@@ -0,0 +1 @@
+export const args = ["--devtool", "source-map"];

bin/compare-scenarios.js

@@ -0,0 +1,34 @@
+import compare from "../lib/compare.js";
+import { formatDiffTable, formatResultTable } from "../lib/utils.js";
+import { mkdir, writeFile } from "fs/promises";
+import { resolve } from "path";
+import { fileURLToPath } from "url";
+
+const [
+	,
+	,
+	caseName = "minimal",
+	baseline = "development-default-build",
+	scenarioName = "development-cached-build",
+] = process.argv;
+
+const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
+
+(async () => {
+	const { diff, result } = await compare(caseName, scenarioName, {
+		runs: 30,
+		verboseSetup: true,
+		baselineScenario: baseline,
+	});
+	console.log(formatResultTable(result, { colors: true, verbose: true }));
+	console.log();
+	console.log(formatDiffTable(diff, { colors: true, verbose: true }));
+	await mkdir(resolve(rootDir, "output"), { recursive: true });
+	await writeFile(
+		resolve(rootDir, `output/${caseName}.json`),
+		JSON.stringify(diff, null, 2)
+	);
+})().catch((err) => {
+	process.exitCode = 1;
+	console.error(err.stack);
+});

bin/compare.js

@@ -1,5 +1,5 @@
 import compare from "../lib/compare.js";
-import { formatDiffTable } from "../lib/utils.js";
+import { formatDiffTable, formatResultTable } from "../lib/utils.js";
 import { mkdir, writeFile } from "fs/promises";
 import { resolve } from "path";
 import { fileURLToPath } from "url";
@@ -16,7 +16,7 @@ const [
 const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
 
 (async () => {
-	const diff = await compare(caseName, scenarioName, {
+	const { diff, result } = await compare(caseName, scenarioName, {
 		runs: 30,
 		verboseSetup: true,
 		baselineDependencies: {
@@ -26,6 +26,8 @@ const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
 			webpack: `webpack/webpack#${current}`,
 		},
 	});
+	console.log(formatResultTable(result, { colors: true, verbose: true }));
+	console.log();
 	console.log(formatDiffTable(diff, { colors: true, verbose: true }));
 	await mkdir(resolve(rootDir, "output"), { recursive: true });
 	await writeFile(

bin/measure-mean.js

@@ -1,4 +1,5 @@
 import measure from "../lib/measure.js";
+import { formatResultTable } from "../lib/utils.js";
 
 const [
 	,
@@ -8,7 +9,9 @@ const [
 ] = process.argv;
 
 (async () => {
-	console.log(
-		await measure(caseName, scenarioName, { runs: 20, verboseSetup: true })
-	);
+	const result = await measure(caseName, scenarioName, {
+		runs: 30,
+		verboseSetup: true,
+	});
+	console.log(formatResultTable(result, { colors: true, verbose: true }));
 })().catch((err) => console.error(err.stack));

cases/common-libs/package.json

@@ -2,9 +2,11 @@
   "name": "common-libs",
   "dependencies": {
     "@babel/runtime": "^7.12.13",
+    "@material-ui/core": "^4.11.3",
+    "@material-ui/icons": "^4.11.2",
+    "@material-ui/lab": "^4.0.0-alpha.57",
     "acorn": "^8.0.5",
     "classnames": "^2.2.6",
-    "core-js": "^3.8.3",
     "date-fns": "^2.17.0",
     "jquery": "^3.5.1",
     "lodash": "^4.17.20",

cases/common-libs/src/index.js

@@ -1,7 +1,10 @@
 import "./babel-runtime";
+import * as core from "@material-ui/core";
+import * as lab from "@material-ui/lab";
+import * as icons from "@material-ui/icons";
+console.log(core, lab, icons);
 import "acorn";
 import "classnames";
-import "core-js";
 import * as dateFn from "date-fns";
 import * as dateFnEsm from "date-fns/esm";
 console.log(dateFn, dateFnEsm);

lib/build.js

@@ -4,9 +4,13 @@ import { alterFile, revertFile } from "./utils.js";
 export default (args) => {
 	const run = async (options) => {
 		let start = Date.now();
+		let currentArgs = args;
+		for (const addon of options.addons) {
+			if (addon.args) currentArgs = currentArgs.concat(addon.args);
+		}
 		const p = spawn(
 			"yarn",
-			["webpack", ...args, ...(options.progress ? ["--progress"] : [])],
+			["webpack", ...currentArgs, ...(options.progress ? ["--progress"] : [])],
 			{
 				shell: true,
 				stdio: options.verbose
@@ -15,14 +19,17 @@ export default (args) => {
 			}
 		);
 		let counter = 0;
+		let dataSetCounter = -1;
 		let promise = Promise.resolve();
 		const data = {};
 		const processLine = (line) => {
 			if (line === "#!# start") {
 				start = Date.now();
+				dataSetCounter = 0;
 			} else if (line === "#!# next") {
 				promise = promise.then(async () => {
 					counter++;
+					if (dataSetCounter >= 0) dataSetCounter++;
 					await new Promise((r) =>
 						setTimeout(r, Math.max(300, 1000 / counter))
 					);
@@ -47,6 +54,11 @@ export default (args) => {
 		data.execTime = Date.now() - start;
 		await promise;
 		if (exitCode !== 0) throw new Error(`Build failed with ${exitCode}`);
+		if (dataSetCounter > 1) {
+			for (const key of Object.keys(data)) {
+				data[key] /= dataSetCounter;
+			}
+		}
 		return data;
 	};
 	const warmup = async (options) => {
@@ -60,6 +72,9 @@ export default (args) => {
 		await alterFile(
 			"webpack.config.js",
 			(content) => {
+				for (const addon of options.addons) {
+					if (addon.config) content = addon.config(content, options);
+				}
 				return `${content}
 
 module.exports.plugins = module.exports.plugins || [];

lib/compare.js

@@ -7,9 +7,11 @@ import { compareStatistics } from "./utils.js";
 const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
 
 const getResult = async (caseName, scenarioName, options) => {
-	const cacheIdentifier = `${caseName}-${scenarioName}-${Buffer.from(
-		JSON.stringify(options.dependencies)
-	).toString("hex")}.json`;
+	const cacheIdentifier = `${caseName}-${scenarioName}-${
+		options.dependencies
+			? Buffer.from(JSON.stringify(options.dependencies)).toString("hex")
+			: ""
+	}.json`;
 	try {
 		return JSON.parse(
 			await readFile(resolve(rootDir, ".cache", cacheIdentifier), "utf-8")
@@ -27,13 +29,16 @@ const getResult = async (caseName, scenarioName, options) => {
 };
 
 export default async (caseName, scenarioName, options) => {
-	const baseline = options.baselineDependencies || {
-		webpack: "latest",
-	};
-	const baselineResult = await getResult(caseName, scenarioName, {
+	const baselineDependencies =
+		options.baselineDependencies || options.dependencies;
+	const baselineScenario = options.baselineScenario || scenarioName;
+	const baselineResult = await getResult(caseName, baselineScenario, {
 		...options,
-		dependencies: baseline,
+		dependencies: baselineDependencies,
 	});
 	const currentResult = await getResult(caseName, scenarioName, options);
-	return compareStatistics(baselineResult, currentResult);
+	return {
+		diff: compareStatistics(baselineResult, currentResult),
+		result: currentResult,
+	};
 };

lib/measure.js

@@ -11,19 +11,26 @@ import {
 
 const rootDir = resolve(fileURLToPath(import.meta.url), "../..");
 
-export default async (caseName, scenarioName, options) => {
+export default async (caseName, scenarioAndAddons, options) => {
+	const [scenarioName, ...addonNames] = scenarioAndAddons.split("+");
 	const casePath = resolve(rootDir, "cases", caseName);
 	const pkg = resolve(casePath, "package.json");
 	const lockfile = resolve(casePath, "yarn.lock");
 	const rootPkg = resolve(rootDir, "package.json");
 	const rootLockfile = resolve(rootDir, "yarn.lock");
 	const scenarioModule = await import(`../scenarios/${scenarioName}.js`);
 	const scenario = { ...scenarioModule.default, ...scenarioModule };
+	const addons = await Promise.all(
+		addonNames.map((name) => import(`../addons/${name}.js`))
+	);
+	options = { ...options, addons };
 	await clearCaches(casePath);
 	let pkgData;
 	await alterFile(pkg, (content) => {
 		pkgData = JSON.parse(content);
-		if (!options.dependencies) return content;
+		for (const addon of addons) {
+			if (addon.packageJson) pkgData = addon.packageJson(pkgData, options);
+		}
 		pkgData = {
 			...pkgData,
 			dependencies: {
@@ -44,21 +51,39 @@ export default async (caseName, scenarioName, options) => {
 	const cwd = process.cwd();
 	try {
 		process.chdir(casePath);
+		const installOptions = {};
+		// addons
+		for (const addon of addons) {
+			if (addon.beforeInstall) await addon.beforeInstall(options);
+			if (addon.installOptions)
+				installOptions = { ...installOptions, ...addon.installOptions };
+		}
+
 		// install
-		await install(scenario.installArguments, options);
+		await install(installOptions, options);
 		await revertFile(rootPkg);
 		await revertFile(rootLockfile);
 
+		// addons
+		for (const addon of addons) {
+			if (addon.beforeSetup) await addon.beforeSetup(options);
+		}
+
 		// setup
 		if (pkgData.scripts && pkgData.scripts["bench:setup"])
 			await runScript("bench:setup", options);
 		if (scenario.setup) await scenario.setup(options);
 
+		// addons
+		for (const addon of addons) {
+			if (addon.setup) await addon.setup(options);
+		}
+
 		// warmup
-		console.log(`${caseName} ${scenarioName} warmup`);
-		console.time(`${caseName} ${scenarioName} warmup`);
+		console.log(`${caseName} ${scenarioAndAddons} warmup`);
+		console.time(`${caseName} ${scenarioAndAddons} warmup`);
 		await scenario.warmup(options);
-		console.timeEnd(`${caseName} ${scenarioName} warmup`);
+		console.timeEnd(`${caseName} ${scenarioAndAddons} warmup`);
 
 		// measure
 		const results = [];
@@ -72,13 +97,17 @@ export default async (caseName, scenarioName, options) => {
 			const runtime = Date.now() - start;
 			const percentage = Math.max(runtime / timeout, (i + 1) / maxRuns);
 			console.log(
-				`${caseName} ${scenarioName} ${Math.round(percentage * 100)}%`
+				`${caseName} ${scenarioAndAddons} ${Math.round(percentage * 100)}%`
 			);
 			if (runtime > timeout) break;
 		}
 		return options.noStatistics ? results[0] : calcStatistics(results);
 	} finally {
 		if (scenario.teardown) await scenario.teardown(options);
+		// addons
+		for (const addon of addons) {
+			if (addon.teardown) await addon.teardown(options);
+		}
 		await revertFile(pkg);
 		await revertFile(lockfile);
 		await revertFile(rootPkg);