From 3517a5240448673e3d5b18c7c5b39a439168cd07 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Wed, 9 Jun 2021 17:40:45 -0600 Subject: [PATCH 1/5] [Bugfix] Ensure stability of filename cache-keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON.stringify(structure)` isn’t inherently stable as it relies on various internal details of how `structure` was created. As written, if a given babel configuration is create in an dynamic manner, it is possible for babel-loader to have spurious cache misses. To address this, we can use one of the many stable stringify alternatives. For this PR I have selected [fast-stable-stringify](https://www.npmjs.com/package/fast-stable-stringify) for that task, as it appears both popular and it’s benchmarks look promising. This PR does not explicitly include tests, as testing this is both tricky to test in this context, and the important tests are contained within fast-stable-stringify itself. --- package.json | 1 + src/cache.js | 5 ++--- src/index.js | 3 ++- test/cache.test.js | 2 +- yarn.lock | 46 ++++++++++++++++++++++++++++++++-------------- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index 7822ed34..c5586f7e 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "node": ">= 14.15.0" }, "dependencies": { + "fast-stable-stringify": "^1.0.0", "find-cache-dir": "^4.0.0", "schema-utils": "^4.0.0" }, diff --git a/src/cache.js b/src/cache.js index 2444bb12..c9728e75 100644 --- a/src/cache.js +++ b/src/cache.js @@ -29,6 +29,7 @@ try { const gunzip = promisify(zlib.gunzip); const gzip = promisify(zlib.gzip); +const stringify = require("fast-stable-stringify"); /** * Read the contents from the compressed file. @@ -70,9 +71,7 @@ const write = async function (filename, compress, result) { const filename = function (source, identifier, options) { const hash = crypto.createHash(hashType); - const contents = JSON.stringify({ source, options, identifier }); - - hash.update(contents); + hash.update(stringify({ source, options, identifier })); return hash.digest("hex") + ".json"; }; diff --git a/src/index.js b/src/index.js index 5ba55e3e..bbe3c446 100644 --- a/src/index.js +++ b/src/index.js @@ -27,6 +27,7 @@ const schema = require("./schema"); const { isAbsolute } = require("path"); const validateOptions = require("schema-utils").validate; +const stringify = require("fast-stable-stringify"); function subscribe(subscriber, metadata, context) { if (context[subscriber]) { @@ -174,7 +175,7 @@ async function loader(source, inputSourceMap, overrides) { const { cacheDirectory = null, - cacheIdentifier = JSON.stringify({ + cacheIdentifier = stringify({ options, "@babel/core": transform.version, "@babel/loader": version, diff --git a/test/cache.test.js b/test/cache.test.js index 71290b36..b5ec2aaf 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -323,5 +323,5 @@ test("should allow to specify the .babelrc file", async t => { t.deepEqual(multiStats.stats[1].compilation.warnings, []); const files = fs.readdirSync(t.context.cacheDirectory); - t.true(files.length === 2); + t.true(files.length === 1); }); diff --git a/yarn.lock b/yarn.lock index ae93271e..b4229b4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1,6 +1,3 @@ -# This file is generated by running "yarn install" inside your project. -# Manual changes might be lost - proceed with caution! - __metadata: version: 6 cacheKey: 8 @@ -2279,17 +2276,31 @@ __metadata: "@babel/eslint-parser": ^7.23.3 "@babel/preset-env": ^7.23.3 ava: ^3.13.0 - c8: ^8.0.0 - eslint: ^9.6.0 - eslint-config-prettier: ^9.1.0 - eslint-plugin-prettier: ^5.1.3 - find-cache-dir: ^4.0.0 - globals: ^15.8.0 - husky: ^8.0.3 - lint-staged: ^13.2.3 - prettier: ^3.0.0 - schema-utils: ^4.0.0 - webpack: ^5.89.0 + babel-eslint: ^10.0.1 + babel-plugin-istanbul: ^6.0.0 + babel-plugin-react-intl: ^8.2.15 + cross-env: ^7.0.2 + eslint: ^7.13.0 + eslint-config-babel: ^9.0.0 + eslint-config-prettier: ^6.3.0 + eslint-plugin-flowtype: ^5.2.0 + eslint-plugin-prettier: ^3.0.0 + fast-stable-stringify: ^1.0.0 + find-cache-dir: ^3.3.1 + husky: ^4.3.0 + lint-staged: ^10.5.1 + loader-utils: ^1.4.0 + make-dir: ^3.1.0 + nyc: ^15.1.0 + pnp-webpack-plugin: ^1.6.4 + prettier: ^2.1.2 + react: ^17.0.1 + react-intl: ^5.9.4 + react-intl-webpack-plugin: ^0.3.0 + rimraf: ^3.0.0 + schema-utils: ^2.6.5 + semver: 7.3.2 + webpack: ^5.34.0 peerDependencies: "@babel/core": ^7.12.0 webpack: ">=5" @@ -3555,6 +3566,13 @@ __metadata: languageName: node linkType: hard +"fast-stable-stringify@npm:^1.0.0": + version: 1.0.0 + resolution: "fast-stable-stringify@npm:1.0.0" + checksum: 972291ed8b8a1a1e13130c91062852dc9142dceed14dc1655c1d0aebc07adcf8a60678252dd1410ec0cceddd3842c87cfea2a937ee3043dfaa6068c3f578f515 + languageName: node + linkType: hard + "fastq@npm:^1.6.0": version: 1.17.1 resolution: "fastq@npm:1.17.1" From 22f0e416c37960d6ab9ea27e3e944f9bccdafb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 11 Jul 2024 12:15:03 -0400 Subject: [PATCH 2/5] update yarn.lock --- yarn.lock | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/yarn.lock b/yarn.lock index b4229b4a..28d5d027 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1,3 +1,6 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + __metadata: version: 6 cacheKey: 8 @@ -2276,31 +2279,18 @@ __metadata: "@babel/eslint-parser": ^7.23.3 "@babel/preset-env": ^7.23.3 ava: ^3.13.0 - babel-eslint: ^10.0.1 - babel-plugin-istanbul: ^6.0.0 - babel-plugin-react-intl: ^8.2.15 - cross-env: ^7.0.2 - eslint: ^7.13.0 - eslint-config-babel: ^9.0.0 - eslint-config-prettier: ^6.3.0 - eslint-plugin-flowtype: ^5.2.0 - eslint-plugin-prettier: ^3.0.0 + c8: ^8.0.0 + eslint: ^9.6.0 + eslint-config-prettier: ^9.1.0 + eslint-plugin-prettier: ^5.1.3 fast-stable-stringify: ^1.0.0 - find-cache-dir: ^3.3.1 - husky: ^4.3.0 - lint-staged: ^10.5.1 - loader-utils: ^1.4.0 - make-dir: ^3.1.0 - nyc: ^15.1.0 - pnp-webpack-plugin: ^1.6.4 - prettier: ^2.1.2 - react: ^17.0.1 - react-intl: ^5.9.4 - react-intl-webpack-plugin: ^0.3.0 - rimraf: ^3.0.0 - schema-utils: ^2.6.5 - semver: 7.3.2 - webpack: ^5.34.0 + find-cache-dir: ^4.0.0 + globals: ^15.8.0 + husky: ^8.0.3 + lint-staged: ^13.2.3 + prettier: ^3.0.0 + schema-utils: ^4.0.0 + webpack: ^5.89.0 peerDependencies: "@babel/core": ^7.12.0 webpack: ">=5" @@ -3569,7 +3559,7 @@ __metadata: "fast-stable-stringify@npm:^1.0.0": version: 1.0.0 resolution: "fast-stable-stringify@npm:1.0.0" - checksum: 972291ed8b8a1a1e13130c91062852dc9142dceed14dc1655c1d0aebc07adcf8a60678252dd1410ec0cceddd3842c87cfea2a937ee3043dfaa6068c3f578f515 + checksum: ef1203d246a7e8ac15e2bfbda0a89fa375947bccf9f7910be0ea759856dbe8ea5024a0d8cc2cceabe18a9cb67e95927b78bb6173a3ae37ec55a518cf36e5244b languageName: node linkType: hard From 6b6551ac1e6d10fbab55379796cc03f4fbe89d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 11 Jul 2024 12:22:24 -0400 Subject: [PATCH 3/5] perf: avoid serialize options twice The options will be serialized in the cache#filename function with the cache identifier, so we don't have to include options in the cache identifier. --- src/index.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/index.js b/src/index.js index bbe3c446..a2b85015 100644 --- a/src/index.js +++ b/src/index.js @@ -27,7 +27,6 @@ const schema = require("./schema"); const { isAbsolute } = require("path"); const validateOptions = require("schema-utils").validate; -const stringify = require("fast-stable-stringify"); function subscribe(subscriber, metadata, context) { if (context[subscriber]) { @@ -175,11 +174,7 @@ async function loader(source, inputSourceMap, overrides) { const { cacheDirectory = null, - cacheIdentifier = stringify({ - options, - "@babel/core": transform.version, - "@babel/loader": version, - }), + cacheIdentifier = "core" + transform.version + "," + "loader" + version, cacheCompression = true, metadataSubscribers = [], } = loaderOptions; From 0d69c901eb9a201b6db6d039261346104cd74988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 11 Jul 2024 13:50:58 -0400 Subject: [PATCH 4/5] polish: use our own config serializer --- package.json | 1 - src/cache.js | 6 ++-- src/serialize.js | 83 ++++++++++++++++++++++++++++++++++++++++++++++ test/cache.test.js | 2 ++ yarn.lock | 8 ----- 5 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 src/serialize.js diff --git a/package.json b/package.json index c5586f7e..7822ed34 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,6 @@ "node": ">= 14.15.0" }, "dependencies": { - "fast-stable-stringify": "^1.0.0", "find-cache-dir": "^4.0.0", "schema-utils": "^4.0.0" }, diff --git a/src/cache.js b/src/cache.js index c9728e75..15b10b77 100644 --- a/src/cache.js +++ b/src/cache.js @@ -13,10 +13,11 @@ const zlib = require("zlib"); const crypto = require("crypto"); const { promisify } = require("util"); const { readFile, writeFile, mkdir } = require("fs/promises"); +// Lazily instantiated when needed const findCacheDirP = import("find-cache-dir"); const transform = require("./transform"); -// Lazily instantiated when needed +const serialize = require("./serialize"); let defaultCacheDirectory = null; let hashType = "sha256"; @@ -29,7 +30,6 @@ try { const gunzip = promisify(zlib.gunzip); const gzip = promisify(zlib.gzip); -const stringify = require("fast-stable-stringify"); /** * Read the contents from the compressed file. @@ -71,7 +71,7 @@ const write = async function (filename, compress, result) { const filename = function (source, identifier, options) { const hash = crypto.createHash(hashType); - hash.update(stringify({ source, options, identifier })); + hash.update(serialize([options, source, identifier])); return hash.digest("hex") + ".json"; }; diff --git a/src/serialize.js b/src/serialize.js new file mode 100644 index 00000000..78e177a8 --- /dev/null +++ b/src/serialize.js @@ -0,0 +1,83 @@ +var objToString = Object.prototype.toString; +var objKeys = Object.getOwnPropertyNames; + +/** + * A custom Babel options serializer + * + * Intentional deviation from JSON.stringify: + * 1. Object properties are sorted before seralizing + * 2. The output is NOT a valid JSON: e.g. + * The output does not enquote strings, which means a JSON-like string '{"a":1}' + * will share the same result with an JS object { a: 1 }. This is not an issue + * for Babel options, but it can not be used for general serialization purpose + * 3. Only 20% slower than the native JSON.stringify on V8 + * + * This function is a fork from https://github.com/nickyout/fast-stable-stringify + * @param {*} val Babel options + * @param {*} isArrayProp + * @returns serialized Babel options + */ +function serialize(val, isArrayProp) { + var i, max, str, keys, key, propVal, toStr; + if (val === true) { + return "!0"; + } + if (val === false) { + return "!1"; + } + switch (typeof val) { + case "object": + if (val === null) { + return null; + } else if (val.toJSON && typeof val.toJSON === "function") { + return serialize(val.toJSON(), isArrayProp); + } else { + toStr = objToString.call(val); + if (toStr === "[object Array]") { + str = "["; + max = val.length - 1; + for (i = 0; i < max; i++) { + str += serialize(val[i], true) + ","; + } + if (max > -1) { + str += serialize(val[i], true); + } + return str + "]"; + } else if (toStr === "[object Object]") { + // only object is left + keys = objKeys(val).sort(); + max = keys.length; + str = "{"; + i = 0; + while (i < max) { + key = keys[i]; + propVal = serialize(val[key], false); + if (propVal !== undefined) { + if (str) { + str += ","; + } + str += '"' + key + '":' + propVal; + } + i++; + } + return str + "}"; + } else { + return JSON.stringify(val); + } + } + case "function": + case "undefined": + return isArrayProp ? null : undefined; + case "string": + return val; + default: + return isFinite(val) ? val : null; + } +} + +module.exports = function (val) { + var returnVal = serialize(val, false); + if (returnVal !== undefined) { + return "" + returnVal; + } +}; diff --git a/test/cache.test.js b/test/cache.test.js index b5ec2aaf..0e81ba90 100644 --- a/test/cache.test.js +++ b/test/cache.test.js @@ -323,5 +323,7 @@ test("should allow to specify the .babelrc file", async t => { t.deepEqual(multiStats.stats[1].compilation.warnings, []); const files = fs.readdirSync(t.context.cacheDirectory); + // The two configs resolved to same Babel config because "fixtures/babelrc" + // is { "presets": ["@babel/preset-env"] } t.true(files.length === 1); }); diff --git a/yarn.lock b/yarn.lock index 28d5d027..ae93271e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2283,7 +2283,6 @@ __metadata: eslint: ^9.6.0 eslint-config-prettier: ^9.1.0 eslint-plugin-prettier: ^5.1.3 - fast-stable-stringify: ^1.0.0 find-cache-dir: ^4.0.0 globals: ^15.8.0 husky: ^8.0.3 @@ -3556,13 +3555,6 @@ __metadata: languageName: node linkType: hard -"fast-stable-stringify@npm:^1.0.0": - version: 1.0.0 - resolution: "fast-stable-stringify@npm:1.0.0" - checksum: ef1203d246a7e8ac15e2bfbda0a89fa375947bccf9f7910be0ea759856dbe8ea5024a0d8cc2cceabe18a9cb67e95927b78bb6173a3ae37ec55a518cf36e5244b - languageName: node - linkType: hard - "fastq@npm:^1.6.0": version: 1.17.1 resolution: "fastq@npm:1.17.1" From 12c8eac7fb39b623fa5b984c27fd0c5989c217c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Thu, 11 Jul 2024 14:00:42 -0400 Subject: [PATCH 5/5] update cacheIdentifier docs --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 890ffd48..377a6f2f 100644 --- a/README.md +++ b/README.md @@ -88,11 +88,7 @@ This loader also supports the following loader-specific option: * `cacheDirectory`: Default `false`. When set, the given directory will be used to cache the results of the loader. Future webpack builds will attempt to read from the cache to avoid needing to run the potentially expensive Babel recompilation process on each run. If the value is set to `true` in options (`{cacheDirectory: true}`), the loader will use the default cache directory in `node_modules/.cache/babel-loader` or fallback to the default OS temporary file directory if no `node_modules` folder could be found in any root directory. -* `cacheIdentifier`: Default is a string composed by - - the `@babel/core`'s version and the `babel-loader`'s version - - the [merged](https://babeljs.io/docs/configuration#how-babel-merges-config-items) [Babel config](https://babeljs.io/docs/config-files), including options passed to `babel-loader` and the contents of `babel.config.js` or `.babelrc` file if they exist - - the value of the environment variable `BABEL_ENV` with a fallback to the `NODE_ENV` environment variable. - This can be set to a custom value to force cache busting if the identifier changes. +* `cacheIdentifier`: Default is a string composed by the `@babel/core`'s version and the `babel-loader`'s version. The final cache id will be determined by the input file path, the [merged](https://babeljs.io/docs/configuration#how-babel-merges-config-items) Babel config via `Babel.loadPartialConfigAsync` and the `cacheIdentifier`. The merged Babel config will be determined by the `babel.config.js` or `.babelrc` file if they exist, or the value of the environment variable `BABEL_ENV` and `NODE_ENV`. `cacheIdentifier` can be set to a custom value to force cache busting if the identifier changes. * `cacheCompression`: Default `true`. When set, each Babel transform output will be compressed with Gzip. If you want to opt-out of cache compression, set it to `false` -- your project may benefit from this if it transpiles thousands of files.