From e336b9f85164970cba6eb731f7124da70de31a41 Mon Sep 17 00:00:00 2001 From: lzxb <1340641314@qq.com> Date: Tue, 10 Dec 2024 19:23:42 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20fetch=20=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/fetch/.stylelintignore | 13 ++ packages/fetch/.stylelintrc.js | 5 + packages/fetch/biome.json | 122 +++++++++++++++++++ packages/fetch/build.config.ts | 18 +++ packages/fetch/package.json | 43 +++++++ packages/fetch/src/fetch.test.ts | 12 ++ packages/fetch/src/index.ts | 198 +++++++++++++++++++++++++++++++ packages/fetch/tsconfig.json | 43 +++++++ pnpm-lock.yaml | 69 +++++++++++ 9 files changed, 523 insertions(+) create mode 100644 packages/fetch/.stylelintignore create mode 100644 packages/fetch/.stylelintrc.js create mode 100644 packages/fetch/biome.json create mode 100644 packages/fetch/build.config.ts create mode 100644 packages/fetch/package.json create mode 100644 packages/fetch/src/fetch.test.ts create mode 100644 packages/fetch/src/index.ts create mode 100644 packages/fetch/tsconfig.json diff --git a/packages/fetch/.stylelintignore b/packages/fetch/.stylelintignore new file mode 100644 index 00000000..c55d3401 --- /dev/null +++ b/packages/fetch/.stylelintignore @@ -0,0 +1,13 @@ +.root +node_modules +*_back +dist +types +**/i18n/** +.rpt2_cache +**.log +coverage +.idea +.git +template +public diff --git a/packages/fetch/.stylelintrc.js b/packages/fetch/.stylelintrc.js new file mode 100644 index 00000000..c643a1ce --- /dev/null +++ b/packages/fetch/.stylelintrc.js @@ -0,0 +1,5 @@ +// Delete this comment line, the template will no longer be written +export default { + extends: [import.meta.resolve('@gez/lint/css')], + rules: {} +}; diff --git a/packages/fetch/biome.json b/packages/fetch/biome.json new file mode 100644 index 00000000..0e8d6bea --- /dev/null +++ b/packages/fetch/biome.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.2/schema.json", + "vcs": { + "enabled": false + }, + "overrides": [ + { + "include": [ + "*.vue" + ], + "linter": { + "rules": { + "style": { + "useConst": "off", + "useImportType": "off" + } + } + } + } + ], + "files": { + "ignoreUnknown": false, + "ignore": [ + "node_modules/**/*", + "node_modules_back/**/*", + "dist/**/*", + "types/**/*", + ".rpt2_cache/**/*", + "coverage/**/*", + ".idea/**/*", + ".git/**/*", + "public/**/*", + ".root/**/*", + "**/i18n/**", + "**/svg-icons.ts", + "build.config.ts", + "*.min.js" + ] + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 4 + }, + "organizeImports": { + "enabled": true + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true, + "suspicious": { + "noEmptyInterface": "off", + "noExplicitAny": "off", + "noAssignInExpressions": "off", + "useGetterReturn": "warn", + "noGlobalIsNan": "warn", + "noImplicitAnyLet": "warn", + "noShadowRestrictedNames": "warn", + "useIsArray": "warn", + "noThenProperty": "warn" + }, + "security": { + "noGlobalEval": "warn" + }, + "complexity": { + "noForEach": "off", + "noThisInStatic": "off", + "noBannedTypes": "off", + "useOptionalChain": "warn", + "noUselessConstructor": "warn", + "noUselessSwitchCase": "warn", + "noStaticOnlyClass": "warn" + }, + "style": { + "useFilenamingConvention": { + "level": "warn", + "options": { + "strictCase": true, + "requireAscii": true, + "filenameCases": [ + "kebab-case" + ] + } + }, + "useExponentiationOperator": "off", + "noUselessElse": "off", + "useTemplate": "off", + "noUnusedTemplateLiteral": "off", + "noNonNullAssertion": "off", + "noParameterAssign": "off", + "noArguments": "warn", + "useNodejsImportProtocol": "warn", + "useDefaultParameterLast": "warn" + }, + "performance": { + "noDelete": "warn", + "noAccumulatingSpread": "warn" + }, + "correctness": { + "noUnsafeOptionalChaining": "warn", + "noVoidTypeReturn": "warn" + } + } + }, + "javascript": { + "formatter": { + "quoteStyle": "single", + "trailingCommas": "none" + } + }, + "json": { + "formatter": { + "enabled": false + } + }, + "css": { + "formatter": { + "enabled": false + } + } +} diff --git a/packages/fetch/build.config.ts b/packages/fetch/build.config.ts new file mode 100644 index 00000000..7a406f4c --- /dev/null +++ b/packages/fetch/build.config.ts @@ -0,0 +1,18 @@ +// Template generation, do not manually modify +import { defineBuildConfig } from 'unbuild'; + +export default defineBuildConfig({ + clean: true, + entries: [ + { + input: './src/', + format: 'esm', + ext: 'mjs', + cleanDist: true, + declaration: true, + esbuild: { + target: 'node22' + } + } + ] +}); diff --git a/packages/fetch/package.json b/packages/fetch/package.json new file mode 100644 index 00000000..4d4febcb --- /dev/null +++ b/packages/fetch/package.json @@ -0,0 +1,43 @@ +{ + "name": "@gez/fetch", + "template": "library-node", + "scripts": { + "lint:css": "stylelint '**/*.{css,vue}' --fix --aei", + "lint:type": "tsc --noEmit", + "test": "vitest --pass-with-no-tests", + "coverage": "vitest run --coverage --pass-with-no-tests", + "lint:js": "biome check --write --no-errors-on-unmatched", + "build": "unbuild" + }, + "devDependencies": { + "@biomejs/biome": "1.9.4", + "@gez/lint": "3.0.0-beta.27", + "@types/cli-progress": "^3.11.6", + "@types/node": "22.9.0", + "@vitest/coverage-v8": "2.1.5", + "axios": "^1.7.7", + "cli-progress": "^3.12.0", + "stylelint": "16.10.0", + "typescript": "5.6.3", + "unbuild": "2.0.0", + "vitest": "2.1.5" + }, + "version": "0.0.0", + "type": "module", + "private": false, + "exports": { + ".": { + "import": "./dist/index.mjs", + "types": "./dist/index.d.ts" + } + }, + "module": "dist/index.mjs", + "types": "./dist/index.d.ts", + "files": [ + "src", + "dist", + "*.mjs", + "template", + "public" + ] +} diff --git a/packages/fetch/src/fetch.test.ts b/packages/fetch/src/fetch.test.ts new file mode 100644 index 00000000..d46c01ca --- /dev/null +++ b/packages/fetch/src/fetch.test.ts @@ -0,0 +1,12 @@ +import { test } from 'vitest'; +import { fetchPkgsWithProgress } from '.'; + +test('base', async () => { + await fetchPkgsWithProgress({ + baseURL: 'https://dp-os.github.io/gez/', + urls: ['ssr-html/versions/latest.tgz', 'ssr-html/versions/1.0.tgz'], + timeout: 4500 + }).then((...args) => { + console.log(...args); + }); +}, 5000); diff --git a/packages/fetch/src/index.ts b/packages/fetch/src/index.ts new file mode 100644 index 00000000..ad0624e3 --- /dev/null +++ b/packages/fetch/src/index.ts @@ -0,0 +1,198 @@ +/* + +完成远程依赖下载实现。 +从网络或本地获取文件，并缓存到本地。如果有缓存则使用缓存。 + +例子： + +fetchPkgsWithProgress({ + baseURL: 'https://dp-os.github.io/gez/', + urls: [ + 'ssr-html/versions/latest.tgz', + 'ssr-html/versions/1.0.tgz', + ], +}).then((...args) => { + console.log(...args); +}); + + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { URL } from 'node:url'; +import axios, { type AxiosResponse, type AxiosRequestConfig } from 'axios'; +import { MultiBar, type SingleBar } from 'cli-progress'; + +export interface FetchPkgWithCacheOptions extends AxiosRequestConfig { + /** + * 缓存文件路径 默认为 .root/packages + */ + cachePath?: string; + /** + * 日志输出函数 默认为 console.log + * @param str string 输出的字符串 + * @returns void + */ + logger?: (str: string) => void; +} + +export type FetchResult = { + /** + * 资源是从哪个 URL 获取的 + */ + url: string; +} & ( + | { + /** + * 缓存文件路径 默认为 {cachePath}/prj/path/filename-hash.ext + */ + cacheFilePath: string; + /** + * 文件 hash 值 + */ + hash: string; + /** + * 是否命中缓存 + */ + hitCache: boolean; + } + | { + /** + * 错误信息 + */ + error: Error; + } +); + +/** + * 获取文件，并缓存到本地。如果有缓存则使用缓存 + */ +export async function fetchPkgWithCache({ + cachePath = '.root/packages', + logger = console.log, + ...axiosOptions +}: FetchPkgWithCacheOptions): Promise { + const url = (axiosOptions.baseURL || '') + axiosOptions.url || ''; + logger(`[fetch] Start ${url}`); + const pathInfo = path.parse(new URL(url).pathname); + const pkgPath = path.join(cachePath, pathInfo.dir); + + if (!fs.existsSync(pkgPath)) fs.mkdirSync(pkgPath, { recursive: true }); + + // 获取 hash + const hashUrl = url.replace(new RegExp(path.extname(url) + '$'), '.txt'); + let hash = ''; + try { + hash = ( + await axios.get(hashUrl, { + ...axiosOptions, + responseType: 'text' + }) + ).data; + } catch (error: any) { + logger(`[fetch] Get hash error ${url}: ${error.message}`); + return { url, error }; + } + const cacheFilePath = path.join( + pkgPath, + pathInfo.name + '-' + hash + pathInfo.ext + ); + if (fs.existsSync(cacheFilePath)) { + logger(`[fetch] Hit cache ${url}: ${cacheFilePath}`); + return { url, cacheFilePath, hash, hitCache: true }; + } + + // 下载文件 + let result: AxiosResponse | null = null; + try { + result = await axios.get(url, { + ...axiosOptions, + responseType: 'stream' + }); + } catch (error: any) { + logger(`[fetch] Error ${url}: ${error.message}`); + return { url, error }; + } + const fileStream = fs.createWriteStream(cacheFilePath); + const streamPromise = new Promise((resolve, reject) => { + fileStream.on('finish', resolve); + fileStream.on('error', reject); + }); + result?.data.pipe(fileStream); + await streamPromise; + + logger(`[fetch] Downloaded ${url}: ${cacheFilePath}`); + return { url, cacheFilePath, hash, hitCache: false }; +} + +export interface FetchPkgsWithProgressOptions extends AxiosRequestConfig { + /** + * 缓存文件路径 默认为 .root/packages + */ + cachePath?: string; + /** + * 要获取的 URL 列表 + */ + urls: string[]; + /** + * axios 配置字典，用于指定不同 URL 的配置 + */ + axiosOptions?: { [url: string]: AxiosRequestConfig }; +} + +/** + * 获取多个文件，并缓存到本地。如果有缓存则使用缓存。带有进度条 + */ +export async function fetchPkgsWithProgress({ + cachePath = '.root/packages', + urls, + axiosOptions, + ...comAxiosOptions +}: FetchPkgsWithProgressOptions): Promise { + const multiBar = new MultiBar({ + stopOnComplete: true, + format: ' [{bar}] {percentage}% | {eta_formatted}/{duration_formatted} | {value}/{total} | {url}', + forceRedraw: true, + barCompleteChar: '#', + barIncompleteChar: '_' + }); + const logger = (str = '') => { + // multiBar.log 最后一个字符需要是换行符 + multiBar.log(str.trimEnd() + '\n'); + multiBar.update(); // force redraw + }; + const bars: { [url: string]: SingleBar } = urls.reduce( + (obj, url) => ({ + ...obj, + [url]: multiBar.create(1, 0, { url }) + }), + {} + ); + // 不知为何，有的时候不会更新，强制每秒重绘一次 + const timer = setInterval(() => multiBar.update(), 1000); + const results = await Promise.all( + urls.map((url) => + fetchPkgWithCache({ + cachePath, + ...comAxiosOptions, + ...(axiosOptions?.[url] || {}), + url, + onDownloadProgress(progressEvent) { + bars[url].setTotal( + progressEvent?.total ?? progressEvent.loaded + 1 + ); + bars[url].update(progressEvent.loaded, { url }); + bars[url].updateETA(); + if (axiosOptions?.[url]?.onDownloadProgress) + axiosOptions[url].onDownloadProgress(progressEvent); + else comAxiosOptions.onDownloadProgress?.(progressEvent); + }, + logger + }) + ) + ); + clearInterval(timer); + multiBar.update(); // force redraw + multiBar.stop(); + return results; +} diff --git a/packages/fetch/tsconfig.json b/packages/fetch/tsconfig.json new file mode 100644 index 00000000..9ac77b28 --- /dev/null +++ b/packages/fetch/tsconfig.json @@ -0,0 +1,43 @@ +{ + "compilerOptions": { + "isolatedModules": true, + "allowJs": false, + "experimentalDecorators": true, + "resolveJsonModule": true, + "types": [ + "@types/node" + ], + "target": "ESNext", + "module": "ESNext", + "importHelpers": false, + "declaration": true, + "sourceMap": true, + "strict": true, + "noImplicitAny": false, + "noImplicitReturns": false, + "noFallthroughCasesInSwitch": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "moduleResolution": "node", + "esModuleInterop": true, + "skipLibCheck": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true + }, + "include": [ + "server", + "test", + "src", + "private", + "public", + "bin", + "**.cjs", + "**.js", + "**.mjs", + "**.ts" + ], + "exclude": [ + "dist" + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c770990a..29d8871f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -315,6 +315,42 @@ importers: specifier: 2.1.5 version: 2.1.5(@types/node@22.9.0)(less@4.2.0)(sass-embedded@1.81.0)(terser@5.36.0) + packages/fetch: + devDependencies: + '@biomejs/biome': + specifier: 1.9.4 + version: 1.9.4 + '@gez/lint': + specifier: 3.0.0-beta.27 + version: 3.0.0-beta.27(postcss-html@1.7.0)(postcss@8.4.49)(stylelint@16.10.0(typescript@5.6.3)) + '@types/cli-progress': + specifier: ^3.11.6 + version: 3.11.6 + '@types/node': + specifier: 22.9.0 + version: 22.9.0 + '@vitest/coverage-v8': + specifier: 2.1.5 + version: 2.1.5(vitest@2.1.5(@types/node@22.9.0)(less@4.2.0)(sass-embedded@1.81.0)(terser@5.36.0)) + axios: + specifier: ^1.7.7 + version: 1.7.7 + cli-progress: + specifier: ^3.12.0 + version: 3.12.0 + stylelint: + specifier: 16.10.0 + version: 16.10.0(typescript@5.6.3) + typescript: + specifier: 5.6.3 + version: 5.6.3 + unbuild: + specifier: 2.0.0 + version: 2.0.0(typescript@5.6.3)(vue-tsc@2.1.10(typescript@5.6.3)) + vitest: + specifier: 2.1.5 + version: 2.1.5(@types/node@22.9.0)(less@4.2.0)(sass-embedded@1.81.0)(terser@5.36.0) + packages/import: dependencies: '@import-maps/resolve': @@ -1148,6 +1184,11 @@ packages: cpu: [x64] os: [win32] + '@gez/lint@3.0.0-beta.27': + resolution: {integrity: sha512-j59f766Rljr30Q97UQixCuuTFx97dehb9YiRCBvT8IMd9DvP321X39VJ9/UYXooALF6O49i7LTYK7Bmx+NEJ4A==} + peerDependencies: + stylelint: '>=16.5.0' + '@import-maps/resolve@2.0.0': resolution: {integrity: sha512-RwzRTpmrrS6Q1ZhQExwuxJGK1Wqhv4stt+OF2JzS+uawewpwNyU7EJL1WpBex7aDiiGLs4FsXGkfUBdYuX7xiQ==} @@ -1802,6 +1843,9 @@ packages: '@types/cacache@17.0.2': resolution: {integrity: sha512-IrqHzVX2VRMDQQKa7CtKRnuoCLdRJiLW6hWU+w7i7+AaQ0Ii5bKwJxd5uRK4zBCyrHd3tG6G8zOm2LplxbSfQg==} + '@types/cli-progress@3.11.6': + resolution: {integrity: sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA==} + '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} @@ -2438,6 +2482,10 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-truncate@4.0.0: resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} engines: {node: '>=18'} @@ -6524,6 +6572,19 @@ snapshots: '@esbuild/win32-x64@0.24.0': optional: true + '@gez/lint@3.0.0-beta.27(postcss-html@1.7.0)(postcss@8.4.49)(stylelint@16.10.0(typescript@5.6.3))': + dependencies: + stylelint: 16.10.0(typescript@5.6.3) + stylelint-config-html: 1.1.0(postcss-html@1.7.0)(stylelint@16.10.0(typescript@5.6.3)) + stylelint-config-recess-order: 5.1.1(stylelint@16.10.0(typescript@5.6.3)) + stylelint-config-recommended-less: 3.0.1(postcss@8.4.49)(stylelint@16.10.0(typescript@5.6.3)) + stylelint-config-recommended-vue: 1.5.0(postcss-html@1.7.0)(stylelint@16.10.0(typescript@5.6.3)) + stylelint-config-standard: 36.0.1(stylelint@16.10.0(typescript@5.6.3)) + stylelint-order: 6.0.4(stylelint@16.10.0(typescript@5.6.3)) + transitivePeerDependencies: + - postcss + - postcss-html + '@import-maps/resolve@2.0.0': {} '@isaacs/cliui@8.0.2': @@ -7274,6 +7335,10 @@ snapshots: dependencies: '@types/node': 20.17.6 + '@types/cli-progress@3.11.6': + dependencies: + '@types/node': 20.17.6 + '@types/connect@3.4.38': dependencies: '@types/node': 20.17.6 @@ -8180,6 +8245,10 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-truncate@4.0.0: dependencies: slice-ansi: 5.0.0