From 3f149b9bd193c8d670672f6bd04f103c2292900e Mon Sep 17 00:00:00 2001 From: remorses Date: Wed, 20 Oct 2021 00:15:57 +0200 Subject: [PATCH 1/3] rate limit requests --- package.json | 3 ++- pnpm-lock.yaml | 7 +++++++ src/helpers/helpers.ts | 10 ++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 940c8658..6980c427 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ }, "dependencies": { "env-cmd": "^10.1.0", - "p-limit": "^4.0.0" + "p-limit": "^4.0.0", + "p-ratelimit": "^1.0.1" }, "devDependencies": { "@types/jest": "^27.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7324b6ee..2ffdaed7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,7 @@ specifiers: eslint-config-prettier: ^8.3.0 jest: ^27.3.1 p-limit: ^4.0.0 + p-ratelimit: ^1.0.1 prettier: ^2.4.1 sucrase: ^3.20.3 terser: ^5.9.0 @@ -20,6 +21,7 @@ specifiers: dependencies: env-cmd: 10.1.0 p-limit: 4.0.0 + p-ratelimit: 1.0.1 devDependencies: '@types/jest': 27.0.2 @@ -3641,6 +3643,11 @@ packages: p-limit: 2.3.0 dev: true + /p-ratelimit/1.0.1: + resolution: {integrity: sha512-tKBGoow6aWRH68K2eQx+qc1gSegjd5VLirZYc1Yms9pPFsYQ9TFI6aMn0vJH2vmvzjNpjlWZOFft4aPUen2w0A==} + engines: {node: '>=10.23.0'} + dev: false + /p-try/2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index 739ddde0..1f2fb336 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -2,6 +2,14 @@ import { promises as fs } from 'fs' import { exec as execCallback } from 'child_process' import pLimit from 'p-limit' const { access } = fs +import { pRateLimit } from 'p-ratelimit' + +// limit to 100 requests per minute (actual limit is 120 per minute) +export const limit = pRateLimit({ + interval: 1000 * 60, + rate: 100, + concurrency: 4, +}) export const exists = (path: string): Promise => access(path) @@ -12,5 +20,3 @@ export const exec = (cmd: string): Promise => new Promise((res, _rej) => execCallback(cmd, (_, stdout, stderr) => res(stdout + stderr)), ) - -export const limit = pLimit(8) From 67fdd86f9bbbbdb8aeb6ffb1c286eb257bdbf60a Mon Sep 17 00:00:00 2001 From: remorses Date: Wed, 20 Oct 2021 00:44:39 +0200 Subject: [PATCH 2/3] remove p-limit --- package.json | 1 - pnpm-lock.yaml | 14 -------------- src/helpers/helpers.ts | 1 - 3 files changed, 16 deletions(-) diff --git a/package.json b/package.json index 6980c427..8000522b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ }, "dependencies": { "env-cmd": "^10.1.0", - "p-limit": "^4.0.0", "p-ratelimit": "^1.0.1" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ffdaed7..0172eaad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,7 +11,6 @@ specifiers: eslint: ^7.32.0 eslint-config-prettier: ^8.3.0 jest: ^27.3.1 - p-limit: ^4.0.0 p-ratelimit: ^1.0.1 prettier: ^2.4.1 sucrase: ^3.20.3 @@ -20,7 +19,6 @@ specifiers: dependencies: env-cmd: 10.1.0 - p-limit: 4.0.0 p-ratelimit: 1.0.1 devDependencies: @@ -3629,13 +3627,6 @@ packages: p-try: 2.2.0 dev: true - /p-limit/4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - dependencies: - yocto-queue: 1.0.0 - dev: false - /p-locate/4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -4549,8 +4540,3 @@ packages: y18n: 5.0.8 yargs-parser: 20.2.9 dev: true - - /yocto-queue/1.0.0: - resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} - engines: {node: '>=12.20'} - dev: false diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index 1f2fb336..773ec96f 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -1,6 +1,5 @@ import { promises as fs } from 'fs' import { exec as execCallback } from 'child_process' -import pLimit from 'p-limit' const { access } = fs import { pRateLimit } from 'p-ratelimit' From 94ff234284d3451c1f8b0cda2a98b389f70dd9cc Mon Sep 17 00:00:00 2001 From: remorses Date: Wed, 20 Oct 2021 11:30:22 +0200 Subject: [PATCH 3/3] smaller interval and limits, tried locally --- src/helpers/helpers.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/helpers/helpers.ts b/src/helpers/helpers.ts index 773ec96f..aa7ea32b 100644 --- a/src/helpers/helpers.ts +++ b/src/helpers/helpers.ts @@ -3,11 +3,10 @@ import { exec as execCallback } from 'child_process' const { access } = fs import { pRateLimit } from 'p-ratelimit' -// limit to 100 requests per minute (actual limit is 120 per minute) export const limit = pRateLimit({ - interval: 1000 * 60, - rate: 100, - concurrency: 4, + interval: 1000 * 10, + rate: 9, + concurrency: 3, }) export const exists = (path: string): Promise =>