From 940d110d8f5c042b1a6feed53aa585bde3eabcd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andony=20N=C3=BA=C3=B1ez?= Date: Tue, 12 Dec 2023 10:52:28 -0600 Subject: [PATCH] [RND-665] Lower rate limiting for OAuth (#331) --- .../package.json | 6 +-- .../meadowlark-mongodb-backend/package.json | 8 +-- .../package.json | 6 +-- .../package.json | 8 +-- Meadowlark-js/lerna.json | 2 +- Meadowlark-js/package-lock.json | 50 +++++++++---------- Meadowlark-js/package.json | 2 +- .../meadowlark-authz-server/package.json | 4 +- .../packages/meadowlark-core/package.json | 4 +- .../meadowlark-utilities/package.json | 2 +- Meadowlark-js/sample-load-balancer.conf | 8 +++ .../services/meadowlark-fastify/package.json | 8 +-- Meadowlark-js/tests/e2e/package.json | 4 +- docs/design/rate-limiting-for-oauth/README.md | 36 +++++++++++++ 14 files changed, 96 insertions(+), 52 deletions(-) create mode 100644 docs/design/rate-limiting-for-oauth/README.md diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json index 9e315647..1688d4a3 100644 --- a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-elasticsearch-backend", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark backend plugin for elasticsearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, diff --git a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json index dfd90ca6..656d1d6e 100644 --- a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-mongodb-backend", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark backend plugin for MongoDB", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "async-retry": "^1.3.3", "mongodb": "^5.9.0", "ramda": "0.29.1" diff --git a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json index 34c35a5a..b0e14946 100644 --- a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-opensearch-backend", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark backend plugin for OpenSearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json index b5236533..e0f06bcf 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-postgresql-backend", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark backend plugin for PostgreSQL", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" diff --git a/Meadowlark-js/lerna.json b/Meadowlark-js/lerna.json index c57ecb21..f845c7bd 100644 --- a/Meadowlark-js/lerna.json +++ b/Meadowlark-js/lerna.json @@ -3,7 +3,7 @@ "packages": [ "packages/*" ], - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "npmClient": "npm", "useWorkspaces": true } diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index a7bca19c..5ac124bf 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -52,11 +52,11 @@ }, "backends/meadowlark-elasticsearch-backend": { "name": "@edfi/meadowlark-elasticsearch-backend", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, @@ -70,12 +70,12 @@ }, "backends/meadowlark-mongodb-backend": { "name": "@edfi/meadowlark-mongodb-backend", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "async-retry": "^1.3.3", "mongodb": "^5.9.0", "ramda": "0.29.1" @@ -88,11 +88,11 @@ }, "backends/meadowlark-opensearch-backend": { "name": "@edfi/meadowlark-opensearch-backend", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { @@ -105,12 +105,12 @@ }, "backends/meadowlark-postgresql-backend": { "name": "@edfi/meadowlark-postgresql-backend", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" @@ -22677,11 +22677,11 @@ }, "packages/meadowlark-authz-server": { "name": "@edfi/meadowlark-authz-server", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", @@ -22731,11 +22731,11 @@ }, "packages/meadowlark-core": { "name": "@edfi/meadowlark-core", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", @@ -22824,7 +22824,7 @@ }, "packages/meadowlark-utilities": { "name": "@edfi/meadowlark-utilities", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { "pino": "^8.15.4", @@ -22871,12 +22871,12 @@ }, "services/meadowlark-fastify": { "name": "@edfi/meadowlark-fastify", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" @@ -22889,10 +22889,10 @@ }, "tests/e2e": { "name": "@edfi/meadowlark-e2e-tests", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "license": "Apache-2.0", "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/Meadowlark-js/package.json b/Meadowlark-js/package.json index 0f0e5fc6..a918ca7d 100644 --- a/Meadowlark-js/package.json +++ b/Meadowlark-js/package.json @@ -72,7 +72,7 @@ "test:e2e:dev:exit": "bash ../eng/exit-dev-containers.sh", "start:local": "lerna run start:local", "docker:lint": "cat ./Dockerfile | docker run --rm -i hadolint/hadolint", - "docker:install": "npm i lerna@^6.0.2 rimraf@^3.0.2 copyfiles@^2.4.1 typescript@4.8.4 -g && npm ci --only=production", + "docker:install": "npm i lerna@^6.6.2 rimraf@^5.0.5 copyfiles@^2.4.1 typescript@4.9.5 -g && npm ci --only=production", "docker:build": "docker build -t meadowlark .", "docker:build:wsl1": "docker.exe build -t meadowlark .", "docker:start": "docker run -d --env-file ./.env-docker --name ml-api -p 3000:3000 meadowlark", diff --git a/Meadowlark-js/packages/meadowlark-authz-server/package.json b/Meadowlark-js/packages/meadowlark-authz-server/package.json index 7866c5bb..5244f773 100644 --- a/Meadowlark-js/packages/meadowlark-authz-server/package.json +++ b/Meadowlark-js/packages/meadowlark-authz-server/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-authz-server", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark authorization server", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", diff --git a/Meadowlark-js/packages/meadowlark-core/package.json b/Meadowlark-js/packages/meadowlark-core/package.json index 5077f225..0a0b8544 100644 --- a/Meadowlark-js/packages/meadowlark-core/package.json +++ b/Meadowlark-js/packages/meadowlark-core/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-core", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark core functionality", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", diff --git a/Meadowlark-js/packages/meadowlark-utilities/package.json b/Meadowlark-js/packages/meadowlark-utilities/package.json index 25c7468e..8abb767d 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/package.json +++ b/Meadowlark-js/packages/meadowlark-utilities/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-utilities", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark shared utilities", "license": "Apache-2.0", "publishConfig": { diff --git a/Meadowlark-js/sample-load-balancer.conf b/Meadowlark-js/sample-load-balancer.conf index eae47858..b12f2633 100644 --- a/Meadowlark-js/sample-load-balancer.conf +++ b/Meadowlark-js/sample-load-balancer.conf @@ -1,3 +1,5 @@ +limit_req_zone $binary_remote_addr zone=oauthlimit:10m rate=1r/m; + upstream meadowlarkoauth { server meadowlark-oauth-ml-local-1:3000; server meadowlark-oauth-ml-local-2:3000; @@ -34,6 +36,12 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + location /local/oauth/token { + proxy_pass http://meadowlarkoauth; + + limit_req zone=oauthlimit burst=3 nodelay; + } + #error_page 404 /404.html; # redirect server error pages to the static page /50x.html diff --git a/Meadowlark-js/services/meadowlark-fastify/package.json b/Meadowlark-js/services/meadowlark-fastify/package.json index cc9bd15f..6b2ad960 100644 --- a/Meadowlark-js/services/meadowlark-fastify/package.json +++ b/Meadowlark-js/services/meadowlark-fastify/package.json @@ -1,6 +1,6 @@ { "name": "@edfi/meadowlark-fastify", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark service using Fastify", "license": "Apache-2.0", "publishConfig": { @@ -12,9 +12,9 @@ "/package.json" ], "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.6", - "@edfi/meadowlark-core": "0.4.0-pre.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-authz-server": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" diff --git a/Meadowlark-js/tests/e2e/package.json b/Meadowlark-js/tests/e2e/package.json index 102a4fd8..4c732fa4 100644 --- a/Meadowlark-js/tests/e2e/package.json +++ b/Meadowlark-js/tests/e2e/package.json @@ -1,13 +1,13 @@ { "name": "@edfi/meadowlark-e2e-tests", "main": "dist/index.js", - "version": "0.4.0-pre.6", + "version": "0.4.0-pre.7", "description": "Meadowlark Ed-Fi API end to end tests", "license": "Apache-2.0", "private": true, "files": [], "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.0-pre.6", + "@edfi/meadowlark-utilities": "0.4.0-pre.7", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/docs/design/rate-limiting-for-oauth/README.md b/docs/design/rate-limiting-for-oauth/README.md new file mode 100644 index 00000000..e0f06a17 --- /dev/null +++ b/docs/design/rate-limiting-for-oauth/README.md @@ -0,0 +1,36 @@ +# Rate Limiting for OAuth + +To prevent a possible attack, we want to limit the users from authenticating +multiple times in a short period of time, this can be done with the load +balancer. As an example, we will use the NGINX +[sample-load-balancer](../../../Meadowlark-js/sample-load-balancer.conf) + +## Configuration + +NGINX has a built in setup of rate limiting in a configuration file per endpoint +(can be done at a higher level). See +https://www.nginx.com/blog/rate-limiting-nginx/ for guidance, and the +[documentation](https://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req) + +### Setup + +1. Define the `limit_req_zone` at the beginning of the + [file](../../../Meadowlark-js/sample-load-balancer.conf#L1). The zone means the + memory used to save IP addresses, and the `rate` represent how often the + count will be restarted (every minute). + +2. For the desired endpoint, specify the `limit_req` with the name of the zone + specified in the previous step, and the `burst` which is the amount of time + allowed per time period (the rate). In this example, we set the `rate` to 1 + minute and the `burst` to 3 requests during that minute. + +3. Start Meadowlark fully in Docker, using MongoDB as the backend and OpenSearch + as the search provider. + + ```pwsh + cd Meadowlark-js + ./reset-docker-compose.ps1 + ``` + +With this setup, the endpoint will have the request limit in place, which is the +recommended approach with a load balancer or front end proxy.