From 6947f410b4ee4ebea4e4df73e698de9d5180ff58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andony=20N=C3=BA=C3=B1ez?= Date: Mon, 22 Jan 2024 08:41:30 -0600 Subject: [PATCH] [RND-580] Adding notes in documentation to run load performance tests (#342) --- Meadowlark-js/.env.example | 2 +- .../package.json | 6 +- .../meadowlark-mongodb-backend/package.json | 8 +- .../package.json | 6 +- .../package.json | 8 +- Meadowlark-js/docker-compose.yml | 3 +- Meadowlark-js/lerna.json | 2 +- Meadowlark-js/package-lock.json | 150 +++++++++--------- .../meadowlark-authz-server/package.json | 4 +- .../packages/meadowlark-core/package.json | 4 +- .../meadowlark-utilities/package.json | 2 +- .../services/meadowlark-fastify/package.json | 8 +- Meadowlark-js/tests/e2e/package.json | 4 +- docs/CONFIGURATION.md | 4 +- .../SETUP-PERFORMANCE-SUITE.md | 7 +- .../WRITE-PERFORMANCE.md | 5 + .../load-balancing-API-scale-out.md | 2 +- .../mongo-connection-pooling.md | 2 +- ...ctionalities-of-kafka-connect-separated.md | 2 +- eng/deploy/azure/README.md | 2 +- 20 files changed, 116 insertions(+), 115 deletions(-) diff --git a/Meadowlark-js/.env.example b/Meadowlark-js/.env.example index 62c42c80..90df8083 100644 --- a/Meadowlark-js/.env.example +++ b/Meadowlark-js/.env.example @@ -48,4 +48,4 @@ ALLOW__EXT_PROPERTY=true ALLOW_OVERPOSTING=true LOG_TO_FILE=false -# LOG_FILE_LOCATION=c:/temp/ +# LOG_FILE_LOCATION=/PATH/ diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json index 981f5639..36c3a79a 100644 --- a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-elasticsearch-backend", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark backend plugin for elasticsearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, diff --git a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json index cee397e8..31b12a89 100644 --- a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-mongodb-backend", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark backend plugin for MongoDB", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "async-retry": "^1.3.3", "mongodb": "^5.9.2", "ramda": "0.29.1" diff --git a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json index 8d7ad0d8..ef29a812 100644 --- a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-opensearch-backend", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark backend plugin for OpenSearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json index 32fbf089..645cf710 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-postgresql-backend", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark backend plugin for PostgreSQL", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" diff --git a/Meadowlark-js/docker-compose.yml b/Meadowlark-js/docker-compose.yml index c2bc5422..96209c1f 100644 --- a/Meadowlark-js/docker-compose.yml +++ b/Meadowlark-js/docker-compose.yml @@ -45,7 +45,8 @@ services: POSTGRES_HOST: ${POSTGRES_HOST:-postgres} POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-abcdefgh1!} - LOG_TO_FILE: true + LOG_TO_FILE: ${LOG_TO_FILE:-true} + LOG_FILE_LOCATION: ${LOG_FILE_LOCATION:-/var/log/} restart: unless-stopped healthcheck: &ML_HEALTH test: curl -s http://localhost:3000/local >/dev/null || exit 1 diff --git a/Meadowlark-js/lerna.json b/Meadowlark-js/lerna.json index 0c10e6e2..dbe07961 100644 --- a/Meadowlark-js/lerna.json +++ b/Meadowlark-js/lerna.json @@ -1,5 +1,5 @@ { "lerna": "8.0.2", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "npmClient": "npm" } diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index 683c25be..63ed3041 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -54,11 +54,11 @@ }, "backends/meadowlark-elasticsearch-backend": { "name": "@edfi/meadowlark-elasticsearch-backend", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, @@ -72,12 +72,12 @@ }, "backends/meadowlark-mongodb-backend": { "name": "@edfi/meadowlark-mongodb-backend", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "async-retry": "^1.3.3", "mongodb": "^5.9.2", "ramda": "0.29.1" @@ -90,11 +90,11 @@ }, "backends/meadowlark-opensearch-backend": { "name": "@edfi/meadowlark-opensearch-backend", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { @@ -107,12 +107,12 @@ }, "backends/meadowlark-postgresql-backend": { "name": "@edfi/meadowlark-postgresql-backend", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" @@ -2568,18 +2568,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/@lerna/create/node_modules/is-ci": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz", - "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==", - "dev": true, - "dependencies": { - "ci-info": "^3.2.0" - }, - "bin": { - "is-ci": "bin.js" - } - }, "node_modules/@lerna/create/node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -7991,14 +7979,15 @@ "license": "MIT" }, "node_modules/follow-redirects": { - "version": "1.15.2", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], - "license": "MIT", "engines": { "node": ">=4.0" }, @@ -9155,6 +9144,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-ci": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz", + "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==", + "dev": true, + "dependencies": { + "ci-info": "^3.2.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, "node_modules/is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", @@ -10763,18 +10764,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/lerna/node_modules/is-ci": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-3.0.1.tgz", - "integrity": "sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==", - "dev": true, - "dependencies": { - "ci-info": "^3.2.0" - }, - "bin": { - "is-ci": "bin.js" - } - }, "node_modules/lerna/node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -13587,9 +13576,9 @@ } }, "node_modules/pacote/node_modules/npm-packlist": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-8.0.1.tgz", - "integrity": "sha512-MQpL27ZrsJQ2kiAuQPpZb5LtJwydNRnI15QWXsf3WHERu4rzjRj6Zju/My2fov7tLuu3Gle/uoIX/DDZ3u4O4Q==", + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-8.0.2.tgz", + "integrity": "sha512-shYrPFIS/JLP4oQmAwDyk5HcyysKW8/JLTEA32S0Z5TzvpaeeX2yMFfoK1fjEBnCBvVyIB/Jj/GBFdm0wsgzbA==", "dev": true, "dependencies": { "ignore-walk": "^6.0.4" @@ -13963,8 +13952,9 @@ } }, "node_modules/pino-abstract-transport/node_modules/readable-stream": { - "version": "4.4.2", - "license": "MIT", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -14037,8 +14027,9 @@ } }, "node_modules/pino-pretty/node_modules/readable-stream": { - "version": "4.4.2", - "license": "MIT", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -14051,9 +14042,9 @@ } }, "node_modules/pino-pretty/node_modules/sonic-boom": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.7.0.tgz", - "integrity": "sha512-IudtNvSqA/ObjN97tfgNmOKyDOs4dNcg4cUUsHDebqsgb8wGBBwb31LIgShNO8fye0dFI52X1+tFoKKI6Rq1Gg==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.0.tgz", + "integrity": "sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==", "dependencies": { "atomic-sleep": "^1.0.0" } @@ -16501,9 +16492,10 @@ "license": "Apache-2.0" }, "node_modules/tsconfig-paths": { - "version": "3.14.2", + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", + "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==", "dev": true, - "license": "MIT", "dependencies": { "@types/json5": "^0.0.29", "json5": "^1.0.2", @@ -17421,9 +17413,10 @@ } }, "node_modules/yargs-parser": { - "version": "20.2.4", + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, - "license": "ISC", "engines": { "node": ">=10" } @@ -17513,11 +17506,11 @@ }, "packages/meadowlark-authz-server": { "name": "@edfi/meadowlark-authz-server", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", @@ -17567,11 +17560,11 @@ }, "packages/meadowlark-core": { "name": "@edfi/meadowlark-core", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", @@ -17642,13 +17635,14 @@ "license": "MIT" }, "packages/meadowlark-core/node_modules/regenerator-runtime": { - "version": "0.14.0", - "dev": true, - "license": "MIT" + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "dev": true }, "packages/meadowlark-utilities": { "name": "@edfi/meadowlark-utilities", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { "pino": "^8.15.7", @@ -17656,16 +17650,16 @@ } }, "packages/meadowlark-utilities/node_modules/pino": { - "version": "8.16.2", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.16.2.tgz", - "integrity": "sha512-2advCDGVEvkKu9TTVSa/kWW7Z3htI/sBKEZpqiHk6ive0i/7f5b1rsU8jn0aimxqfnSz5bj/nOYkwhBUn5xxvg==", + "version": "8.17.2", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.17.2.tgz", + "integrity": "sha512-LA6qKgeDMLr2ux2y/YiUt47EfgQ+S9LznBWOJdN3q1dx2sv0ziDLUBeVpyVv17TEcGCBuWf0zNtg3M5m1NhhWQ==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "v1.1.0", "pino-std-serializers": "^6.0.0", - "process-warning": "^2.0.0", + "process-warning": "^3.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", @@ -17681,26 +17675,26 @@ "license": "MIT" }, "packages/meadowlark-utilities/node_modules/process-warning": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.3.2.tgz", - "integrity": "sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz", + "integrity": "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ==" }, "packages/meadowlark-utilities/node_modules/sonic-boom": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.7.0.tgz", - "integrity": "sha512-IudtNvSqA/ObjN97tfgNmOKyDOs4dNcg4cUUsHDebqsgb8wGBBwb31LIgShNO8fye0dFI52X1+tFoKKI6Rq1Gg==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.8.0.tgz", + "integrity": "sha512-ybz6OYOUjoQQCQ/i4LU8kaToD8ACtYP+Cj5qd2AO36bwbdewxWJ3ArmJ2cr6AvxlL2o0PqnCcPGUgkILbfkaCA==", "dependencies": { "atomic-sleep": "^1.0.0" } }, "services/meadowlark-fastify": { "name": "@edfi/meadowlark-fastify", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" @@ -17713,10 +17707,10 @@ }, "tests/e2e": { "name": "@edfi/meadowlark-e2e-tests", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "license": "Apache-2.0", "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/Meadowlark-js/packages/meadowlark-authz-server/package.json b/Meadowlark-js/packages/meadowlark-authz-server/package.json index 85d02dce..fd7d0dad 100644 --- a/Meadowlark-js/packages/meadowlark-authz-server/package.json +++ b/Meadowlark-js/packages/meadowlark-authz-server/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-authz-server", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark authorization server", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", diff --git a/Meadowlark-js/packages/meadowlark-core/package.json b/Meadowlark-js/packages/meadowlark-core/package.json index 727adc96..41fe83ec 100644 --- a/Meadowlark-js/packages/meadowlark-core/package.json +++ b/Meadowlark-js/packages/meadowlark-core/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-core", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark core functionality", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", diff --git a/Meadowlark-js/packages/meadowlark-utilities/package.json b/Meadowlark-js/packages/meadowlark-utilities/package.json index af41c4b8..51190e3c 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/package.json +++ b/Meadowlark-js/packages/meadowlark-utilities/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-utilities", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark shared utilities", "license": "Apache-2.0", "publishConfig": { diff --git a/Meadowlark-js/services/meadowlark-fastify/package.json b/Meadowlark-js/services/meadowlark-fastify/package.json index 2edec3d8..6bbf6c9f 100644 --- a/Meadowlark-js/services/meadowlark-fastify/package.json +++ b/Meadowlark-js/services/meadowlark-fastify/package.json @@ -1,6 +1,6 @@ { "name": "@edfi/meadowlark-fastify", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark service using Fastify", "license": "Apache-2.0", "publishConfig": { @@ -12,9 +12,9 @@ "/package.json" ], "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.8", - "@edfi/meadowlark-core": "0.4.1-pre.8", - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-authz-server": "0.4.1-pre.9", + "@edfi/meadowlark-core": "0.4.1-pre.9", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" diff --git a/Meadowlark-js/tests/e2e/package.json b/Meadowlark-js/tests/e2e/package.json index f289f8f0..95b5a2fa 100644 --- a/Meadowlark-js/tests/e2e/package.json +++ b/Meadowlark-js/tests/e2e/package.json @@ -1,13 +1,13 @@ { "name": "@edfi/meadowlark-e2e-tests", "main": "dist/index.js", - "version": "0.4.1-pre.8", + "version": "0.4.1-pre.9", "description": "Meadowlark Ed-Fi API end to end tests", "license": "Apache-2.0", "private": true, "files": [], "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.9", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 71475995..c3109599 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -103,8 +103,8 @@ To create a new key: | OAUTH_CLIENT_PROVIDED_TOKEN_CACHE_MAX_ENTRIES | 1000 | Control the size of the token cache | | IS_LOCAL | true | When true, prints plain text logs. Else prints structured JSON logs | | LOG_LEVEL | info | Options: error, warn, info, debug | -| SAVE_LOG_TO_FILE | false | When true, save log messages to a file in addition to writing to the console | -| LOG_FILE_LOCATION | /var/log/ | Location of log file if SAVE_LOG_TO_FILE is enabled | +| LOG_TO_FILE | false | When true, save log messages to a file in addition to writing to the console | +| LOG_FILE_LOCATION | /var/log/ | Location of log file if LOG_TO_FILE is enabled | | ALLOW_TYPE_COERCION | false | When true, numbers send as strings (e.g. "1") will be accepted in validation | | ALLOW__EXT_PROPERTY | false | When true, a payload may contain the `_ext` property without causing a validation error | | **BEGIN_ALLOWED_SCHOOL_YEAR** | 1900 | The beginning of the range of valid school years | diff --git a/docs/design/performance-benchmarking/SETUP-PERFORMANCE-SUITE.md b/docs/design/performance-benchmarking/SETUP-PERFORMANCE-SUITE.md index f43bd07a..5c4da75d 100644 --- a/docs/design/performance-benchmarking/SETUP-PERFORMANCE-SUITE.md +++ b/docs/design/performance-benchmarking/SETUP-PERFORMANCE-SUITE.md @@ -19,9 +19,10 @@ Types of testing that can run with these tools: Steps to run against meadowlark: - Configure Meadowlark and verify that it's running. -- Load Sample data with the Invoke-LoadGrandBend or Invoke-LoadPartialGrandBend - scripts (otherwise, the tests will fail without some expected descriptors and - education organization Ids). +- Load Sample data with the Invoke-LoadSouthridge, which is a smaller set of the + Northridge data set that is required by the Suite-3-Performance-Testing (This + can be done with the Invoke-LoadGrandBend or Invoke-LoadPartialGrandBend + scripts, although some data might be missing). - Verify that the data has been loaded correctly through the API or from the database. - Clone the diff --git a/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md b/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md index 15a1f856..ae66b473 100644 --- a/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md +++ b/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md @@ -45,6 +45,11 @@ performance, Pipeclean Tests and Volume Tests. > to have the changes required for Meadowlark. Vinaya and/or StephenF can verify when to remove > this temporary note and use the `main` branch again. +> [!NOTE] +> Suite-3-Performance-Tests runs against the Northridge dataset, we have a Southridge dataset script +> that uses a subset of the Northridge dataset. After running the performance tests, if there are +> any errors regarding missing data, insert the data into Meadowlark and run again. + - Follow steps to [setup performance suite](./SETUP-PERFORMANCE-SUITE.md). - Go to /src/edfi_performance_test folder and run `poetry install`. - Create a user in meadowlark with the role of `vendor`. Save key and secret. diff --git a/docs/performance-testing/load-balancing-API-scale-out.md b/docs/performance-testing/load-balancing-API-scale-out.md index bf709815..0fec57ae 100644 --- a/docs/performance-testing/load-balancing-API-scale-out.md +++ b/docs/performance-testing/load-balancing-API-scale-out.md @@ -73,7 +73,7 @@ END_ALLOWED_SCHOOL_YEAR=2035 ALLOW_TYPE_COERCION=true ALLOW__EXT_PROPERTY=true -SAVE_LOG_TO_FILE=true +LOG_TO_FILE=true LOG_FILE_LOCATION=c:/temp/ ``` diff --git a/docs/performance-testing/mongo-connection-pooling.md b/docs/performance-testing/mongo-connection-pooling.md index a3548ca8..4826a8f0 100644 --- a/docs/performance-testing/mongo-connection-pooling.md +++ b/docs/performance-testing/mongo-connection-pooling.md @@ -82,7 +82,7 @@ END_ALLOWED_SCHOOL_YEAR=2034 ALLOW_TYPE_COERCION=true ALLOW__EXT_PROPERTY=true -SAVE_LOG_TO_FILE=true +LOG_TO_FILE=true LOG_FILE_LOCATION=c:/temp/ ``` diff --git a/docs/performance-testing/two-functionalities-of-kafka-connect-separated.md b/docs/performance-testing/two-functionalities-of-kafka-connect-separated.md index 1e727655..4365dc66 100644 --- a/docs/performance-testing/two-functionalities-of-kafka-connect-separated.md +++ b/docs/performance-testing/two-functionalities-of-kafka-connect-separated.md @@ -76,7 +76,7 @@ END_ALLOWED_SCHOOL_YEAR=2035 ALLOW_TYPE_COERCION=true ALLOW__EXT_PROPERTY=true -SAVE_LOG_TO_FILE=true +LOG_TO_FILE=true LOG_FILE_LOCATION=c:/temp/ ``` diff --git a/eng/deploy/azure/README.md b/eng/deploy/azure/README.md index 0218190a..7cdf1cd9 100644 --- a/eng/deploy/azure/README.md +++ b/eng/deploy/azure/README.md @@ -131,7 +131,7 @@ $signingKey=$(az keyvault secret show --name "SigningKey" --vault-name "meadowla ### Enable Logging To save the logs to a file, for a summarized result, set the flag -`SAVE_LOG_TO_FILE` to true, which will create a `meadowlark.log` file with the +`LOG_TO_FILE` to true, which will create a `meadowlark.log` file with the logs. For a production deployment, it's recommended to send the logs to [Log