diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json index 1688d4a3..cecb0f04 100644 --- a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-elasticsearch-backend", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark backend plugin for elasticsearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, @@ -29,6 +29,6 @@ "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } } diff --git a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json index 656d1d6e..5eb5c8e8 100644 --- a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-mongodb-backend", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark backend plugin for MongoDB", "license": "Apache-2.0", "publishConfig": { @@ -19,11 +19,11 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "async-retry": "^1.3.3", - "mongodb": "^5.9.0", + "mongodb": "^5.9.2", "ramda": "0.29.1" }, "devDependencies": { diff --git a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json index b0e14946..bbdd958d 100644 --- a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-opensearch-backend", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark backend plugin for OpenSearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { @@ -28,6 +28,6 @@ "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } } diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json index e0f06bcf..3ccadd5d 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-postgresql-backend", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark backend plugin for PostgreSQL", "license": "Apache-2.0", "publishConfig": { @@ -19,20 +19,20 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" }, "devDependencies": { - "@testcontainers/postgresql": "^10.3.1", - "@types/pg": "^8.10.3", - "@types/pg-format": "^1.0.3", + "@testcontainers/postgresql": "^10.3.2", + "@types/pg": "^8.10.9", + "@types/pg-format": "^1.0.5", "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } } diff --git a/Meadowlark-js/lerna.json b/Meadowlark-js/lerna.json index f845c7bd..3cb27187 100644 --- a/Meadowlark-js/lerna.json +++ b/Meadowlark-js/lerna.json @@ -3,7 +3,7 @@ "packages": [ "packages/*" ], - "version": "0.4.0-pre.7", + "version": "0.4.0", "npmClient": "npm", "useWorkspaces": true } diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index 5ac124bf..f281aeda 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -16,10 +16,10 @@ "devDependencies": { "@faker-js/faker": "^8.1.0", "@shelf/jest-mongodb": "^4.1.7", - "@types/autocannon": "^7.12.1", - "@types/eslint": "^8.44.3", + "@types/autocannon": "^7.12.5", + "@types/eslint": "^8.44.8", "@types/jest": "^29.5.11", - "@types/node": "18.18.9", + "@types/node": "18.18.14", "@typescript-eslint/eslint-plugin": "5.62.0", "@typescript-eslint/parser": "5.62.0", "autocannon": "^7.12.0", @@ -40,23 +40,23 @@ "jest-environment-node": "^29.7.0", "kafkajs": "^2.2.4", "lerna": "^6.6.2", - "mongodb": "^5.9.0", + "mongodb": "^5.9.2", "pg": "^8.11.3", "pg-format": "^1.0.4", "prettier": "2.8.8", "rimraf": "^5.0.5", "ts-jest": "^29.1.1", - "ts-node": "10.9.1", + "ts-node": "10.9.2", "typescript": "4.9.5" } }, "backends/meadowlark-elasticsearch-backend": { "name": "@edfi/meadowlark-elasticsearch-backend", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, @@ -65,19 +65,19 @@ "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } }, "backends/meadowlark-mongodb-backend": { "name": "@edfi/meadowlark-mongodb-backend", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "async-retry": "^1.3.3", - "mongodb": "^5.9.0", + "mongodb": "^5.9.2", "ramda": "0.29.1" }, "devDependencies": { @@ -88,11 +88,11 @@ }, "backends/meadowlark-opensearch-backend": { "name": "@edfi/meadowlark-opensearch-backend", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { @@ -100,29 +100,29 @@ "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } }, "backends/meadowlark-postgresql-backend": { "name": "@edfi/meadowlark-postgresql-backend", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" }, "devDependencies": { - "@testcontainers/postgresql": "^10.3.1", - "@types/pg": "^8.10.3", - "@types/pg-format": "^1.0.3", + "@testcontainers/postgresql": "^10.3.2", + "@types/pg": "^8.10.9", + "@types/pg-format": "^1.0.5", "copyfiles": "^2.4.1", "dotenv": "^16.3.1", "rimraf": "^5.0.5", - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -1517,9 +1517,10 @@ } }, "node_modules/@clinic/heap-profiler/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dev": true, - "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1631,9 +1632,9 @@ "link": true }, "node_modules/@edfi/metaed-core": { - "version": "4.2.2-dev.15", - "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-core/-/metaed-core-4.2.2-dev.15.tgz", - "integrity": "sha1-1/k2MoNR22qZ5/+BC+cnIWixZvU=", + "version": "4.3.0", + "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-core/-/metaed-core-4.3.0.tgz", + "integrity": "sha1-2LyjL5KRw+yVgqNwSoUTmxmTf2g=", "dev": true, "license": "SEE LICENSE IN LICENSE.md", "dependencies": { @@ -1670,9 +1671,9 @@ } }, "node_modules/@edfi/metaed-core/node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dev": true, "dependencies": { "graceful-fs": "^4.2.0", @@ -1690,14 +1691,14 @@ "dev": true }, "node_modules/@edfi/metaed-plugin-edfi-api-schema": { - "version": "4.2.2-dev.15", - "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-plugin-edfi-api-schema/-/metaed-plugin-edfi-api-schema-4.2.2-dev.15.tgz", - "integrity": "sha1-R66wQZso1IlJiCV+VmpAxoKTgD8=", + "version": "4.3.0", + "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-plugin-edfi-api-schema/-/metaed-plugin-edfi-api-schema-4.3.0.tgz", + "integrity": "sha1-5VE4yA1MMZvBM/DZt20ZqB100I0=", "dev": true, "license": "Apache-2.0", "dependencies": { - "@edfi/metaed-core": "4.2.2-dev.15", - "@edfi/metaed-plugin-edfi-unified": "4.2.2-dev.15", + "@edfi/metaed-core": "4.3.0", + "@edfi/metaed-plugin-edfi-unified": "4.3.0", "ajv": "^8.12.0", "deep-freeze": "^0.0.1", "inflection": "^2.0.1", @@ -1725,13 +1726,13 @@ "license": "MIT" }, "node_modules/@edfi/metaed-plugin-edfi-unified": { - "version": "4.2.2-dev.15", - "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-plugin-edfi-unified/-/metaed-plugin-edfi-unified-4.2.2-dev.15.tgz", - "integrity": "sha1-8ZXvfBHDE3Je1CwQbzqAYlfGf1U=", + "version": "4.3.0", + "resolved": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/@edfi/metaed-plugin-edfi-unified/-/metaed-plugin-edfi-unified-4.3.0.tgz", + "integrity": "sha1-fsGk5IHo9pgmUQnf5UJEPC7YQJg=", "dev": true, "license": "SEE LICENSE IN LICENSE.md", "dependencies": { - "@edfi/metaed-core": "4.2.2-dev.15", + "@edfi/metaed-core": "4.3.0", "ramda": "^0.29.0" } }, @@ -2767,26 +2768,6 @@ "node": "*" } }, - "node_modules/@lerna/legacy-package-management/node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "dev": true, - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/@lerna/legacy-package-management/node_modules/pretty-format": { "version": "29.4.3", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.4.3.tgz", @@ -2837,12 +2818,6 @@ "node": ">=10" } }, - "node_modules/@lerna/legacy-package-management/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true - }, "node_modules/@lerna/legacy-package-management/node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -2852,22 +2827,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/@lerna/legacy-package-management/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true - }, - "node_modules/@lerna/legacy-package-management/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/@lerna/legacy-package-management/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -3104,9 +3063,9 @@ } }, "node_modules/@npmcli/arborist/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -3372,9 +3331,9 @@ } }, "node_modules/@npmcli/metavuln-calculator/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -3476,9 +3435,9 @@ } }, "node_modules/@npmcli/package-json/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -4701,12 +4660,12 @@ } }, "node_modules/@testcontainers/postgresql": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/@testcontainers/postgresql/-/postgresql-10.3.1.tgz", - "integrity": "sha512-WeWSJICEZgcfIoYG5wqK49wtsDZUpSMboqH037CJkmoCOjMTHxeq9LMcHuZpcSgTsTQgkV4cQdOv4dhH5kCIyw==", + "version": "10.3.2", + "resolved": "https://registry.npmjs.org/@testcontainers/postgresql/-/postgresql-10.3.2.tgz", + "integrity": "sha512-PhbmhWe+M5RF9QZGOzg/Vc+Ve6KlT/j9OLv9G11xubvcdhbuAz9Am3u1GjsXS7C1Vt/rwWx+j0kg+FtYwbJQng==", "dev": true, "dependencies": { - "testcontainers": "^10.3.1" + "testcontainers": "^10.3.2" } }, "node_modules/@tootallnate/once": { @@ -4785,9 +4744,9 @@ } }, "node_modules/@types/autocannon": { - "version": "7.12.1", - "resolved": "https://registry.npmjs.org/@types/autocannon/-/autocannon-7.12.1.tgz", - "integrity": "sha512-VV6P9f/P2lwz9EHEfATdlosZKjHzxoGCKkjTxs6aXJo45TMELrd+uwEfqY3yO1fmPedpFcqp4AA63whSlkVaUQ==", + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@types/autocannon/-/autocannon-7.12.5.tgz", + "integrity": "sha512-IGxnlq0ip0DTDOpShCDOjsBW+lclASdli1Jep+YSEOeUwylwyMai3WCVLi0mnXjHPD1cIziz3eiVyX6i7toB7Q==", "dev": true, "dependencies": { "@types/node": "*" @@ -4862,9 +4821,9 @@ } }, "node_modules/@types/eslint": { - "version": "8.44.4", - "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.4.tgz", - "integrity": "sha512-lOzjyfY/D9QR4hY9oblZ76B90MYTB3RrQ4z2vBIJKj9ROCRqdkYl2gSUx1x1a4IWPjKJZLL4Aw1Zfay7eMnmnA==", + "version": "8.44.8", + "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.44.8.tgz", + "integrity": "sha512-4K8GavROwhrYl2QXDXm0Rv9epkA8GBFu0EI+XrrnnuCl7u8CWBRusX7fXJfanhZTDWSAL24gDI/UqXyUM0Injw==", "dev": true, "dependencies": { "@types/estree": "*", @@ -4895,9 +4854,10 @@ } }, "node_modules/@types/inflection": { - "version": "1.13.0", - "dev": true, - "license": "MIT" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@types/inflection/-/inflection-1.13.2.tgz", + "integrity": "sha512-VxXY8dNLrxn7nDvsud77K60uD3a9RSmKfa0k/N/zvP2G55R5/8DSO5Ferz3mQdlAo8jPnpQLilCx9rABdPHSVg==", + "dev": true }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", @@ -4931,9 +4891,9 @@ } }, "node_modules/@types/js-yaml": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.7.tgz", - "integrity": "sha512-RJZP9WAMMr1514KbdSXkLRrKvYQacjr1+HWnY8pui/uBTBoSgD9ZGR17u/d4nb9NpERp0FkdLBe7hq8NIPBgkg==", + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", + "integrity": "sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==", "dev": true }, "node_modules/@types/json-schema": { @@ -4973,9 +4933,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.18.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.9.tgz", - "integrity": "sha512-0f5klcuImLnG4Qreu9hPj/rEfFq6YRc5n2mAjSsH+ec/mJL+3voBH0+8T7o8RpFjH7ovc+TRsL/c7OYIQsPTfQ==", + "version": "18.18.14", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.18.14.tgz", + "integrity": "sha512-iSOeNeXYNYNLLOMDSVPvIFojclvMZ/HDY2dU17kUlcsOsSQETbWIslJbYLZgA+ox8g2XQwSHKTkght1a5X26lQ==", "dependencies": { "undici-types": "~5.26.4" } @@ -4998,9 +4958,9 @@ "dev": true }, "node_modules/@types/pg": { - "version": "8.10.5", - "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.5.tgz", - "integrity": "sha512-GS3ebGcSJQqKSnq4/WnSH1XQvx0vTDLEmqLENk7onKvTnry9BWPsZiZeUMJlEPw+5bCQDzfxZFhxlUztpNCKgQ==", + "version": "8.10.9", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.9.tgz", + "integrity": "sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==", "dev": true, "dependencies": { "@types/node": "*", @@ -5009,9 +4969,9 @@ } }, "node_modules/@types/pg-format": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@types/pg-format/-/pg-format-1.0.3.tgz", - "integrity": "sha512-sg8ptDhnqPt9uKJOYB6UGMfhf3/SJ1diE/1epDmIsgSn2cwqWdukyWbErfEM1NADf3qWpy9HizWl5U/akoe4KQ==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/pg-format/-/pg-format-1.0.5.tgz", + "integrity": "sha512-i+oEEJEC+1I3XAhgqtVp45Faj8MBbV0Aoq4rHsHD7avgLjyDkaWKObd514g0Q/DOUkdxU0P4CQ0iq2KR4SoJcw==", "dev": true }, "node_modules/@types/pg/node_modules/pg-types": { @@ -5067,9 +5027,9 @@ } }, "node_modules/@types/pluralize": { - "version": "0.0.31", - "resolved": "https://registry.npmjs.org/@types/pluralize/-/pluralize-0.0.31.tgz", - "integrity": "sha512-MQh69PPwFlYAL2qz/Mw5Zc34VTdt7pTck0Xbb6pbPSzdt5oaLB87iyJJxEMS5Dco/s7lXHunEezAvQurZZdrsQ==", + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/pluralize/-/pluralize-0.0.33.tgz", + "integrity": "sha512-JOqsl+ZoCpP4e8TDke9W79FDcSgPAR0l6pixx2JHkhnRjvShyYiAYw2LVsnA7K08Y6DeOnaU6ujmENO4os/cYg==", "dev": true }, "node_modules/@types/ramda": { @@ -5087,9 +5047,9 @@ "license": "MIT" }, "node_modules/@types/semver": { - "version": "7.5.3", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.3.tgz", - "integrity": "sha512-OxepLK9EuNEIPxWNME+C6WwbRAOOI2o2BaQEGzz5Lu2e4Z5eDnEo+/aVEDMIXywoJitJ7xWd641wrGLZdtwRyw==", + "version": "7.5.6", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.6.tgz", + "integrity": "sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==", "dev": true }, "node_modules/@types/ssh2": { @@ -5143,9 +5103,9 @@ "license": "MIT" }, "node_modules/@types/uuid": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz", - "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==", + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", + "integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==", "dev": true }, "node_modules/@types/webgl-ext": { @@ -5166,9 +5126,10 @@ } }, "node_modules/@types/xml2js": { - "version": "0.4.12", + "version": "0.4.14", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.14.tgz", + "integrity": "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ==", "dev": true, - "license": "MIT", "dependencies": { "@types/node": "*" } @@ -5576,9 +5537,9 @@ } }, "node_modules/acorn-walk": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.0.tgz", - "integrity": "sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA==", + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.1.tgz", + "integrity": "sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==", "dev": true, "engines": { "node": ">=0.4.0" @@ -6202,9 +6163,9 @@ } }, "node_modules/axios-retry": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.8.0.tgz", - "integrity": "sha512-CfIsQyWNc5/AE7x/UEReRUadiBmQeoBpSEC+4QyGLJMswTsP1tz0GW2YYPnE7w9+ESMef5zOgLDFpHynNyEZ1w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.9.1.tgz", + "integrity": "sha512-8PJDLJv7qTTMMwdnbMvrLYuvB47M81wRtxQmEdV5w4rgbTXTt+vtPkXwajOfOdSyv/wZICJOC+/UhXH4aQ/R+w==", "dependencies": { "@babel/runtime": "^7.15.4", "is-retry-allowed": "^2.2.0" @@ -10809,9 +10770,9 @@ } }, "node_modules/globals": { - "version": "13.23.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", - "integrity": "sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA==", + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, "dependencies": { "type-fest": "^0.20.2" @@ -13917,26 +13878,6 @@ "node": ">=8" } }, - "node_modules/lerna/node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "dev": true, - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/lerna/node_modules/resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", @@ -13964,12 +13905,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/lerna/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true - }, "node_modules/lerna/node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -13979,22 +13914,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/lerna/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true - }, - "node_modules/lerna/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/lerna/node_modules/write-file-atomic": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.1.tgz", @@ -15558,9 +15477,9 @@ } }, "node_modules/mongodb": { - "version": "5.9.1", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.1.tgz", - "integrity": "sha512-NBGA8AfJxGPeB12F73xXwozt8ZpeIPmCUeWRwl9xejozTXFes/3zaep9zhzs1B/nKKsw4P3I4iPfXl3K7s6g+Q==", + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", + "integrity": "sha512-H60HecKO4Bc+7dhOv4sJlgvenK4fQNqqUIlXxZYQNbfEWSALGAwGoyJd/0Qwk4TttFXUOHJ2ZJQe/52ScaUwtQ==", "dependencies": { "bson": "^5.5.0", "mongodb-connection-string-url": "^2.6.0", @@ -15667,9 +15586,10 @@ } }, "node_modules/mongodb-memory-server-core/node_modules/mongodb": { - "version": "4.17.1", + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.17.2.tgz", + "integrity": "sha512-mLV7SEiov2LHleRJPMPrK2PMyhXFZt2UQLC4VD4pnth3jMjYKHhtqfwwkkvS/NXuo/Fp3vbhaNcXrIDaLRb9Tg==", "dev": true, - "license": "Apache-2.0", "dependencies": { "bson": "^4.7.2", "mongodb-connection-string-url": "^2.6.0", @@ -16077,9 +15997,9 @@ "dev": true }, "node_modules/node-fetch": { - "version": "2.6.13", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz", - "integrity": "sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==", + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", "dev": true, "dependencies": { "whatwg-url": "^5.0.0" @@ -16691,9 +16611,9 @@ } }, "node_modules/nx/node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dev": true, "dependencies": { "graceful-fs": "^4.2.0", @@ -17372,9 +17292,9 @@ } }, "node_modules/pacote/node_modules/ignore-walk": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-6.0.3.tgz", - "integrity": "sha512-C7FfFoTA+bI10qfeydT8aZbvr91vAEU+2W5BZUlzPec47oNb07SsOfwYrtxuvOYdUApPP/Qlh4DtAO51Ekk2QA==", + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-6.0.4.tgz", + "integrity": "sha512-t7sv42WkwFkyKbivUCglsQW5YWMskWtbEf4MNKX5u/CCWHKSPzN4FtBQGsQZgCLbxOzpVlcbWVK5KB3auIOjSw==", "dev": true, "dependencies": { "minimatch": "^9.0.0" @@ -17384,9 +17304,9 @@ } }, "node_modules/pacote/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -17557,9 +17477,9 @@ } }, "node_modules/parse-conflict-json/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -17660,13 +17580,10 @@ } }, "node_modules/path-scurry/node_modules/lru-cache": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.0.2.tgz", - "integrity": "sha512-Yj9mA8fPiVgOUpByoTZO5pNrcl5Yk37FcSHsUINpAsaBIEZIuqcCclDZJCVxqQShDsmYX8QG63svJiTbOATZwg==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.1.0.tgz", + "integrity": "sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==", "dev": true, - "dependencies": { - "semver": "^7.3.5" - }, "engines": { "node": "14 || >=16.14" } @@ -18786,9 +18703,9 @@ } }, "node_modules/read-package-json-fast/node_modules/json-parse-even-better-errors": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.0.tgz", - "integrity": "sha512-iZbGHafX/59r39gPwVPRBGw0QQKnA7tte5pSMrhWOW7swGsVvVTjmfyAV9pNqk8YGT7tRCdxRu8uzcgZwoDooA==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-3.0.1.tgz", + "integrity": "sha512-aatBvbL26wVUCLmbWdCpeu9iF5wOyWpagiKkInA+kfws3sWdBrTnsvN2CKcyCYyUrc7rebNBlK6+kteg7ksecg==", "dev": true, "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" @@ -21061,9 +20978,9 @@ } }, "node_modules/testcontainers": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.3.1.tgz", - "integrity": "sha512-w7WgoAD8p2Hd5gE6CluWd3EBg7qb6u/JcKJGl0TTs/Cdq326Fsfw1ARnHjk4UzYEsp3XZ5YtvgY7bl5E3iudfw==", + "version": "10.3.2", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.3.2.tgz", + "integrity": "sha512-IsV6NgS5reHcVF1nJLCDJv1hM9gAWUhLwh9b3ybgzvM3X7T2dcmuLFKt1RAR8qN8k+44tW2Drj7idxW6oeGvvg==", "dev": true, "dependencies": { "@balena/dockerignore": "^1.0.2", @@ -21475,9 +21392,10 @@ } }, "node_modules/ts-node": { - "version": "10.9.1", + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, - "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -22677,11 +22595,11 @@ }, "packages/meadowlark-authz-server": { "name": "@edfi/meadowlark-authz-server", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", @@ -22691,7 +22609,7 @@ "uuid": "^9.0.1" }, "devDependencies": { - "@types/uuid": "^9.0.4", + "@types/uuid": "^9.0.7", "copyfiles": "^2.4.1", "rimraf": "^5.0.5" } @@ -22731,22 +22649,22 @@ }, "packages/meadowlark-core": { "name": "@edfi/meadowlark-core", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", "axios": "^1.5.1", - "axios-retry": "^3.8.0", + "axios-retry": "^3.8.2", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", "emittery": "^0.13.1", "fast-memoize": "^2.5.2", "fs-extra": "^11.1.1", - "inquirer": "^9.2.11", + "inquirer": "^9.2.12", "jsonpath-plus": "^7.2.0", "njwt": "^2.0.0", "ramda": "0.29.1", @@ -22756,16 +22674,16 @@ "xml2js": "^0.6.2" }, "devDependencies": { - "@edfi/metaed-core": "^4.2.2-dev.15", - "@edfi/metaed-plugin-edfi-api-schema": "^4.2.2-dev.15", - "@edfi/metaed-plugin-edfi-unified": "^4.2.2-dev.15", - "@types/fs-extra": "^11.0.2", - "@types/inflection": "^1.13.0", - "@types/js-yaml": "^4.0.6", - "@types/pluralize": "^0.0.31", + "@edfi/metaed-core": "^4.3.0", + "@edfi/metaed-plugin-edfi-api-schema": "^4.3.0", + "@edfi/metaed-plugin-edfi-unified": "^4.3.0", + "@types/fs-extra": "^11.0.4", + "@types/inflection": "^1.13.2", + "@types/js-yaml": "^4.0.9", + "@types/pluralize": "^0.0.33", "@types/ramda": "0.29.5", - "@types/semver": "^7.5.3", - "@types/xml2js": "^0.4.12", + "@types/semver": "^7.5.6", + "@types/xml2js": "^0.4.14", "copyfiles": "^2.4.1", "jsinspect": "^0.12.7", "regenerator-runtime": "^0.14.0", @@ -22802,8 +22720,9 @@ } }, "packages/meadowlark-core/node_modules/fs-extra": { - "version": "11.1.1", - "license": "MIT", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -22824,10 +22743,10 @@ }, "packages/meadowlark-utilities": { "name": "@edfi/meadowlark-utilities", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "pino": "^8.15.4", + "pino": "^8.15.7", "pino-pretty": "^10.2.3" } }, @@ -22857,9 +22776,9 @@ "license": "MIT" }, "packages/meadowlark-utilities/node_modules/process-warning": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.3.0.tgz", - "integrity": "sha512-N6mp1+2jpQr3oCFMz6SeHRGbv6Slb20bRhj4v3xR99HqNToAcOe1MFOp4tytyzOfJn+QtN8Rf7U/h2KAn4kC6g==" + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.3.2.tgz", + "integrity": "sha512-n9wh8tvBe5sFmsqlg+XQhaQLumwpqoAUruLwjCopgTmUBjJ/fjtBsJzKleCaIGBOMXYEhp1YfKl4d7rJ5ZKJGA==" }, "packages/meadowlark-utilities/node_modules/sonic-boom": { "version": "3.7.0", @@ -22871,12 +22790,12 @@ }, "services/meadowlark-fastify": { "name": "@edfi/meadowlark-fastify", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" @@ -22884,15 +22803,15 @@ "devDependencies": { "copyfiles": "^2.4.1", "rimraf": "^5.0.5", - "ts-node": "^10.9.1" + "ts-node": "^10.9.2" } }, "tests/e2e": { "name": "@edfi/meadowlark-e2e-tests", - "version": "0.4.0-pre.7", + "version": "0.4.0", "license": "Apache-2.0", "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", @@ -22907,9 +22826,10 @@ } }, "tests/e2e/node_modules/fs-extra": { - "version": "11.1.1", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dev": true, - "license": "MIT", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", diff --git a/Meadowlark-js/package.json b/Meadowlark-js/package.json index a918ca7d..2d004989 100644 --- a/Meadowlark-js/package.json +++ b/Meadowlark-js/package.json @@ -5,10 +5,10 @@ "devDependencies": { "@faker-js/faker": "^8.1.0", "@shelf/jest-mongodb": "^4.1.7", - "@types/autocannon": "^7.12.1", - "@types/eslint": "^8.44.3", + "@types/autocannon": "^7.12.5", + "@types/eslint": "^8.44.8", "@types/jest": "^29.5.11", - "@types/node": "18.18.9", + "@types/node": "18.18.14", "@typescript-eslint/eslint-plugin": "5.62.0", "@typescript-eslint/parser": "5.62.0", "autocannon": "^7.12.0", @@ -29,13 +29,13 @@ "jest-environment-node": "^29.7.0", "kafkajs": "^2.2.4", "lerna": "^6.6.2", - "mongodb": "^5.9.0", + "mongodb": "^5.9.2", "pg": "^8.11.3", "pg-format": "^1.0.4", "prettier": "2.8.8", "rimraf": "^5.0.5", "ts-jest": "^29.1.1", - "ts-node": "10.9.1", + "ts-node": "10.9.2", "typescript": "4.9.5" }, "workspaces": [ @@ -73,14 +73,14 @@ "start:local": "lerna run start:local", "docker:lint": "cat ./Dockerfile | docker run --rm -i hadolint/hadolint", "docker:install": "npm i lerna@^6.6.2 rimraf@^5.0.5 copyfiles@^2.4.1 typescript@4.9.5 -g && npm ci --only=production", - "docker:build": "docker build -t meadowlark .", - "docker:build:wsl1": "docker.exe build -t meadowlark .", - "docker:start": "docker run -d --env-file ./.env-docker --name ml-api -p 3000:3000 meadowlark", - "docker:start:wsl1": "docker.exe run -d --env-file ./.env-docker --name ml-api -p 3000:3000 meadowlark", + "docker:build": "docker build -t meadowlark-api .", + "docker:build:wsl1": "docker.exe build -t meadowlark-api .", + "docker:start": "docker run -d --env-file ./.env-docker --name ml-api -p 3000:3000 meadowlark-api", + "docker:start:wsl1": "docker.exe run -d --env-file ./.env-docker --name ml-api -p 3000:3000 meadowlark-api", "docker:halt": "docker stop ml-api && docker rm ml-api", "docker:halt:wsl1": "docker.exe stop ml-api && docker rm ml-api", - "docker:debug": "docker run -d --env-file ./.env-docker -p 3000:3000 --name ml-api meadowlark tail -F /dev/null && docker exec -it ml-api bash", - "docker:debug:wsl1": "docker.exe run -d --env-file ./.env-docker -p 3000:3000 --name ml-api meadowlark tail -F /dev/null && docker exec -it ml-api bash", + "docker:debug": "docker run -d --env-file ./.env-docker -p 3000:3000 --name ml-api meadowlark-api tail -F /dev/null && docker exec -it ml-api bash", + "docker:debug:wsl1": "docker.exe run -d --env-file ./.env-docker -p 3000:3000 --name ml-api meadowlark-api tail -F /dev/null && docker exec -it ml-api bash", "compose:up": "docker compose --env-file .env-docker up -d", "compose:up:wsl1": "docker.exe compose --env-file .env-docker up -d", "compose:down": "docker compose --env-file .env-docker down", diff --git a/Meadowlark-js/packages/meadowlark-authz-server/package.json b/Meadowlark-js/packages/meadowlark-authz-server/package.json index 5244f773..c7119e90 100644 --- a/Meadowlark-js/packages/meadowlark-authz-server/package.json +++ b/Meadowlark-js/packages/meadowlark-authz-server/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-authz-server", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark authorization server", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", @@ -24,7 +24,7 @@ "uuid": "^9.0.1" }, "devDependencies": { - "@types/uuid": "^9.0.4", + "@types/uuid": "^9.0.7", "copyfiles": "^2.4.1", "rimraf": "^5.0.5" }, diff --git a/Meadowlark-js/packages/meadowlark-core/package.json b/Meadowlark-js/packages/meadowlark-core/package.json index 0a0b8544..ebe5ad11 100644 --- a/Meadowlark-js/packages/meadowlark-core/package.json +++ b/Meadowlark-js/packages/meadowlark-core/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-core", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark core functionality", "license": "Apache-2.0", "publishConfig": { @@ -14,18 +14,18 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", "axios": "^1.5.1", - "axios-retry": "^3.8.0", + "axios-retry": "^3.8.2", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", "emittery": "^0.13.1", "fast-memoize": "^2.5.2", "fs-extra": "^11.1.1", - "inquirer": "^9.2.11", + "inquirer": "^9.2.12", "jsonpath-plus": "^7.2.0", "njwt": "^2.0.0", "ramda": "0.29.1", @@ -35,16 +35,16 @@ "xml2js": "^0.6.2" }, "devDependencies": { - "@edfi/metaed-core": "^4.2.2-dev.15", - "@edfi/metaed-plugin-edfi-api-schema": "^4.2.2-dev.15", - "@edfi/metaed-plugin-edfi-unified": "^4.2.2-dev.15", - "@types/fs-extra": "^11.0.2", - "@types/inflection": "^1.13.0", - "@types/js-yaml": "^4.0.6", - "@types/pluralize": "^0.0.31", + "@edfi/metaed-core": "^4.3.0", + "@edfi/metaed-plugin-edfi-api-schema": "^4.3.0", + "@edfi/metaed-plugin-edfi-unified": "^4.3.0", + "@types/fs-extra": "^11.0.4", + "@types/inflection": "^1.13.2", + "@types/js-yaml": "^4.0.9", + "@types/pluralize": "^0.0.33", "@types/ramda": "0.29.5", - "@types/semver": "^7.5.3", - "@types/xml2js": "^0.4.12", + "@types/semver": "^7.5.6", + "@types/xml2js": "^0.4.14", "copyfiles": "^2.4.1", "jsinspect": "^0.12.7", "regenerator-runtime": "^0.14.0", diff --git a/Meadowlark-js/packages/meadowlark-utilities/package.json b/Meadowlark-js/packages/meadowlark-utilities/package.json index 8abb767d..22dc468a 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/package.json +++ b/Meadowlark-js/packages/meadowlark-utilities/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-utilities", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark shared utilities", "license": "Apache-2.0", "publishConfig": { @@ -19,7 +19,7 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "pino": "^8.15.4", + "pino": "^8.15.7", "pino-pretty": "^10.2.3" } } diff --git a/Meadowlark-js/services/meadowlark-fastify/package.json b/Meadowlark-js/services/meadowlark-fastify/package.json index 6b2ad960..4460b059 100644 --- a/Meadowlark-js/services/meadowlark-fastify/package.json +++ b/Meadowlark-js/services/meadowlark-fastify/package.json @@ -1,6 +1,6 @@ { "name": "@edfi/meadowlark-fastify", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark service using Fastify", "license": "Apache-2.0", "publishConfig": { @@ -12,9 +12,9 @@ "/package.json" ], "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.0-pre.7", - "@edfi/meadowlark-core": "0.4.0-pre.7", - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-authz-server": "0.4.0", + "@edfi/meadowlark-core": "0.4.0", + "@edfi/meadowlark-utilities": "0.4.0", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" @@ -22,7 +22,7 @@ "devDependencies": { "copyfiles": "^2.4.1", "rimraf": "^5.0.5", - "ts-node": "^10.9.1" + "ts-node": "^10.9.2" }, "scripts": { "build": "npm run build:clean && npm run build:copy-non-ts && npm run build:dist", diff --git a/Meadowlark-js/services/meadowlark-fastify/readme.md b/Meadowlark-js/services/meadowlark-fastify/readme.md index 476eacc1..d42621e6 100644 --- a/Meadowlark-js/services/meadowlark-fastify/readme.md +++ b/Meadowlark-js/services/meadowlark-fastify/readme.md @@ -12,15 +12,20 @@ Instructions for running a local "developer" environment: * If using MongoDB set the value to `@edfi/meadowlark-mongodb-backend` * Update `MONGO_URI` to the location of your Mongo Instance * If running MongoDB via the local [docker compose configuration](../../docker-compose.yml), - the default value for `MONGO_URI` is + the default value for `MONGO_URI` is `mongodb://mongo:abcdefgh1!@mongo1:27017,mongo2:27018,mongo3:27019/?replicaSet=rs0` * If using PostgreSQL set the value to `@edfi/meadowlark-postgresql-backend` * Update `POSTGRES_USERNAME` and `POSTGRES_PASSWORD` * If your PostgreSQL instance is not running on the default port (5432), you can set `POSTGRES_PORT` to the port PostgreSQL is using - * `QUERY_HANDLER_PLUGIN` and `LISTENER1_PLUGIN` - Uncomment these two for GET query support using OpenSearch - * `AUTHORIZATION_STORE_PLUGIN` This is the plugin for the authorization store, it should be set to a NPM package name - * To use the Meadowlark-js provided Authorization plugin, this should be set to `@edfi/meadowlark-mongodb-backend` + * `QUERY_HANDLER_PLUGIN` and `LISTENER1_PLUGIN` - The query handler and listener that Meadowlark will use. + * If using OpenSearch set the value to `@edfi/meadowlark-opensearch-backend` + * Set the `OPENSEARCH_ENDPOINT`, `OPENSEARCH_USERNAME` and `OPENSEARCH_PASSWORD` + * If using ElasticSearch set the value to `@edfi/meadowlark-elasticsearch-backend` + * Set the `ELASTICSEARCH_ENDPOINT` + * `AUTHORIZATION_STORE_PLUGIN` This is the plugin for the authorization store + * If using MongoDB set the value to `@edfi/meadowlark-mongodb-backend` + * If using PostgreSQL set the value to `@edfi/meadowlark-postgresql-backend` * `OAUTH_SIGNING_KEY` - The `OAUTH_SIGNING_KEY` may need quotation marks around the value, unlike other keys. * Logging configuration, as this can affect performance. For better performance, avoid `debug` and `info` levels for `LOG_LEVEL`, and set `LOG_PRETTY_PRINT` to false. Rather than log to console, use `LOG_TO_FILE` set to true and set the log file directory with `LOG_FILE_LOCATION`. 1. If you have already built the Meadowlark-js project, you can skip to step 4, otherwise: diff --git a/Meadowlark-js/tests/e2e/package.json b/Meadowlark-js/tests/e2e/package.json index 4c732fa4..59db0e51 100644 --- a/Meadowlark-js/tests/e2e/package.json +++ b/Meadowlark-js/tests/e2e/package.json @@ -1,13 +1,13 @@ { "name": "@edfi/meadowlark-e2e-tests", "main": "dist/index.js", - "version": "0.4.0-pre.7", + "version": "0.4.0", "description": "Meadowlark Ed-Fi API end to end tests", "license": "Apache-2.0", "private": true, "files": [], "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.0-pre.7", + "@edfi/meadowlark-utilities": "0.4.0", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/Meadowlark-js/tests/e2e/readme.md b/Meadowlark-js/tests/e2e/readme.md index 3a2861b7..d6f2c6f5 100644 --- a/Meadowlark-js/tests/e2e/readme.md +++ b/Meadowlark-js/tests/e2e/readme.md @@ -1,64 +1,86 @@ # Running End to End tests -To run the tests: - -1. Create a .env file based on .env.example in this folder. -2. Verify that Docker is running -3. There are two ways to run the e2e tests: - - To create the meadowlark docker image and run the tests against that image, run `npm run test:e2e:build` - - To use a previously built image (can be built once with `npm run docker:build`), run `npm run test:e2e` - -## Support - -Postgresql support is currently disabled and should be included again once Postgresql support is matched on main code. +To run the tests, select one of the available modes ## Modes of running End to End tests ### Local Mode -This will run the end to end tests against your existing local environment. This will not load any new environment variables -and use the ones defined for fastify. +This will run the end to end tests against your existing local environment. This +will not load any new environment variables and use the ones defined for +[fastify](../../services/meadowlark-fastify/readme.md). + +Run `npm run test:e2e:jest:local` to run the tests using the existing local +environment. This will read the .env file inside the services/meadowlark-fastify +folder instead of the e2e environment. + +This allows to do changes to the running service and see the changes +immediately, as well as debugging the local Meadowlark code from VSCode by +setting breakpoints and running with the included jest VSCode debugger. > [!NOTE] -> This mode is recommended when doing new development into Meadowlark and you want to test your changes. +> This mode is recommended when doing new development into Meadowlark and you +> want to test your changes. > [!WARNING] -> If the tests does not have a correct teardown step or if there was an error cleaning the database, you will have to -> manually clean your databases and/or indexes to avoid unexpected behavior. +> +> * This requires the current environment to be already running. +> * If the tests does not have a correct teardown step or if there was an error +> cleaning the database, you will have to manually clean your databases and/or +> indexes to avoid unexpected behavior. ### Isolated mode -This will run the end to end tests with [TestContainers](https://node.testcontainers.org/). It will automatically setup the -containers to run the tests against and will delete the containers afterwards. +This will run the end to end tests with +[TestContainers](https://node.testcontainers.org/). It will automatically setup +the containers to run the tests against and will delete the containers +afterwards. + +* To setup, create a .env-e2e file in the same location as the + [example](./setup/.env-e2e.example) with the required values +* Run `npm run test:e2e:build` to build the meadowlark Docker image +* Verify that the environment variable `DEVELOPER_MODE` is not set to true +* Run the tests in isolated mode `npm run test:e2e:jest:tc` + +#### Using published image -- To setup, create a .env-e2e file in the same location as the [example](./setup/.env-e2e.example) with the required values. -- Run `test:e2e:build` to build the code and then run the tests in isolated mode. +If you want to run the tests against and existing Docker image, set the variable +`API_IMAGE_NAME` in the .env file. Run the tests with `npm run +test:e2e:jest:tc`. This will pull the image and run the tests against that image. > [!NOTE] -> This mode is recommended when exploring or verifying an installation, or verifying that the tests are passing correctly in -> the local environment. +> This mode is recommended when exploring or verifying an installation, or +> verifying that the tests are passing correctly in the local environment. > [!WARNING] -> This mode will run against a compiled version of the code, so after making changes in the Meadowlark code (not the tests), -> you must build the code again. This mode ignores all values in the different .env files other than the env-e2e file. +> This mode will run against a compiled version of the code, so after making +> changes in the Meadowlark code (not the tests), you must build the code again. +> This mode ignores all values in the different .env files other than the +> env-e2e file. ### Developer Mode -This is a special mode that will setup a group of containers in different ports (denoted with the names -test) and will not -delete the containers after each run. +This is a special mode that will setup a group of containers in different ports +(denoted with the names -test) and will not delete the containers after each +run. -- To setup, run: `npm run test:e2e:dev:setup`. This will configure the test containers +- To setup, run: `npm run test:e2e:dev:setup`. This will configure the test + containers - Set the environment variable `DEVELOPER_MODE=true` -- Run `npm run test:e2e:jest:tc` to run the tests. Save the Admin Key and Secret (as specified in -[.env.example](./setup/.env.example)) to be able to run the tests without cleaning the environment -- When done, run: `npm run test:e2e:dev:exit` or execute the `exit-dev-containers.ps1` script to clean the environment. +- Run `npm run test:e2e:jest:tc` to run the tests +- Save the Admin Key and Secret (as specified in +[.env-e2e.example](./setup/.env-e2e.example)) to be able to run the tests +without cleaning the environment +- When done, run: `npm run test:e2e:dev:exit` or execute the + `exit-dev-containers.ps1` script to clean the environment. > [!NOTE] > This mode is recommended when adding new end to ends tests. > [!WARNING] -> When running side to side with a "real" installation of Meadowlark, verify the ports being used to make sure that you are -> running against the TestContainers and not the real ones. +> When running side to side with a "real" installation of Meadowlark, verify the +> ports being used to make sure that you are running against the TestContainers +> and not the real ones. ## Troubleshooting @@ -70,13 +92,15 @@ Review the step 3 and check that the image is being generated Verify that Docker is up and running -:warning: When running on Windows, there might be some issues related to Docker and WSL. :warning: +:warning: When running on Windows, there might be some issues related to Docker +and WSL. :warning: `Error: docker endpoint for "default" not found` -This is an issue with Docker Compose and WSL 2, see [full discussion](https://github.com/docker/compose/issues/9956). To fix, -go to C:\Users\your-username and delete the .docker folder. Restart Docker and run the tests again. This does not affects -your running containers or configuration. +This is an issue with Docker Compose and WSL 2, see [full +discussion](https://github.com/docker/compose/issues/9956). To fix, go to +C:\Users\your-username and delete the .docker folder. Restart Docker and run the +tests again. This does not affects your running containers or configuration. `Error: (HTTP code 500) server error - i/o timeout` diff --git a/Meadowlark-js/tests/e2e/setup/containers/ApiContainer.ts b/Meadowlark-js/tests/e2e/setup/containers/ApiContainer.ts index 1fff707f..4cba4b37 100644 --- a/Meadowlark-js/tests/e2e/setup/containers/ApiContainer.ts +++ b/Meadowlark-js/tests/e2e/setup/containers/ApiContainer.ts @@ -13,10 +13,12 @@ export async function setup(network: StartedNetwork) { const fastifyPort = parseInt(process.env.FASTIFY_PORT ?? '3001', 10); - console.info(`Configuring Meadowlark API with docker image: ${process.env.API_IMAGE_NAME ?? 'meadowlark'}`); + const meadowlarkApiImageName = process.env.API_IMAGE_NAME ?? 'meadowlark-api'; + + console.info(`Configuring Meadowlark API with docker image: ${meadowlarkApiImageName}`); try { - container = new GenericContainer(process.env.API_IMAGE_NAME ?? 'meadowlark') + container = new GenericContainer(meadowlarkApiImageName) .withName('meadowlark-api-test') .withNetwork(network) .withLogConsumer(async (stream) => setAPILog(stream)) diff --git a/README.md b/README.md index b3997979..90118432 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,17 @@ src="https://raw.githubusercontent.com/Ed-Fi-Exchange-OSS/Meadowlark/main/images/cropped-meadowlark-cc-by-nc-4.0-naturenerd_joel.png" align="right" width="300"> -Project Meadowlark is a research and development effort to explore potential for -use of new technologies, including managed cloud services, for starting up a -"cloud native" Ed-Fi compatible API. - -While it was originally intended as a proof-of-concept, it is now _moving -toward_ production readiness. [Milestone -0.3.0](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/releases/tag/v0.3.0) has -been released with Docker and real OAuth2 support. Next steps will be to take -feedback from pilot tests, flesh out the PostgreSQL support, tune performance, -and work toward full streaming support. - -See [Project Meadowlark - Exploring Next Generation -Technologies](https://techdocs.ed-fi.org/x/RwJqBw) in Tech Docs for more +Project Meadowlark is a research and development effort to explore potential for use of new technologies, including managed +cloud services, for starting up a "cloud native" Ed-Fi compatible API. + + +- [Milestone 0.3.0](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/releases/tag/v0.3.0) has been released with Docker and +real OAuth2 support. + +- [Milestone 0.4.0](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/releases/tag/v0.4.0) includes full PostgreSQL support, +load balancer support with NGINX, instructions to use Kafka and performance evaluation. + +See [Project Meadowlark - Exploring Next Generation Technologies](https://techdocs.ed-fi.org/x/RwJqBw) in Tech Docs for more information on the background and design decisions for this project. ## Getting Started @@ -26,10 +24,10 @@ information on the background and design decisions for this project. * [Data Authorization](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/DATA-AUTHORIZATION.md) * [Developer getting started notes](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/README.md) * [Additional technical details](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/TECHNICAL.md) -* [Docker for Local Meadowlark Development](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/DOCKER-LOCAL-DEV.md) +* [Docker for Local Meadowlark + Development](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/DOCKER-LOCAL-DEV.md) * [How to Submit an Issue](https://techdocs.ed-fi.org/x/Y8uIBg) (Tech Docs) -* [How Submit a Feature Request](https://techdocs.ed-fi.org/x/0YADAQ) (Tech - Docs) +* [How Submit a Feature Request](https://techdocs.ed-fi.org/x/0YADAQ) (Tech Docs) ## Deployment and Operations @@ -39,30 +37,28 @@ information on the background and design decisions for this project. ## Contributing -The Ed-Fi Alliance welcomes code contributions from the community. Please read -the [Contributor's Guide](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/CONTRIBUTING.md) -for detailed information on how to contribute source code. +The Ed-Fi Alliance welcomes code contributions from the community. Please read the [Contributor's +Guide](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/docs/CONTRIBUTING.md) for detailed information on how to +contribute source code. Looking for an easy way to get started? Review [Meadowlark up-for-grabs -tickets](https://tracker.ed-fi.org/issues/?filter=15400) for a list of useful -tickets that are currently of low priority to the core development team, and -thus won't be addressed in upcoming sprints. +tickets](https://tracker.ed-fi.org/issues/?filter=15400) for a list of useful tickets that are currently of low priority to +the core development team, and thus won't be addressed in upcoming sprints. ## Legal Information Copyright (c) 2023 Ed-Fi Alliance, LLC and contributors. -Licensed under the [Apache License, Version 2.0](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/LICENSE) (the "License"). +Licensed under the [Apache License, Version 2.0](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/LICENSE) (the +"License"). -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS +IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language +governing permissions and limitations under the License. -See [NOTICES](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/NOTICES.md) for additional copyright and license notifications. -See [CONTRIBUTORS](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/CONTRIBUTORS.md) for a list of external contributors to the -project. +See [NOTICES](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/NOTICES.md) for additional copyright and license +notifications. See [CONTRIBUTORS](https://github.com/Ed-Fi-Exchange-OSS/Meadowlark/blob/main/CONTRIBUTORS.md) for a list of +external contributors to the project. -Meadowlark image: Copyright © Joel Chamberlain, [some rights -reserved](http://creativecommons.org/licenses/by-nc/4.0/). +Meadowlark image: Copyright © Joel Chamberlain, [some rights reserved](http://creativecommons.org/licenses/by-nc/4.0/). [Original source](https://www.inaturalist.org/observations/38032376). diff --git a/docker/mongodb/README.md b/docker/mongodb/README.md index bc5542c0..8c3ed74b 100644 --- a/docker/mongodb/README.md +++ b/docker/mongodb/README.md @@ -1,6 +1,6 @@ # Meadowlark MongoDB Container -> **Warning** +> [!WARNING] > This image is not hardened for production use. Use at your own risk. This images was created to support loading [Ed-Fi diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index d1bb7c87..71475995 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -2,18 +2,21 @@ ## Services and Network Topology -As of release 0.3.0, "Meadowlark" provides a Fastify-based API that implements (imperfectly) an Ed-Fi API compliant with Data -Standard 3.3, and implements an OAuth2 API supporting the client-credentials flow. These services are backed by MongoDB for -primary data storage. The Ed-Fi API also includes OpenSearch and ElasticSearch to handle `GET` all items and `GET` by -querystring filters. - -The MongoDB configuration needs to have a replica set to support atomic transactions. Technically the replica set can contain -only a single node, though we recommend at least three nodes. OpenSearch clustering has not been tested, and the development -team does not yet know how clustering works with this tool. - -Ideally, all three services would be in the same network segment and able to communicate through unencrypted channels. Indeed, -the development team has not tested any alternative. However, only the API port should be open to outside traffic, except as -needed for debugging. +As of release 0.3.0, "Meadowlark" provides a Fastify-based API that implements +(imperfectly) an Ed-Fi API compliant with Data Standard 3.3, and implements an +OAuth2 API supporting the client-credentials flow. These services are backed by +MongoDB for primary data storage. The Ed-Fi API also includes OpenSearch and +ElasticSearch to handle `GET` all items and `GET` by querystring filters. + +The MongoDB configuration needs to have a replica set to support atomic +transactions. Technically the replica set can contain only a single node, though +we recommend at least three nodes. OpenSearch clustering has not been tested, +and the development team does not yet know how clustering works with this tool. + +Ideally, all three services would be in the same network segment and able to +communicate through unencrypted channels. Indeed, the development team has not +tested any alternative. However, only the API port should be open to outside +traffic, except as needed for debugging. ┌────────────────────────────────────────────────────────┐ │ │ @@ -34,16 +37,16 @@ needed for debugging. ## Environment Variables -Meadowlark has a wealth of configuration options for fine tuning a deployment. At this time, they are all set via environment -variable, though many of them have sensible default values. The variables are grouped below by category to help the page -remain readable and to aid in searching out specific areas for configuration. +Meadowlark has a wealth of configuration options for fine tuning a deployment. +At this time, they are all set via environment variable, though many of them +have sensible default values. The variables are grouped below by category to +help the page remain readable and to aid in searching out specific areas for +configuration. Required configuration values are in bold. ### Database Connectivity -:exclamation: Note: PostgreSQL as an alternative to MongoDB is not supported in release 0.3.0. - | Name | Default | Explanation | | -------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------- | | MEADOWLARK_DATABASE_NAME | meadowlark | For both MongoDB and PostgreSQL | @@ -51,13 +54,13 @@ Required configuration values are in bold. | MONGODB_MAX_NUMBER_OF_RETRIES | 1 | Meadowlark can be configured to retry on MongoDB failure, for example when two concurrent transactions attempt to access the same resource. By default, Meadowlark will retry once. This can be changed with the MONGODB_MAX_NUMBER_OF_RETRIES environment variable. | | MONGO_WRITE_CONCERN | majority | See [MongoDB: Write Concern](https://www.mongodb.com/docs/v4.0/reference/write-concern/) | | MONGO_READ_CONCERN | majority | See [MongoDB: Read Concern](https://www.mongodb.com/docs/v4.0/reference/read-concern/) | -| **OPENSEARCH_ENDPOINT** | (none) | Only required when the OpenSearch listener is configured for Fastify. Example: "http://localhost:9200" | +| **OPENSEARCH_ENDPOINT** | (none) | Only required when the OpenSearch listener is configured. Example: "http://localhost:9200" | | OPENSEARCH_USERNAME | x | Username for connecting to OpenSearch | | OPENSEARCH_PASSWORD | y | Password for connecting to OpenSearch | | OPENSEARCH_REQUEST_TIMEOUT | 30000 | In milliseconds | -| **ELASTICSEARCH_ENDPOINT** | (none) | Only required when the ElasticSearch listener is configured for Fastify. | +| **ELASTICSEARCH_ENDPOINT** | (none) | Only required when the ElasticSearch listener is configured. | | ELASTICSEARCH_REQUEST_TIMEOUT | 30000 | In milliseconds | -| POSTGRES_HOST | localhost | Server/host name for PostgreSQL | +| **POSTGRES_HOST** | localhost | Server/host name for PostgreSQL | | POSTGRES_PORT | 5432 | Port number for PostgreSQL | | POSTGRES_USER | (none) | Username for accessing PostgreSQL | | POSTGRES_PASSWORD | (none) | Password for accessing PostgreSQL | @@ -73,8 +76,8 @@ Required configuration values are in bold. To create a new key: -* Try running the application and accessing endpoint `/{stage}/createSigningKey` -* Or run `openssl rand -base64 256` from a bash terminal shell. +* Run `npm run createKey` +* or `openssl rand -base64 256` from a bash terminal shell. ### Fastify API @@ -89,8 +92,8 @@ To create a new key: | LISTENER1_PLUGIN | (none) | "@edfi/meadowlark-opensearch-backend" or "@edfi/meadowlark-elasticsearch-backend"; if not set, `GET` queries will fail | | LISTENER2_PLUGIN | (none) | No options at this time | | QUERY_HANDLER_PLUGIN | (none) | "@edfi/meadowlark-opensearch-backend" or "@edfi/meadowlark-elasticsearch-backend"; if not set, `GET` queries will fail | -| DOCUMENT_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | Future alternative: "@edfi/meadowlark-postgresql-back | -| AUTHORIZATION_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | No alternative at this time. | +| DOCUMENT_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | "@edfi/meadowlark-mongodb-backend" or "@edfi/meadowlark-postgresql-backend | +| AUTHORIZATION_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | "@edfi/meadowlark-mongodb-backend" or "@edfi/meadowlark-postgresql-backend | | **OAUTH_SERVER_ENDPOINT_FOR_OWN_TOKEN_REQUEST** | (none) | Ex: http://localhost:3000/local/oauth/token | | **OAUTH_SERVER_ENDPOINT_FOR_TOKEN_VERIFICATION** | (none) | Ex: http://localhost:3000/local/oauth/verify | | **OWN_OAUTH_CLIENT_ID_FOR_CLIENT_AUTH** | (none) | Client ID with role "verify-only" | diff --git a/docs/DOCKER-LOCAL-DEV.md b/docs/DOCKER-LOCAL-DEV.md index c2e8956c..52055bfe 100644 --- a/docs/DOCKER-LOCAL-DEV.md +++ b/docs/DOCKER-LOCAL-DEV.md @@ -3,9 +3,10 @@ _Also see [DOCKER](DOCKER.md) for more general information about using Docker for Meadowlark in development, testing, and production_. -:exclamation: These solutions should only be used on localhost with proper -firewalls around external network access to the workstation. Not appropriate for -production use. +> [!WARNING] +> These solutions should only be used on localhost with proper +> firewalls around external network access to the workstation. Not appropriate +> for production use. These compose files require [Docker Compose v2](https://github.com/docker/compose) (which comes with Docker Desktop for @@ -96,4 +97,7 @@ and builds it there. There is a convenience script for PowerShell called up the local environment correctly. This is particularly useful for setting up a clean environment between performance test runs. -> **Note**: you'll need a .env file for this. +> [!NOTE] +> This requires a local _meadowlark-api_ image built, +> this can be done with `npm run docker:build`. +> You'll also need a .env file for this. diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 25c71eb8..d1fa8900 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -26,6 +26,9 @@ developer does not need to do anything at runtime to make this work, other than follow the one-time "Global Docker Configuration" steps in [DOCKER-LOCAL-DEV](DOCKER-LOCAL-DEV.md). +To run the End-to-End tests, follow the steps defined in the [e2e +README](../Meadowlark-js/tests/e2e/readme.md). + ## Dockerfile for Meadowlark API In the above scenarios, the Meadowlark API is running in Node.js directly in the @@ -60,9 +63,10 @@ The Dockerfile copies the TypeScript code into a container and runs the TypeScript build in that context. It does not run eslint or any tests - so before building, be sure that all test are passing at the command line. -:exclamation: You will need to rebuild the docker image with `docker:build` any -time the source code changes. Be sure to stop and re-start any running -Meadowlark API containers in order to see the changes. +> [!WARNING] +> You will need to rebuild the docker image with `docker:build` any +> time the source code changes. Be sure to stop and re-start any running +> Meadowlark API containers in order to see the changes. ### Testing the Meadowlark Image with Docker Compose @@ -75,11 +79,9 @@ A better approach is to startup a new set of containers in a single network together, completely independent from the "Local Dev" containers. For that, you can use the local [docker-compose.yml](../Meadowlark-js/docker-compose.yml) file. This single file stitches together the Meadowlark API, OpenSearch, and -MongoDB. Note that this does not support PostgreSQL at this time, because -PostgreSQL is temporarily de-prioritized. The file also contains commented out -code for running Debezium and Kafka; the code was left commented because the -data were not flowing, and configuring the correct data flow is not important -for the current release milestone. +MongoDB. The file also contains commented out code for running PostgreSQL, +Debezium and Kafka, to use this, follow the instructions defined in the readme +for each backend. Again the main package.json has a few useful shortcut commands: @@ -94,9 +96,12 @@ Again the main package.json has a few useful shortcut commands: ### One Time Initialization -💡 Windows users: you need to run this from Window Subsystem for Linux. Make sure that you run a Bash prompt from WSL, not -the one provided by Git. You also need to have Node.js 18 installed _inside WSL_. The development team likes using -[NVM](https://github.com/nvm-sh/nvm) to manage this installation of Node 18. +> [!TIP] +> Windows users: you need to run this from Window Subsystem for Linux. +> Make sure that you run a Bash prompt from WSL, not the one provided by Git. +> You also need to have Node.js 18 installed _inside WSL_. The development team +> likes using [NVM](https://github.com/nvm-sh/nvm) to manage this installation +> of Node 18. From a Bash prompt, run init-docker-compose.sh. @@ -123,4 +128,5 @@ Based on the default configuration: | OpenSearch Dashboard | http://localhost:5602 | | MongoDB | mongodb://mongo:${MONGODB_PASS}@mongo1:27027,mongo2:27028,mongo3:27029/?replicaSet=rs0 | -You can get the MongoDB password from the generated `Meadowlark-js/.env-docker` file. +You can get the MongoDB password from the generated `Meadowlark-js/.env-docker` +file. diff --git a/docs/LOCALHOST.md b/docs/LOCALHOST.md index 5412ac56..ddd2ba8d 100644 --- a/docs/LOCALHOST.md +++ b/docs/LOCALHOST.md @@ -7,12 +7,12 @@ Instructions for running a local "developer" environment on localhost: 3. Review the [Docker Guidance for Meadowlark](./DOCKER.md) to startup relevant backend services. 4. The Meadowlark runtime currently requires running either PostgreSQL or - MongoDB as the primary datastore, and OpenSearch as a secondary storage for - high-performance queries. Before running the Meadowlark code, startup local - instances of the data stores that you wish to use. The repository comes with - Docker compose files for easily starting up all three. Either run - `eng/Run-DevContainers.ps1` in PowerShell to start all three data stores at - the same time (using default configuration), or see the individual + MongoDB as the primary datastore, and OpenSearch or ElasticSearch as a + secondary storage for high-performance queries. Before running the Meadowlark + code, startup local instances of the data stores that you wish to use. The + repository comes with Docker compose files for easily starting up all three. + Either run `eng/Run-DevContainers.ps1` in PowerShell to start all three data + stores at the same time (using default configuration), or see the individual directories if you wish to customize or to run `docker compose` directly in the directory containing the compose file: * [MongoDB](../Meadowlark-js/backends/meadowlark-mongodb-backend/docker) @@ -34,18 +34,19 @@ Instructions for running a local "developer" environment on localhost: ## Using Kafka Alternatively, you can use Kafka and Kafka-connect to listen to MongoDB changes -and write them to OpenSearch (PostgreSQL and ElasticSearch will be added in the -future). To do so, run the +and write them to OpenSearch (PostgreSQL and ElasticSearch are not supported at +the moment). To do so, run the [Kafka](../Meadowlark-js/backends/meadowlark-kafka-stream/docker) setup and set the `LISTENER1_PLUGIN` as an empty variable in the .env file. ## Clearing Out Local Databases Sometimes it is useful to reset your local environment to a fresh state, with no -records. It is important to do this in all running backend data stores: MongoDB, -PostgreSQL, and OpenSearch. One mechanism is to stop the Docker containers and -then delete the volumes they were using, then restart Docker. If you do not want -to delete the volumes, then you can manually delete records. Examples: +records. It is important to do this in all running backend data stores: MongoDB +or PostgreSQL, and OpenSearch or ElasticSearch. One mechanism is to stop the +Docker containers and then delete the volumes they were using, then restart +Docker. If you do not want to delete the volumes, then you can manually delete +records. Examples: ### OpenSearch and ElasticSearch (Same commands work for both) @@ -84,63 +85,3 @@ delete from meadowlark.references; delete from meadowlark.aliases; delete from meadowlark.authorization; ``` - -## Running E2E Tests - -### Setup - -1. Create a .env file based on .env.example in this folder. -2. Verify that Docker is running - -There are multiple ways of running e2e tests, depending on your scenario: - -### Run or Debug local changes - -To run the tests against a running local environment with -*@edfi/meadowlark-fastify*, run `npm run test:e2e:jest:local`. This will read -the .env file inside the services/meadowlark-fastify folder instead of the e2e -environment. - -This allows to do changes to the running service and see the changes -immediately, as well as debugging the local Meadowlark code from VSCode by -setting breakpoints and running with the included jest VSCode debugger. - -> **Warning** -> -> * This requires the current environment to be already running. -> * No data should be added after the run, since all tests should clean the -> resources created after execution. When adding new functionality, verify -> that created resources are removed after executing the test. - -### Building local image - -If you're adding new features to Meadowlark, and want to test the changes in an -isolated environment, run `npm run test:e2e:build`. This will build the Docker -image and run the tests against that image, cleaning the environment afterwards. - -### Using published image - -If you want to run the tests against and existing Docker image, set the variable -`API_IMAGE_NAME` in the .env file. Run the tests with `npm run -test:e2e:jest:tc`. This will pull the image and run the tests in an isolated -environment, cleaning the environment afterwards. - -### Developer Mode - -If you're adding new e2e tests or want to test in an environment without -clearing the test images set the variable `DEVELOPER_MODE` to true in the .env -file. Run `test:e2e:dev:setup`, this will generate docker containers with the --test prefix that are used for testing. Run the tests with `npm run -test:e2e:jest`. Notice that you *could* use `npm run test:e2e:build` but the -generated image will be ignored because the tests are already running. - -> **Note** -> -> * This will use a locally built image or the image provided in the -> `API_IMAGE_NAME`. -> * This will generate an `ADMIN_KEY` and `ADMIN_SECRET` that can only be -> generated once, therefore, after the first run, this must be added to the -> .env variables. -> -> To exit this mode, run `test:e2e:dev:exit`. This will stop and delete the -> docker containers generated. diff --git a/docs/README.md b/docs/README.md index 5d9a0041..cd2889d8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -38,8 +38,6 @@ See [Running on Localhost](LOCALHOST.md). * Start Visual Studio Code and open the `meadowlark-js` directory. * Install all extension recommendations. * Open a terminal in the meadowlark-js directory and run `npm install`. - * Note that `npm install` will add a git hook to run linters before pushing - with git. ## Test a local deploy diff --git a/docs/design/README.md b/docs/design/README.md index 576c69b7..56e15fd4 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -7,3 +7,4 @@ * [Open Telemetry](open-telemetry/) * [Reference Validation Correctness](reference-validation-correctness/) * [Resource Caching for Performance Improvements](caching/) +* [Rate Limiting for OAuth](rate-limiting-for-oauth/) diff --git a/docs/design/caching/README.md b/docs/design/caching/README.md index f0dc12b7..fe28de2e 100644 --- a/docs/design/caching/README.md +++ b/docs/design/caching/README.md @@ -188,7 +188,7 @@ sequenceDiagram B-->>A: response ``` -> **Note** +> [!NOTE] > In subsequent sequence diagrams, the `FrontEndFacade` will be > omitted. diff --git a/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md b/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md index 6db9f2fc..15a1f856 100644 --- a/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md +++ b/docs/design/performance-benchmarking/WRITE-PERFORMANCE.md @@ -12,7 +12,8 @@ consistency of resource availability and limitations. ## Bulk Loading -> **Note** To run the bulk loading tests it's important to start with a clean +> [!NOTE] +> To run the bulk loading tests it's important to start with a clean > database. To load the data, there are functions to load the GrandBend and PartialGrandBend @@ -30,13 +31,6 @@ The script receives two parameters: This script will enter the data into Meadowlark and will print the execution time. -> **Warning** The LoadGrandBend script is returning errors related to incorrect casing on properties that -> begin with acronyms ([METAED-1360](https://tracker.ed-fi.org/browse/METAED-1360)). - -List of issues: - -- [RND-583](https://tracker.ed-fi.org/browse/RND-583) - ## Performance Testing Suite There are two performance test types that paths that cover the write @@ -44,7 +38,7 @@ performance, Pipeclean Tests and Volume Tests. ### Setup -> **Warning** +> [!WARNING] > _For now_: The tests will be executed with > [Suite3-Performance-Testing](https://github.com/Ed-Fi-Exchange-OSS/Suite-3-Performance-Testing) > on branch [meadowlark-updates](https://github.com/Ed-Fi-Exchange-OSS/Suite-3-Performance-Testing/tree/meadowlark-updates) @@ -59,15 +53,3 @@ performance, Pipeclean Tests and Volume Tests. and secret. Set the values required for Meadowlark. - Run `poetry run python edfi_performance_test -t "VALUE"` where value can be "pipeclean" or "volume". [More details](https://github.com/Ed-Fi-Exchange-OSS/Suite-3-Performance-Testing/tree/main/src/edfi-performance-test) - -> **Warning** Currently there are a list of problems to run the performance tests, -> therefore the process will not finish and you have to manually terminate the execution. - -List of issues: - -- [RND-583](https://tracker.ed-fi.org/browse/RND-583) -- [RND-584](https://tracker.ed-fi.org/browse/RND-584) -- [RND-585](https://tracker.ed-fi.org/browse/RND-585) -- [PERF-298](https://tracker.ed-fi.org/browse/PERF-298) - -After fixing this errors, the tests should be executed as part of [RND-580](https://tracker.ed-fi.org/browse/RND-580) diff --git a/docs/performance-testing/README.md b/docs/performance-testing/README.md index 3ee5a9a0..f73ad0f2 100644 --- a/docs/performance-testing/README.md +++ b/docs/performance-testing/README.md @@ -1,6 +1,37 @@ # Ed-Fi Meadowlark Performance Testing Results +* [RND-538: Load balancing API scale out](load-balancing-API-scale-out.md). +Summary: There are no performance improvements when using a load balancer, we + suspect this is caused by a bottleneck in MongoDB. + * [RND-380: MongoDB Connection Pooling](mongo-connection-pooling.md). Summary: - when using the clustered mode, no discernible benefit to tuning MongoDB +When using the clustered mode, no discernible benefit to tuning MongoDB connection pooling while loading the "partial Grand Bend" dataset. Also includes comparison with ODS/API v5.3-patch4. + +* [RND-538: MongoDB performance testing](mongo-performance-testing.md). Summary: +There are performance improvements when removing the write locks and the read +performance improve when removing indices. + +* [RND-647: PostgreSQL performance testing](postgres-performance-testing.md). +Summary: Removing indices and increasing the connection pool can improve the +performance. + +* [RND-643: PostgreSQL Serializable Snapshot Isolation (SSI) performance +testing](postgres-SSI-performance-testing.md). Summary: For SSI to be effective, +retry logic must be implemented, otherwise many transactions would fail without +altering any records. + +* [RND-644 - Investigate alternatives to MongoDB backend +read-for-write-locking](RND-644.md). Summary: There are no significant + differences on changing the current approach. + +* [RND-658 - Comparison of PostgreSQL and MongoDB performance](RND-658.md). + Summary: See document for detailed results + +* [RND-668 - Compare ODS/API 5.3 bulk load with outcomes of + RND-658](RND-668.md). Summary: See document for detailed results + +* [RND-604: Separate kafka connect containers to separate sources from + sink](two-functionalities-of-kafka-connect-separated.md). Summary: There are + no significant differences when separating the source from the sink. diff --git a/docs/performance-testing/RND-644.md b/docs/performance-testing/RND-644.md index 17c878a8..0812b23b 100644 --- a/docs/performance-testing/RND-644.md +++ b/docs/performance-testing/RND-644.md @@ -1,5 +1,9 @@ # RND-644 - Investigate alternatives to MongoDB backend read-for-write-locking +## Goal + +Measure if there are significant differences in changing the approach of emulating the write lock for MongoDB. + ## Virtual Machine setup It's essentially a **t2.2xlarge** on AWS: @@ -43,3 +47,7 @@ Ran the tests/profiling/AutocannonSchools.ts script for 160 seconds, with 1 conn |------------------|----------|------------|-------------|-------------|----------------| | 1 Connection | 10477 | 0 | 14.536 | 65.336 | 22591.816 | | 25 Connections | 895.6 | 0 | 2319.514 | 5.414 | 1888.624 | + +## Conclusion + +Based on the results, there are no significant differences on changing the current approach. diff --git a/docs/performance-testing/load-balancing-API-scale-out.md b/docs/performance-testing/load-balancing-API-scale-out.md index 025d2364..bf709815 100644 --- a/docs/performance-testing/load-balancing-API-scale-out.md +++ b/docs/performance-testing/load-balancing-API-scale-out.md @@ -2,8 +2,8 @@ ## Goal -Evaluate impact on application performance when scaling it out. -We expect to get a better performance with the load balancer than without it. +Evaluate impact on application performance when scaling it out. We expect to get +a better performance with the load balancer than without it. ## Methodology @@ -29,10 +29,10 @@ We expect to get a better performance with the load balancer than without it. ## Environment -The bulk load client runs on the host machine. It has 16 GB of RAM, -Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 2.59 GHz processor, 6 cores and -12 logical processors, using WSL2. Docker has been configured to use 8GB of RAM -and 10 cores. +The bulk load client runs on the host machine. It has 16 GB of RAM, Intel(R) +Core(TM) i7-9750H CPU @ 2.60GHz 2.59 GHz processor, 6 cores and 12 logical +processors, using WSL2. Docker has been configured to use 8GB of RAM and 10 +cores. Baseline `.env` configuration file: @@ -90,10 +90,11 @@ LOG_FILE_LOCATION=c:/temp/ ## Further analysis -Given that we did not see any improvement with the nginx load balancer, we decided to investigate -a little further, using this tool called [cadvisor](https://github.com/google/cadvisor) and -[mongodb compass](https://www.mongodb.com/products/compass). -We suspect that mongodb is causing the bottle neck. +Given that we did not see any improvement with the nginx load balancer, we +decided to investigate a little further, using this tool called +[cadvisor](https://github.com/google/cadvisor) and [mongodb +compass](https://www.mongodb.com/products/compass). We suspect that mongodb is +causing the bottle neck. | With load balancing and 4 Fastify threads | | | -------------------------------------------- | ------------ | @@ -123,6 +124,6 @@ We suspect that mongodb is causing the bottle neck. | mongodb compass | [Screenshot](./load-balancing-API-scale-out-screenshots/WithoutLB_4FastifyThreads/mongo1_compas.jpg) | | Overall | [Screenshot](./load-balancing-API-scale-out-screenshots/WithoutLB_4FastifyThreads/Overall.jpg) | -Given the results we got with the tools indicated above, everything indicates that -MongoDB is the bottle neck we have, and the reason why we are not getting a +Given the results we got with the tools indicated above, everything indicates +that MongoDB is the bottle neck we have, and the reason why we are not getting a better performance with the load balancer. diff --git a/docs/performance-testing/mongo-connection-pooling.md b/docs/performance-testing/mongo-connection-pooling.md index 05443a5c..a3548ca8 100644 --- a/docs/performance-testing/mongo-connection-pooling.md +++ b/docs/performance-testing/mongo-connection-pooling.md @@ -131,7 +131,8 @@ the performance is discernibly worse with only one thread, whether using one or scenario does clearly yield an improved experience, reducing the average time to complete the test by roughly 69%. -> **Note** Five executions of each test appears to be useful, but where timings +> [!NOTE] +> Five executions of each test appears to be useful, but where timings > are very close to one another, the number of data points is insufficient for > giving a useful statistical significance. diff --git a/docs/performance-testing/mongo-performance-testing.md b/docs/performance-testing/mongo-performance-testing.md index b59586a6..13bc66fa 100644 --- a/docs/performance-testing/mongo-performance-testing.md +++ b/docs/performance-testing/mongo-performance-testing.md @@ -2,14 +2,15 @@ ## Goal -50% of Meadowlark's time is spent in the MongoDB driver code as of today, so now -is a good time to start examining MongoDB performance. We have assumptions on how -our indexes should be making our queries perform well, but we should test that. +Analyze the performance of MongoDB, since 50% of Meadowlark's time is spent in +the MongoDB driver code as of today. Measure if indexes improve the performance +of Meadowlark ## Method 1 -As a first step we are execute the bulk load tool and the Autocannon tool, once the write lock is removed: -That is removing every reference to Db.writeLockReferencedDocuments(). +As a first step we are execute the bulk load tool and the Autocannon tool, once +the write lock is removed: That is removing every reference to +Db.writeLockReferencedDocuments(). 1. Start mongodb and open search using mongo containers. @@ -29,7 +30,8 @@ That is removing every reference to Db.writeLockReferencedDocuments(). 6. Repeat the measurement process. 7. Repeat the process, but this time use the autocannon tool to measure. - 1. To do this, follow the steps in ./../../Meadowlark-js/tests/profiling/readme.md + 1. To do this, follow the steps in + ./../../Meadowlark-js/tests/profiling/readme.md ### Method 1, AVG Results with the bulk load tool @@ -43,19 +45,23 @@ The results can be found here: ./mongo-performance-testing/ ### Method 1, Conclusion -The performance improves a lot when we remove that feature from mongo +The performance improves significantly when we remove that feature from mongo ## Method 2 -As a second step, we are going to run performance tests (that is bulk load and Autocannon) -when the code that creates the indexes as been removed. That is on Db.ts, on the mongo backend. +As a second step, we are going to run performance tests (that is bulk load and +Autocannon) when the code that creates the indexes as been removed. That is on +Db.ts, on the mongo backend. -As a first step we want to know the performance impact of the creation of the indexes, that is when we insert the data. -We are going to do this with the bulk load tool. +As a first step we want to know the performance impact of the creation of the +indexes, that is when we insert the data. We are going to do this with the bulk +load tool. -Based on that, we can then run tests for the reads, with and without indexes. For this step we need to use Autocannon to, -first create a number of schools, and then randomly execute a bunch of reads on those schools. -For this step, I changed the AutocannonSchools.ts script to execute 100000 Gets on those schools previously created. +Based on that, we can then run tests for the reads, with and without indexes. +For this step we need to use Autocannon to, first create a number of schools, +and then randomly execute a bunch of reads on those schools. For this step, the +AutocannonSchools.ts script was updated to execute 100000 Gets on those schools +previously created. 1. Start mongodb and open search using mongo containers. @@ -75,15 +81,16 @@ For this step, I changed the AutocannonSchools.ts script to execute 100000 Gets 6. Repeat the measurement process. 7. Repeat the process, but this time use the autocannon tool to measure. - 1. To do this, follow the steps in ./../../Meadowlark-js/tests/profiling/readme.md + 1. To do this, follow the steps in + [profiling](../../Meadowlark-js/tests/profiling/readme.md) 2. Measurement is done on inserts and reads. ### Method 2, AVG Results with the bulk load tool With the code as it is the average time result is: 04:19:798 -When I comment out the index creation (one line at a time) the time is 4:06. Meaning that, for each index creation -that I comment out, the performance improves about 5% +Commenting out the index creation (one line at a time) the time is 4:06. Meaning +that, for each index creation commented, the performance improves about 5% ### Method 2, AVG Results with the Autocannon tool @@ -91,93 +98,77 @@ This is avg time after 5 times executed. With the code as it is the average time result is: 01:50:336 -When I comment out the index creation for documentUuid: 1:44:711 +Commenting out the index creation for documentUuid: 1:44:711 -When I comment out the index creation for outboundRefs: 1:43:527 +Commenting out the index creation for outboundRefs: 1:43:527 -When I comment out the index creation for aliasMeadowlarkIds: 1:46:556 +Commenting out the index creation for aliasMeadowlarkIds: 1:46:556 ### Method 2, Conclusion -The performance improves about 5% when I comment out each index creation, individually. But for the reads, the performance -doesn't improve. The reason could be that we don't have enough data in the mongo database. +The performance improves about 5% when I comment out each index creation, +individually. But for the reads, the performance doesn't improve. The reason +could be that we don't have enough data in the mongo database. ### Method 3, general review of the memory usage, cpu, connections, etc -The bulk load client runs on the host machine. It has 16 GB of RAM, -Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 2.59 GHz processor, 6 cores and -12 logical processors, using WSL2. Docker has been configured to use 8GB of RAM -and 10 cores. +The bulk load client runs on the host machine. It has 16 GB of RAM, Intel(R) +Core(TM) i7-9750H CPU @ 2.60GHz 2.59 GHz processor, 6 cores and 12 logical +processors, using WSL2. Docker has been configured to use 8GB of RAM and 10 +cores. -Ran bulk load scripts with 8Gb assigned to docker and the time it takes to execute the scripts don't change significantly. +Ran bulk load scripts with 8Gb assigned to docker and the time it takes to +execute the scripts don't change significantly. -While the script is running: - In terms of memory, all 3 instances of mongodb use similar amount of memory. - Mongo1, uses a lot more cpu: about 170%. This is because it is receiving all the write operations. - Maximum number of connections was 101. - Maximum amount of memory used by mongo 1 was about 350Mb. - Maximum number of operations per second, about 1700 +While the script is running: In terms of memory, all 3 instances of mongodb use + similar amount of memory. Mongo1, uses a lot more cpu: about 170%. This is + because it is receiving all the write operations. Maximum number of + connections was 101. Maximum amount of memory used by mongo 1 was about + 350Mb. Maximum number of operations per second, about 1700 -Meadowlark: - Maximum memory usage for the each instance 650Mb. Average is about 450Mb - Maximum CPU percentage is 140% while average is about 40%. +Meadowlark: Maximum memory usage for the each instance 650Mb. Average is about + 450Mb Maximum CPU percentage is 140% while average is about 40%. ### Method 4, general review with Mongodb Compass stats while changing the number of connections For these tests I used Autocannon -1. 5000 requests and 1 connection: -AVG time: 2:44:849 -AVG number of operations per second: 187 -AVG number of connections: 27 +1. 5000 requests and 1 connection: AVG time: 2:44:849 AVG number of operations +per second: 187 AVG number of connections: 27 -2. 5000 requests and 10 connections: -AVG time: 1:30:880 -AVG number of operations per second: 1393 -AVG number of connections: 51 +2. 5000 requests and 10 connections: AVG time: 1:30:880 AVG number of operations +per second: 1393 AVG number of connections: 51 -3. 5000 requests and 25 connections: -AVG time: 3:17:107 -AVG number of operations per second: 1705 -AVG number of connections: 91 +3. 5000 requests and 25 connections: AVG time: 3:17:107 AVG number of operations +per second: 1705 AVG number of connections: 91 -4. 5000 requests and 50 connections: -AVG time: 4:36:431 -AVG number of operations per second: 1857 -AVG number of connections: 172 +4. 5000 requests and 50 connections: AVG time: 4:36:431 AVG number of operations +per second: 1857 AVG number of connections: 172 ### Method 5, Reviewing Connection pooling -I initially used the bulk load tool, with different connection pooling configurations. +I initially used the bulk load tool, with different connection pooling +configurations. -1. maxPoolSize: 100 and minPoolSize: 10: -AVG time: 3:42:418 -AVG number of operations per second: 1330 -AVG number of connections: 82 +1. maxPoolSize: 100 and minPoolSize: 10: AVG time: 3:42:418 AVG number of +operations per second: 1330 AVG number of connections: 82 -2. maxPoolSize: 200 and minPoolSize: 10: -AVG time: 4:09:067 -AVG number of operations per second: 1248 -AVG number of connections: 82 +2. maxPoolSize: 200 and minPoolSize: 10: AVG time: 4:09:067 AVG number of +operations per second: 1248 AVG number of connections: 82 -3. maxPoolSize: 50 and minPoolSize: 10: -AVG time: 3:32:659 -AVG number of operations per second: 1543 -AVG number of connections: 85 +3. maxPoolSize: 50 and minPoolSize: 10: AVG time: 3:32:659 AVG number of +operations per second: 1543 AVG number of connections: 85 Then I tried with Autocannon tool, to make changes on the number of connections -1. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 requests and 100 connections: -AVG time: 4:54:574 -AVG number of operations per second: 2738 -AVG number of connections: 289 +1. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 +requests and 100 connections: AVG time: 4:54:574 AVG number of operations per +second: 2738 AVG number of connections: 289 -2. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 requests and 50 connections: -AVG time: 4:37:150 -AVG number of operations per second: 1936 -AVG number of connections: 148 +2. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 +requests and 50 connections: AVG time: 4:37:150 AVG number of operations per +second: 1936 AVG number of connections: 148 -3. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 requests and 25 connections: -AVG time: 3:06:876 -AVG number of operations per second: 1763 -AVG number of connections: 95 +3. maxPoolSize: 50 and on mongo client minPoolSize: 10. On Autocannon: 5000 +requests and 25 connections: AVG time: 3:06:876 AVG number of operations per +second: 1763 AVG number of connections: 95 diff --git a/docs/performance-testing/postgres-SSI-performance-testing.md b/docs/performance-testing/postgres-SSI-performance-testing.md index 46617111..9a7f6085 100644 --- a/docs/performance-testing/postgres-SSI-performance-testing.md +++ b/docs/performance-testing/postgres-SSI-performance-testing.md @@ -57,8 +57,9 @@ These results are consistent with the Autocannon results: - Also, SSI version returned more tuples but affected tuples are less compared to previous version. That behavior is consistent with a retry mechanism, because when we need to retry, we need to restart the process. ## Conclusion + - The version with SSI reduces latency but also generates a large amount of rollback, so the number of altered records is smaller. - Additionally, in cases like locking\delete.test.ts, removing the SELECT...FOR UPDATE/SHARE can cause errors and deadlocks. -- For SSI to be effective, retry logic must be implemented, otherwise many transactions would fail without altering any records. \ No newline at end of file +- For SSI to be effective, retry logic must be implemented, otherwise many transactions would fail without altering any records. diff --git a/docs/performance-testing/postgres-performance-testing.md b/docs/performance-testing/postgres-performance-testing.md index 74a624e4..f8771a6f 100644 --- a/docs/performance-testing/postgres-performance-testing.md +++ b/docs/performance-testing/postgres-performance-testing.md @@ -1,12 +1,13 @@ -# RND-647: Postgresql performance testing +# RND-647: PostgreSQL performance testing ## Goal -Run Postgresql performance testing to see the time required to execute Meadowlark on Postgresql. +Run PostgreSQL performance testing to see the time required to execute Meadowlark on PostgreSQL. ## Description To test performance we have two Methods: + - Autocannon tool - Bulk Load data diff --git a/eng/deploy/azure/README.md b/eng/deploy/azure/README.md index 4c38c973..ea4a5f95 100644 --- a/eng/deploy/azure/README.md +++ b/eng/deploy/azure/README.md @@ -38,7 +38,8 @@ az container create --resource-group $resourceGroup -n ml-opensearch ` ``` -> **Note** See [Enable Logging](#enable-logging) before setting up meadowlark container if you want to get log information. +> > [!NOTE] +> See [Enable Logging](#enable-logging) before setting up meadowlark container if you want to get log information. ```pwsh @@ -92,7 +93,8 @@ Copy the `customerId` from the result. Copy the `primarySharedKey` from the result. -> **Note** This information can be retrieved from the Portal, in Log Analytics Workspaces -> Settings -> Agents. [More +> [!NOTE] +> This information can be retrieved from the Portal, in Log Analytics Workspaces -> Settings -> Agents. [More > information](https://learn.microsoft.com/en-us/azure/container-instances/container-instances-log-analytics#get-log-analytics-credentials) When creating the **meadowlark container**, include the customerId (workspace_id) and the primarySharedKey (workspace_key) as @@ -107,45 +109,6 @@ az container create ... ` To view the log information, follow [these steps](https://learn.microsoft.com/en-us/azure/container-instances/container-instances-log-analytics#view-logs) -## Deploy with Docker Azure Integration - -> **Warning** The Docker Azure Integration will be retired in November 2023. - -- [Log into Azure from Docker](https://docs.docker.com/cloud/aci-integration/#log-into-azure). - -- [Create an ACI Docker context](https://docs.docker.com/cloud/aci-integration/#create-an-aci-context) - -- Browse to `../eng/deploy/azure` - -- Create a .env file. Set the OAUTH_SIGNING_KEY, AZURE_REGION and ED_FI_DOMAIN_NAME. The combination of domain name an azure - region must be globally unique. - -- Execute the following script: - -```pwsh - -# Switch to the ACI Context -docker context use myacicontext - -docker compose -p meadowlark --file ./azure-docker-compose.yml up -d - -# Initialize replica set -az container exec --resource-group {resource group name} -n meadowlark ` - --container-name ml-mongo1 --exec-command 'mongo --eval rs.initiate()' - -``` - -> **Note** Not all functionality available in a Docker Compose file is available when deploying to ACI. To review the -> available features, check [the documentation](https://docs.docker.com/cloud/aci-compose-features/) . - -### Removing the containers - -Given that `docker compose down` is not available. To remove all the containers in the group, execute: - -```pwsh -az container delete --resource-group {resource group name} -n meadowlark -``` - ## Test your deployment To verify your deployment, run: @@ -156,7 +119,8 @@ curl http://$meadowlarkDnsLabel.southcentralus.azurecontainer.io:3000/stg | Conv This will output the summary of the deployment -> **Warning** Not ready for production usage. This example is using a single mongo node with a simulated replica set and +> [!WARNING] +> Not ready for production usage. This example is using a single mongo node with a simulated replica set and > bypassing security with a direct connection, also, it's using the OAUTH hardcoded credentials. The current configuration is > initializing the mongo replica manually, and this is not saved. Therefore, if the container instance is stopped, it's > necessary to reinitialize the replica set. diff --git a/eng/ods-api/README.md b/eng/ods-api/README.md index cb763fd3..87a6dfec 100644 --- a/eng/ods-api/README.md +++ b/eng/ods-api/README.md @@ -4,4 +4,5 @@ This directory supports starting the ODS/API v5.3-patch4 in sandbox mode, with change queries, profiles, and composites disabled. Useful for head-to-head test comparisons with Meadowlark. -> **Warning** do not publish to Docker Hub. +> [!WARNING] +> do not publish to Docker Hub.