From 849e026dcd872dbdd20a55cf2a6dd13f53420735 Mon Sep 17 00:00:00 2001 From: Brad Banister Date: Fri, 19 Jan 2024 13:20:38 -0600 Subject: [PATCH] [RND-701] put descriptor load under authorization (#343) --- .../package.json | 6 +-- .../meadowlark-mongodb-backend/package.json | 8 +-- .../package.json | 6 +-- .../package.json | 8 +-- Meadowlark-js/lerna.json | 2 +- Meadowlark-js/package-lock.json | 50 +++++++++---------- .../meadowlark-authz-server/package.json | 4 +- .../packages/meadowlark-core/package.json | 4 +- .../src/handler/DescriptorLoader.ts | 2 +- .../src/handler/FrontendFacade.ts | 23 +++++++++ .../packages/meadowlark-core/src/index.ts | 3 +- .../meadowlark-utilities/package.json | 2 +- .../services/meadowlark-fastify/package.json | 8 +-- .../src/handler/DescriptorLoader.ts | 6 +-- Meadowlark-js/tests/e2e/package.json | 4 +- .../scenarios/ResourcesCRUDValidation.test.ts | 5 +- 16 files changed, 82 insertions(+), 59 deletions(-) diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json index 7e96de81..981f5639 100644 --- a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-elasticsearch-backend", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark backend plugin for elasticsearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, diff --git a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json index 18f5511b..cee397e8 100644 --- a/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-mongodb-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-mongodb-backend", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark backend plugin for MongoDB", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "async-retry": "^1.3.3", "mongodb": "^5.9.2", "ramda": "0.29.1" diff --git a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json index a8d50836..8d7ad0d8 100644 --- a/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-opensearch-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-opensearch-backend", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark backend plugin for OpenSearch", "license": "Apache-2.0", "publishConfig": { @@ -19,8 +19,8 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json index 1d91c758..32fbf089 100644 --- a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json +++ b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-postgresql-backend", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark backend plugin for PostgreSQL", "license": "Apache-2.0", "publishConfig": { @@ -19,9 +19,9 @@ "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" }, "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" diff --git a/Meadowlark-js/lerna.json b/Meadowlark-js/lerna.json index 36d64068..0c10e6e2 100644 --- a/Meadowlark-js/lerna.json +++ b/Meadowlark-js/lerna.json @@ -1,5 +1,5 @@ { "lerna": "8.0.2", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "npmClient": "npm" } diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index 3df8a613..683c25be 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -54,11 +54,11 @@ }, "backends/meadowlark-elasticsearch-backend": { "name": "@edfi/meadowlark-elasticsearch-backend", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@elastic/elasticsearch": "^8.10.0", "@elastic/transport": "^8.3.4" }, @@ -72,12 +72,12 @@ }, "backends/meadowlark-mongodb-backend": { "name": "@edfi/meadowlark-mongodb-backend", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "async-retry": "^1.3.3", "mongodb": "^5.9.2", "ramda": "0.29.1" @@ -90,11 +90,11 @@ }, "backends/meadowlark-opensearch-backend": { "name": "@edfi/meadowlark-opensearch-backend", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@opensearch-project/opensearch": "^2.4.0" }, "devDependencies": { @@ -107,12 +107,12 @@ }, "backends/meadowlark-postgresql-backend": { "name": "@edfi/meadowlark-postgresql-backend", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "pg": "^8.11.3", "pg-format": "^1.0.4", "ramda": "0.29.1" @@ -17513,11 +17513,11 @@ }, "packages/meadowlark-authz-server": { "name": "@edfi/meadowlark-authz-server", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", @@ -17567,11 +17567,11 @@ }, "packages/meadowlark-core": { "name": "@edfi/meadowlark-core", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", @@ -17648,7 +17648,7 @@ }, "packages/meadowlark-utilities": { "name": "@edfi/meadowlark-utilities", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { "pino": "^8.15.7", @@ -17695,12 +17695,12 @@ }, "services/meadowlark-fastify": { "name": "@edfi/meadowlark-fastify", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" @@ -17713,10 +17713,10 @@ }, "tests/e2e": { "name": "@edfi/meadowlark-e2e-tests", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "license": "Apache-2.0", "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/Meadowlark-js/packages/meadowlark-authz-server/package.json b/Meadowlark-js/packages/meadowlark-authz-server/package.json index 9b223f08..85d02dce 100644 --- a/Meadowlark-js/packages/meadowlark-authz-server/package.json +++ b/Meadowlark-js/packages/meadowlark-authz-server/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-authz-server", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark authorization server", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "ajv": "^8.12.0", "didyoumean2": "^6.0.1", "dotenv": "^16.3.1", diff --git a/Meadowlark-js/packages/meadowlark-core/package.json b/Meadowlark-js/packages/meadowlark-core/package.json index 85aed796..727adc96 100644 --- a/Meadowlark-js/packages/meadowlark-core/package.json +++ b/Meadowlark-js/packages/meadowlark-core/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-core", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark core functionality", "license": "Apache-2.0", "publishConfig": { @@ -14,7 +14,7 @@ ], "dependencies": { "@apideck/better-ajv-errors": "^0.3.6", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@isaacs/ttlcache": "^1.4.1", "ajv": "^8.12.0", "ajv-formats": "^2.1.1", diff --git a/Meadowlark-js/packages/meadowlark-core/src/handler/DescriptorLoader.ts b/Meadowlark-js/packages/meadowlark-core/src/handler/DescriptorLoader.ts index 45a3c4b6..ca049bf9 100644 --- a/Meadowlark-js/packages/meadowlark-core/src/handler/DescriptorLoader.ts +++ b/Meadowlark-js/packages/meadowlark-core/src/handler/DescriptorLoader.ts @@ -169,7 +169,7 @@ async function loadParsedDescriptors(descriptorData: XmlDescriptorData): Promise Logger.info(`${loadCount} descriptors loaded`, '-'); } -export async function loadDescriptors(): Promise { +export async function loadDescriptorsFromFile(): Promise { Logger.info('Loading descriptors', 'n/a'); await loadDocumentStore(); const data: XmlDescriptorData = await readDescriptors(descriptorPath); diff --git a/Meadowlark-js/packages/meadowlark-core/src/handler/FrontendFacade.ts b/Meadowlark-js/packages/meadowlark-core/src/handler/FrontendFacade.ts index 9c417808..a48ca50a 100644 --- a/Meadowlark-js/packages/meadowlark-core/src/handler/FrontendFacade.ts +++ b/Meadowlark-js/packages/meadowlark-core/src/handler/FrontendFacade.ts @@ -26,6 +26,7 @@ import { logTheResponse } from '../middleware/ResponseLoggingMiddleware'; import { equalityConstraintValidation } from '../middleware/ValidateEqualityConstraintMiddleware'; import { timestampRequest } from '../middleware/TimestampRequestMiddleware'; import { loadApiSchema } from '../middleware/ApiSchemaLoadingMiddleware'; +import { loadDescriptorsFromFile } from './DescriptorLoader'; type MiddlewareStack = (model: MiddlewareModel) => Promise; @@ -271,3 +272,25 @@ export async function closeConnection(): Promise { writeErrorToLog(moduleName, '', 'closeConnection', 500, e); } } + +/** + * Entry point for loading descriptors + */ +export async function loadDescriptors(frontendRequest: FrontendRequest): Promise { + try { + await initialize(); + + const authorizeResponse: FrontendResponse | null = (await authorize({ frontendRequest, frontendResponse: null })) + .frontendResponse; + + // if there is a response posted by authorize, there was an auth failure so we are done + if (authorizeResponse != null) return authorizeResponse; + + await loadDescriptorsFromFile(); + + return { statusCode: 202 }; + } catch (e) { + writeErrorToLog(moduleName, frontendRequest.traceId, 'loadDescriptors', 500, e); + return { statusCode: 500 }; + } +} diff --git a/Meadowlark-js/packages/meadowlark-core/src/index.ts b/Meadowlark-js/packages/meadowlark-core/src/index.ts index 109549f6..5c6c3a06 100644 --- a/Meadowlark-js/packages/meadowlark-core/src/index.ts +++ b/Meadowlark-js/packages/meadowlark-core/src/index.ts @@ -50,8 +50,7 @@ export type { DocumentObjectKey } from './model/api-schema/DocumentObjectKey'; export type { SemVer } from './model/api-schema/SemVer'; // Handlers -export { upsert, deleteIt, get, update, closeConnection } from './handler/FrontendFacade'; -export { loadDescriptors } from './handler/DescriptorLoader'; +export { upsert, deleteIt, get, update, loadDescriptors, closeConnection } from './handler/FrontendFacade'; export { apiVersion, openApiUrlList, diff --git a/Meadowlark-js/packages/meadowlark-utilities/package.json b/Meadowlark-js/packages/meadowlark-utilities/package.json index 082ec4bc..af41c4b8 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/package.json +++ b/Meadowlark-js/packages/meadowlark-utilities/package.json @@ -1,7 +1,7 @@ { "name": "@edfi/meadowlark-utilities", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark shared utilities", "license": "Apache-2.0", "publishConfig": { diff --git a/Meadowlark-js/services/meadowlark-fastify/package.json b/Meadowlark-js/services/meadowlark-fastify/package.json index 03f2e08d..2edec3d8 100644 --- a/Meadowlark-js/services/meadowlark-fastify/package.json +++ b/Meadowlark-js/services/meadowlark-fastify/package.json @@ -1,6 +1,6 @@ { "name": "@edfi/meadowlark-fastify", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark service using Fastify", "license": "Apache-2.0", "publishConfig": { @@ -12,9 +12,9 @@ "/package.json" ], "dependencies": { - "@edfi/meadowlark-authz-server": "0.4.1-pre.7", - "@edfi/meadowlark-core": "0.4.1-pre.7", - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-authz-server": "0.4.1-pre.8", + "@edfi/meadowlark-core": "0.4.1-pre.8", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@fastify/rate-limit": "^6.0.1", "dotenv": "^16.3.1", "fastify": "^3.29.5" diff --git a/Meadowlark-js/services/meadowlark-fastify/src/handler/DescriptorLoader.ts b/Meadowlark-js/services/meadowlark-fastify/src/handler/DescriptorLoader.ts index 6c428b6d..aaf9ea0f 100644 --- a/Meadowlark-js/services/meadowlark-fastify/src/handler/DescriptorLoader.ts +++ b/Meadowlark-js/services/meadowlark-fastify/src/handler/DescriptorLoader.ts @@ -6,16 +6,16 @@ /* istanbul ignore file */ import { loadDescriptors as meadowlarkLoadDescriptors } from '@edfi/meadowlark-core'; import { FastifyRequest, FastifyReply } from 'fastify'; +import { respondWith, fromRequest } from './MeadowlarkConverter'; /** * A trigger to call the loadDescriptors function via a Fastify endpoint. * Only available when the stage is explicitly set to "local". */ -export async function loadDescriptors(_request: FastifyRequest, reply: FastifyReply): Promise { +export async function loadDescriptors(request: FastifyRequest, reply: FastifyReply): Promise { if (process.env.MEADOWLARK_STAGE !== 'local') { await reply.code(404).send(''); return; } - await meadowlarkLoadDescriptors(); - await reply.code(202).send(''); + respondWith(await meadowlarkLoadDescriptors(fromRequest(request)), reply); } diff --git a/Meadowlark-js/tests/e2e/package.json b/Meadowlark-js/tests/e2e/package.json index 21a7c85d..f289f8f0 100644 --- a/Meadowlark-js/tests/e2e/package.json +++ b/Meadowlark-js/tests/e2e/package.json @@ -1,13 +1,13 @@ { "name": "@edfi/meadowlark-e2e-tests", "main": "dist/index.js", - "version": "0.4.1-pre.7", + "version": "0.4.1-pre.8", "description": "Meadowlark Ed-Fi API end to end tests", "license": "Apache-2.0", "private": true, "files": [], "devDependencies": { - "@edfi/meadowlark-utilities": "0.4.1-pre.7", + "@edfi/meadowlark-utilities": "0.4.1-pre.8", "@testcontainers/mongodb": "^10.3.1", "@testcontainers/postgresql": "^10.3.1", "@types/chance": "^1.1.6", diff --git a/Meadowlark-js/tests/e2e/scenarios/ResourcesCRUDValidation.test.ts b/Meadowlark-js/tests/e2e/scenarios/ResourcesCRUDValidation.test.ts index b45df5d2..a5f3bfe4 100644 --- a/Meadowlark-js/tests/e2e/scenarios/ResourcesCRUDValidation.test.ts +++ b/Meadowlark-js/tests/e2e/scenarios/ResourcesCRUDValidation.test.ts @@ -73,8 +73,9 @@ describe('when performing crud operations', () => { expect(getResponse.body).toEqual(expect.objectContaining(resourceBody)); }); - it('returns 404 when the resource does not exist', async () => { - const response = await getResourceByLocation(`${createdResourceLocation.slice(0, -1)}F`); + it('returns 404 when getting a different resource', async () => { + const randomUuid = '9fab6588-4082-4f1f-aca2-1ea345075dcf'; + const response = await getResourceByLocation(`${createdResourceLocation.slice(0, -36)}${randomUuid}`); expect(response.statusCode).toEqual(404); });