[RND-651] Ability to loosely allow overposted material for parity

Meadowlark-js/.env.example

@@ -41,6 +41,9 @@ ALLOW_TYPE_COERCION=true
 # true when bulk uploading the ODS/API "populated template".
 ALLOW__EXT_PROPERTY=true
 
+# Allow the API to ignore elements that are not part of the schema are ignored.
+# if false, it returns an error message if the element is not part of the schema.
+ALLOW_OVERPOSTING=true
 
 SAVE_LOG_TO_FILE=false
 # LOG_FILE_LOCATION=c:/temp/

Meadowlark-js/docker-compose.yml

@@ -40,6 +40,7 @@ services:
       END_ALLOWED_SCHOOL_YEAR: ${END_ALLOWED_SCHOOL_YEAR:-2034}
       ALLOW_TYPE_COERCION: ${ALLOW_TYPE_COERCION:-true}
       ALLOW__EXT_PROPERTY: ${ALLOW__EXT_PROPERTY:-true}
+      ALLOW_OVERPOSTING: ${ALLOW_OVERPOSTING:-true}
       MEADOWLARK_DATABASE_NAME: ${MEADOWLARK_DATABASE_NAME:-meadowlark}
       POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
       POSTGRES_USER: ${POSTGRES_USER:-postgres}

Meadowlark-js/packages/meadowlark-core/src/metaed/MetaEdValidation.ts

@@ -24,15 +24,16 @@ export type ResourceSchemaValidators = {
 };
 
 function initializeAjv(): Ajv {
+  const removeAdditional = getBooleanFromEnvironment('ALLOW_OVERPOSTING', false);
   const coerceTypes = getBooleanFromEnvironment('ALLOW_TYPE_COERCION', false);
 
-  const ajv = new Ajv({ allErrors: true, coerceTypes });
+  const ajv = new Ajv({ allErrors: true, coerceTypes, removeAdditional });
   addFormatsTo(ajv);
 
   return ajv;
 }
 
-const ajv = initializeAjv();
+let ajv;
 
 // simple cache implementation, see: https://rewind.io/blog/simple-caching-in-aws-lambda-functions/
 /** This is a cache mapping MetaEd model objects to compiled ajv JSON Schema validators for the API resource */
@@ -44,7 +45,7 @@ const validatorCache: Map<TopLevelEntity, ResourceSchemaValidators> = new Map();
 function getSchemaValidatorsFor(metaEdModel: TopLevelEntity): ResourceSchemaValidators {
   const cachedValidators: ResourceSchemaValidators | undefined = validatorCache.get(metaEdModel);
   if (cachedValidators != null) return cachedValidators;
-
+  ajv = initializeAjv();
   const resourceValidators: ResourceSchemaValidators = {
     insertValidator: ajv.compile(metaEdModel.data.edfiApiSchema.jsonSchemaForInsert),
     updateValidator: ajv.compile(metaEdModel.data.edfiApiSchema.jsonSchemaForUpdate),

Meadowlark-js/packages/meadowlark-core/src/middleware/ValidateDocumentMiddleware.ts

@@ -3,7 +3,7 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
-import { validateDocument } from '../validation/DocumentValidator';
+import { ValidationFailure, validateDocument } from '../validation/DocumentValidator';
 import { writeDebugObject, writeRequestToLog } from '../Logger';
 import { MiddlewareModel } from './MiddlewareModel';
 
@@ -17,18 +17,17 @@ export async function documentValidation({ frontendRequest, frontendResponse }:
   // if there is a response already posted, we are done
   if (frontendResponse != null) return { frontendRequest, frontendResponse };
   writeRequestToLog(moduleName, frontendRequest, 'documentValidation');
-  const error: object | null = await validateDocument(
+  const validationError: ValidationFailure | null = await validateDocument(
     frontendRequest.middleware.parsedBody,
     frontendRequest.middleware.matchingMetaEdModel,
     isUpdate,
   );
-
-  if (error != null) {
+  if (validationError) {
     const statusCode = 400;
-    writeDebugObject(moduleName, frontendRequest, 'documentValidation', statusCode, error);
+    writeDebugObject(moduleName, frontendRequest, 'documentValidation', statusCode, validationError);
     return {
       frontendRequest,
-      frontendResponse: { body: error, statusCode, headers: frontendRequest.middleware.headerMetadata },
+      frontendResponse: { body: validationError, statusCode, headers: frontendRequest.middleware.headerMetadata },
     };
   }
 

Meadowlark-js/packages/meadowlark-core/test/metaed/MetaEdValidation.test.ts

@@ -3,6 +3,7 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+import { Config, Environment } from '@edfi/meadowlark-utilities';
 import { newTopLevelEntity, newEntityProperty, TopLevelEntity } from '@edfi/metaed-core';
 import { validateQueryParametersAgainstSchema } from '../../src/metaed/MetaEdValidation';
 
@@ -99,15 +100,25 @@ describe('given query parameters have a valid property', () => {
   });
 });
 
-describe('given query parameters have two invalid properties and a valid one', () => {
+describe('given query parameters with allow overposting is false have two invalid properties and a valid one', () => {
   let validationResult: string[];
 
   beforeAll(() => {
     const queryParameters = { uniqueId: 'a', one: 'one', two: 'two' };
-
+    jest.clearAllMocks();
+    jest.spyOn(Environment, 'getBooleanFromEnvironment').mockImplementationOnce((key: Config.ConfigKeys) => {
+      if (key === 'ALLOW_OVERPOSTING') {
+        return false;
+      }
+      return Environment.getBooleanFromEnvironment(key);
+    });
     validationResult = validateQueryParametersAgainstSchema(createModel(), queryParameters);
   });
 
+  afterAll(() => {
+    jest.restoreAllMocks();
+  });
+
   it('should have two errors', () => {
     expect(validationResult).toHaveLength(2);
   });
@@ -121,6 +132,29 @@ describe('given query parameters have two invalid properties and a valid one', (
   });
 });
 
+describe('given query parameters with allow overposting is true have two extraneous properties and a valid one', () => {
+  let validationResult: string[];
+
+  beforeAll(() => {
+    const queryParameters = { uniqueId: 'a', one: 'one', two: 'two' };
+    jest.clearAllMocks();
+    jest.spyOn(Environment, 'getBooleanFromEnvironment').mockImplementationOnce((key: Config.ConfigKeys) => {
+      if (key === 'ALLOW_OVERPOSTING') {
+        return true;
+      }
+      return Environment.getBooleanFromEnvironment(key);
+    });
+    validationResult = validateQueryParametersAgainstSchema(createModel(), queryParameters);
+  });
+  afterAll(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('should not have errors', () => {
+    expect(validationResult).toHaveLength(0);
+  });
+});
+
 describe('given a boolean query parameter value of true', () => {
   let validationResult: string[];
 

Meadowlark-js/packages/meadowlark-utilities/src/Config.ts

@@ -21,6 +21,7 @@ export type ConfigKeys =
   | 'OAUTH_HARD_CODED_CREDENTIALS_ENABLED'
   | 'ALLOW_TYPE_COERCION'
   | 'ALLOW__EXT_PROPERTY'
+  | 'ALLOW_OVERPOSTING'
   | 'FASTIFY_NUM_THREADS'
   | 'MEADOWLARK_DATABASE_NAME'
   | 'MONGO_URI'
@@ -151,6 +152,7 @@ export async function initializeConfig(provider: ConfigPlugin) {
 
   set('ALLOW_TYPE_COERCION', await provider.getBool('ALLOW_TYPE_COERCION', false));
   set('ALLOW__EXT_PROPERTY', await provider.getBool('ALLOW__EXT_PROPERTY', false));
+  set('ALLOW_OVERPOSTING', await provider.getBool('ALLOW_OVERPOSTING', true));
   set('FASTIFY_NUM_THREADS', await provider.getInt('FASTIFY_NUM_THREADS', CpuCount));
   set('FASTIFY_PORT', await provider.getInt('FASTIFY_PORT', 3000));
 

Meadowlark-js/packages/meadowlark-utilities/src/index.ts

@@ -4,6 +4,7 @@
 // See the LICENSE and NOTICES files in the project root for more information.
 
 export * from './Environment';
+export * as Environment from './Environment';
 export { Logger, initializeLogging, isDebugEnabled, isInfoEnabled, writeErrorToLog } from './Logger';
 export { LOCATION_HEADER_NAME } from './LocationHeader';
 export * as Config from './Config';

Meadowlark-js/packages/meadowlark-utilities/test/Config.test.ts

@@ -51,6 +51,7 @@ describe('when initializing configuration', () => {
   const OPENSEARCH_USERNAME = 'b5';
   const OPENSEARCH_PASSWORD = 'b6';
   const ALLOW_TYPE_COERCION = true;
+  const ALLOW_OVERPOSTING = true;
   const ALLOW__EXT_PROPERTY = true;
   const FASTIFY_NUM_THREADS = 99;
   const OAUTH_EXPIRATION_MINUTES = -10;
@@ -94,6 +95,7 @@ describe('when initializing configuration', () => {
     process.env.FASTIFY_NUM_THREADS = FASTIFY_NUM_THREADS.toString();
     process.env.ALLOW__EXT_PROPERTY = ALLOW__EXT_PROPERTY.toString();
     process.env.ALLOW_TYPE_COERCION = ALLOW_TYPE_COERCION.toString();
+    process.env.ALLOW_OVERPOSTING = ALLOW_OVERPOSTING.toString();
     process.env.OPENSEARCH_PASSWORD = OPENSEARCH_PASSWORD;
     process.env.OPENSEARCH_USERNAME = OPENSEARCH_USERNAME;
     process.env.OPENSEARCH_ENDPOINT = OPENSEARCH_ENDPOINT;
@@ -185,6 +187,7 @@ describe('when initializing configuration', () => {
       ['OPENSEARCH_PASSWORD', OPENSEARCH_PASSWORD],
       ['ALLOW_TYPE_COERCION', ALLOW_TYPE_COERCION],
       ['ALLOW__EXT_PROPERTY', ALLOW__EXT_PROPERTY],
+      ['ALLOW_OVERPOSTING', ALLOW_OVERPOSTING],
       ['FASTIFY_NUM_THREADS', FASTIFY_NUM_THREADS],
       ['OAUTH_EXPIRATION_MINUTES', OAUTH_EXPIRATION_MINUTES],
       ['OAUTH_TOKEN_ISSUER', OAUTH_TOKEN_ISSUER],
@@ -227,6 +230,7 @@ describe('when initializing configuration', () => {
       process.env.QUERY_HANDLER_PLUGIN = undefined;
       process.env.ALLOW_TYPE_COERCION = undefined;
       process.env.ALLOW__EXT_PROPERTY = undefined;
+      process.env.ALLOW_OVERPOSTING = undefined;
       process.env.FASTIFY_NUM_THREADS = undefined;
       process.env.OAUTH_EXPIRATION_MINUTES = undefined;
       process.env.OAUTH_TOKEN_ISSUER = undefined;
@@ -272,6 +276,7 @@ describe('when initializing configuration', () => {
       ['QUERY_HANDLER_PLUGIN', ''],
       ['ALLOW_TYPE_COERCION', false],
       ['ALLOW__EXT_PROPERTY', false],
+      ['ALLOW_OVERPOSTING', false],
       ['FASTIFY_NUM_THREADS', CpuCount],
       ['OAUTH_EXPIRATION_MINUTES', 60],
       ['OAUTH_TOKEN_ISSUER', 'edfi-meadowlark-issuer'],

Meadowlark-js/services/meadowlark-fastify/.env.example

@@ -92,3 +92,7 @@ ALLOW_TYPE_COERCION=true
 # Allow the API to accept an extra { _ext: {} } property in a payload. Must be
 # true when bulk uploading the ODS/API "populated template".
 ALLOW__EXT_PROPERTY=true
+
+# Allow the API to ignore elements that are not part of the schema are ignored.
+# if false, it returns an error message if the element is not part of the schema.
+ALLOW_OVERPOSTING=true