[RND-651] Ability to loosely allow overposted material for parity (#313)

* [RND-651]  Ability to loosely allow overposted material for parity

- Add parameter removeAdditional to ajv instantiation
- Add flag ALLOW_OVERPOSTING to enable o disable the removeAddtional. By default is false, so it doesn't remove additional fields.
- Add package unit tests  to validate
- Add e2e to validate post and put using the default case that doesn't remove additional.

* Update SchemaValidation.test.ts

Fix delete

* Update SchemaValidation.test.ts

Remove deletes

* Update SchemaValidation.test.ts

Fix linting

* Update SchemaValidation.test.ts

Delete inserted resource

* Add tests

Create additional unit test
Add Integration test to validate upsert and update.

* Update package.json

Update test:integration to add meadwolark-core

* Add function to clear validationCache to test validators

* Update MetaEdValidation.test.ts

Add test to validate cases without extraneous properties.

* Update MetaEdValidation.test.ts

Update tests

* Some little improvements on a few tests descriptions.

---------

Co-authored-by: David Jimenez Barrantes <[email protected]>

Meadowlark-js/.env.example

@@ -41,6 +41,9 @@ ALLOW_TYPE_COERCION=true
 # true when bulk uploading the ODS/API "populated template".
 ALLOW__EXT_PROPERTY=true
 
+# Allow the API to ignore elements that are not part of the schema are ignored.
+# if false, it returns an error message if the element is not part of the schema.
+ALLOW_OVERPOSTING=true
 
 SAVE_LOG_TO_FILE=false
 # LOG_FILE_LOCATION=c:/temp/

Meadowlark-js/docker-compose.yml

@@ -40,6 +40,7 @@ services:
       END_ALLOWED_SCHOOL_YEAR: ${END_ALLOWED_SCHOOL_YEAR:-2034}
       ALLOW_TYPE_COERCION: ${ALLOW_TYPE_COERCION:-true}
       ALLOW__EXT_PROPERTY: ${ALLOW__EXT_PROPERTY:-true}
+      ALLOW_OVERPOSTING: ${ALLOW_OVERPOSTING:-false}
       MEADOWLARK_DATABASE_NAME: ${MEADOWLARK_DATABASE_NAME:-meadowlark}
       POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
       POSTGRES_USER: ${POSTGRES_USER:-postgres}

Meadowlark-js/jest.config.js

@@ -5,6 +5,7 @@ module.exports = {
     '<rootDir>/backends/meadowlark-mongodb-backend/test/config/integration/jest.config.js',
     '<rootDir>/backends/meadowlark-opensearch-backend/test/config/integration/jest.config.js',
     '<rootDir>/backends/meadowlark-postgresql-backend/test/config/integration/jest.config.js',
+    '<rootDir>/packages/meadowlark-core/test/config/integration/jest.config.js',
   ],
   coverageThreshold: {
     global: {

Meadowlark-js/package.json

@@ -54,10 +54,11 @@
     "test:lint:eslint": "eslint --max-warnings 0 --ext .js,.ts .",
     "test:lint:ts": "tsc -p . --noEmit",
     "test:unit": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects tests/config/unit",
-    "test:integration": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration backends/meadowlark-opensearch-backend/test/config/integration backends/meadowlark-elasticsearch-backend/test/config/integration backends/meadowlark-postgresql-backend/test/config/integration --runInBand",
+    "test:integration": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration backends/meadowlark-opensearch-backend/test/config/integration backends/meadowlark-elasticsearch-backend/test/config/integration backends/meadowlark-postgresql-backend/test/config/integration packages/meadowlark-core/test/config/integration --runInBand",
+    "test:integration:elasticsearch": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-elasticsearch-backend/test/config/integration --runInBand",
+    "test:integration:meadowlark-core": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects packages/meadowlark-core/test/config/integration --runInBand",
     "test:integration:mongodb": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration --runInBand",
     "test:integration:opensearch": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-opensearch-backend/test/config/integration --runInBand",
-    "test:integration:elasticsearch": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-elasticsearch-backend/test/config/integration --runInBand",
     "test:integration:postgresql": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-postgresql-backend/test/config/integration --runInBand",
     "test:unit:coverage": "rimraf coverage.unit/ && npm run test:unit",
     "test:unit:coverage:ci": "rimraf coverage.unit/ && npm run test:unit -- --maxWorkers=2 --ci",

Meadowlark-js/packages/meadowlark-core/src/metaed/MetaEdValidation.ts

@@ -24,15 +24,16 @@ export type ResourceSchemaValidators = {
 };
 
 function initializeAjv(): Ajv {
+  const removeAdditional = getBooleanFromEnvironment('ALLOW_OVERPOSTING', false);
   const coerceTypes = getBooleanFromEnvironment('ALLOW_TYPE_COERCION', false);
 
-  const ajv = new Ajv({ allErrors: true, coerceTypes });
+  const ajv = new Ajv({ allErrors: true, coerceTypes, removeAdditional });
   addFormatsTo(ajv);
 
   return ajv;
 }
 
-const ajv = initializeAjv();
+let ajv;
 
 // simple cache implementation, see: https://rewind.io/blog/simple-caching-in-aws-lambda-functions/
 /** This is a cache mapping MetaEd model objects to compiled ajv JSON Schema validators for the API resource */
@@ -44,7 +45,7 @@ const validatorCache: Map<TopLevelEntity, ResourceSchemaValidators> = new Map();
 function getSchemaValidatorsFor(metaEdModel: TopLevelEntity): ResourceSchemaValidators {
   const cachedValidators: ResourceSchemaValidators | undefined = validatorCache.get(metaEdModel);
   if (cachedValidators != null) return cachedValidators;
-
+  ajv = initializeAjv();
   const resourceValidators: ResourceSchemaValidators = {
     insertValidator: ajv.compile(metaEdModel.data.edfiApiSchema.jsonSchemaForInsert),
     updateValidator: ajv.compile(metaEdModel.data.edfiApiSchema.jsonSchemaForUpdate),
@@ -58,6 +59,12 @@ function getSchemaValidatorsFor(metaEdModel: TopLevelEntity): ResourceSchemaVali
   return resourceValidators;
 }
 
+/**
+ * Function to delete all item from validatorCache for testing purposes.
+ */
+export function clearAllValidatorCache(): void {
+  validatorCache.clear();
+}
 /**
  * Creates a new empty ResourceMatchResult object
  */

Meadowlark-js/packages/meadowlark-core/test/ConfigHelper.ts

@@ -53,6 +53,10 @@ export const setupMockConfiguration = (isDebug: boolean = false, disableAnonymiz
         return 'mwc';
       case 'MONGO_READ_CONCERN':
         return 'mrc';
+      case 'ALLOW_TYPE_COERCION':
+        return false;
+      case 'ALLOW_OVERPOSTING':
+        return false;
       default:
         throw new Error(`Key '${key}' not configured`);
     }

Meadowlark-js/packages/meadowlark-core/test/config/integration/jest.config.js

@@ -0,0 +1,20 @@
+const rootDir = '../../../../../';
+// eslint-disable-next-line import/no-extraneous-dependencies
+const defaultConfig = require(`${rootDir}/tests/config/jest.config`);
+
+module.exports = {
+  displayName: 'Integration Tests: meadowlark-core',
+  ...defaultConfig,
+  testMatch: ['**/meadowlark-core/test/integration/**/*.(spec|test).[jt]s?(x)'],
+  watchPathIgnorePatterns: ['globalConfig'], // jest-mongodb setup
+  coverageThreshold: {
+    global: {
+      branches: 52,
+      functions: 58,
+      lines: 60,
+      statements: 60,
+    },
+  },
+  rootDir,
+  workerIdleMemoryLimit: '200MB',
+};