[RND-525] - Adds support for ElasticSearch (#257)

* Clone from opensearch to create elasticsearch backend.

* Code changes. Trying to fix tests.

* Adds elastic/transport

* Trying to fix linting problems. Working on fixing tests.

* Fixes problem with number of documents.

* All integration tests working fine.

* Removes poc. I do not need it.

* First unit tests working fine.

* More unit tests now working fine.

* All tests, integration and unit, working fine.

* Renaming files. Code cleanup.

* Some code clean up. Changes on readme file.

* Adds elastic search to integration tests script.

* Changes on docker-compose file.

* Some changes on documentation files.

* Adds SHA to docker-compose file.

* Adds env values to env.example and docs folder.

* Fixes name for Elasticsearch timeout var.

Author: DavidJGapCR

Meadowlark-js/backends/meadowlark-elasticsearch-backend/.npmrc

@@ -0,0 +1,2 @@
+@edfi:registry=https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/
+always-auth=false

Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3.8"
+
+volumes:
+  esdata01:
+    driver: local
+
+networks:
+  default:
+    name: elastic
+    external: false
+
+services:
+  elasticsearch-node1:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0@sha256:9aaa38551b4d9e655c54d9dc6a1dad24ee568c41952dc8cf1d4808513cfb5f65
+    container_name: elasticsearch-node1
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    environment:
+      - node.name=es01
+      - cluster.name=elasticsearch-node1
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+    mem_limit: 2g

Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/readme.md

@@ -0,0 +1,14 @@
+# Elasticsearch Backend for Meadowlark
+
+:exclamation: This solution should only be used on localhost with proper firewalls around
+external network access to the workstation. Not appropriate for production use.
+
+This Docker Compose file provisions a single node of the ElasticSearch search
+engine.
+
+## Test dependency on older "docker-compose" versus newer "docker compose"
+
+The integration tests use the testcontainers library to spin up an Elasticsearch instance. As of
+Feb 2023, it uses the legacy "docker-compose" command from Compose V1. If tests fail
+with "Error: spawn docker-compose ENOENT", you will need to either [install Compose V1
+standalone](https://docs.docker.com/compose/install/other/) or `alias docker-compose='docker compose'`.

Meadowlark-js/backends/meadowlark-elasticsearch-backend/index.ts

@@ -0,0 +1,4 @@
+// Reminder - this index.ts is for on-the-fly transpile when there is no dist directory
+
+// All exports from the "real" index.ts
+export * from './src/index';

Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@edfi/meadowlark-elasticsearch-backend",
+  "main": "dist/index.js",
+  "version": "0.3.0-pre-36",
+  "description": "Meadowlark backend plugin for elasticsearch",
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "registry": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/"
+  },
+  "files": [
+    "/dist",
+    "/LICENSE.md",
+    "/package.json"
+  ],
+  "scripts": {
+    "build": "npm run build:clean && npm run build:copy-non-ts && npm run build:dist",
+    "build:clean": "rimraf dist",
+    "build:dist": "tsc",
+    "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose"
+  },
+  "dependencies": {
+    "@edfi/meadowlark-core": "^v0.3.0-pre-35",
+    "@edfi/meadowlark-utilities": "^v0.3.0-pre-35",
+    "@edfi/metaed-core": "^4.0.1-dev.13",
+    "@elastic/elasticsearch": "^8.8.0",
+    "@elastic/transport": "^8.3.2"
+  },
+  "devDependencies": {
+    "@elastic/elasticsearch-mock": "^2.0.0",
+    "copyfiles": "^2.4.1",
+    "dotenv": "^16.0.3",
+    "rimraf": "^3.0.2",
+    "testcontainers": "^9.8.0"
+  }
+}

Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/BackendFacade.ts

@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+import {
+  DeleteRequest,
+  DeleteResult,
+  QueryRequest,
+  QueryResult,
+  UpdateRequest,
+  UpdateResult,
+  UpsertRequest,
+  UpsertResult,
+} from '@edfi/meadowlark-core';
+import * as QueryElasticsearch from './repository/QueryElasticsearch';
+import * as UpdateElasticsearch from './repository/UpdateElasticsearch';
+import { getSharedClient, closeSharedConnection } from './repository/Db';
+
+export async function queryDocuments(request: QueryRequest): Promise<QueryResult> {
+  return QueryElasticsearch.queryDocuments(request, await getSharedClient());
+}
+
+export async function afterDeleteDocumentById(request: DeleteRequest, result: DeleteResult): Promise<void> {
+  return UpdateElasticsearch.afterDeleteDocumentById(request, result, await getSharedClient());
+}
+
+export async function afterUpsertDocument(request: UpsertRequest, result: UpsertResult): Promise<void> {
+  return UpdateElasticsearch.afterUpsertDocument(request, result, await getSharedClient());
+}
+
+export async function afterUpdateDocumentById(request: UpdateRequest, result: UpdateResult): Promise<void> {
+  return UpdateElasticsearch.afterUpdateDocumentById(request, result, await getSharedClient());
+}
+
+export async function closeConnection(): Promise<void> {
+  return closeSharedConnection();
+}

Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/index.ts

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+import { QueryHandlerPlugin, Subscribe } from '@edfi/meadowlark-core';
+import {
+  afterDeleteDocumentById,
+  afterUpdateDocumentById,
+  afterUpsertDocument,
+  queryDocuments,
+  closeConnection,
+} from './BackendFacade';
+
+export function initializeQueryHandler(): QueryHandlerPlugin {
+  return {
+    queryDocuments,
+    closeConnection,
+  };
+}
+
+export function initializeListener(subscribe: typeof Subscribe): void {
+  subscribe.afterDeleteDocumentById(afterDeleteDocumentById);
+  subscribe.afterUpsertDocument(afterUpsertDocument);
+  subscribe.afterUpdateDocumentById(afterUpdateDocumentById);
+}

Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/Db.ts

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+import { Client, ClientOptions } from '@elastic/elasticsearch';
+import { Config, Logger } from '@edfi/meadowlark-utilities';
+import { BasicAuth } from '@elastic/transport/lib/types';
+
+let singletonClient: Client | null = null;
+
+const moduleName = 'elasticsearch.repository.Db';
+
+export async function getElasticSearchClient(node?: string, auth?: BasicAuth, requestTimeout?: number): Promise<Client> {
+  Logger.debug(`${moduleName}.getElasticSearchClient creating local client`, null);
+  const clientOpts: ClientOptions = {
+    node,
+    auth,
+    requestTimeout,
+    /* Might need to setup SSL here in the future */
+  };
+  try {
+    return new Client(clientOpts);
+  } catch (e) {
+    const masked = { ...clientOpts } as any;
+    delete masked.auth?.password;
+
+    Logger.error(`${moduleName}.getElasticSearchClient error connecting with options ${JSON.stringify(masked)}`, null, e);
+    throw e;
+  }
+}
+
+/**
+ * Create and return an ElasticSearch connection object
+ */
+export async function getNewClient(): Promise<Client> {
+  Logger.debug(`${moduleName}.getNewClient creating local client`, null);
+  const node = Config.get<string>('ELASTICSEARCH_ENDPOINT');
+  const auth = {
+    username: Config.get<string>('ELASTICSEARCH_USERNAME'),
+    password: Config.get<string>('ELASTICSEARCH_PASSWORD'),
+  };
+  const requestTimeout = Config.get<number>('ELASTICSEARCH_REQUEST_TIMEOUT');
+  return getElasticSearchClient(node, auth, requestTimeout);
+}
+
+/**
+ * Return the shared client
+ */
+export async function getSharedClient(): Promise<Client> {
+  if (singletonClient == null) {
+    singletonClient = await getNewClient();
+  }
+
+  return singletonClient;
+}
+
+export async function closeSharedConnection(): Promise<void> {
+  if (singletonClient != null) {
+    await singletonClient.close();
+  }
+  singletonClient = null;
+  Logger.info(`Elasticsearch connection: closed`, null);
+}

Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/ElasticSearchException.ts

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+import { ElasticsearchClientError, ResponseError } from '@elastic/transport/lib/errors';
+import { Logger } from '@edfi/meadowlark-utilities';
+import { QueryResult } from '@edfi/meadowlark-core';
+
+export async function handleElasticSearchError(
+  err: ElasticsearchClientError | Error,
+  moduleName: string = '',
+  traceId: string = '',
+  elasticSearchRequestId: string = '',
+): Promise<QueryResult> {
+  try {
+    const elasticSearchClientError = err as ElasticsearchClientError;
+    const documentProcessError = `ElasticSearch Error${
+      elasticSearchRequestId ? ` processing the object '${elasticSearchRequestId}'` : ''
+    }:`;
+    if (elasticSearchClientError?.name !== undefined) {
+      switch (elasticSearchClientError.name) {
+        case 'ConfigurationError':
+        case 'ConnectionError':
+        case 'DeserializationError':
+        case 'NoLivingConnectionsError':
+        case 'NotCompatibleError':
+        case 'ElasticSearchClientError':
+        case 'RequestAbortedError':
+        case 'SerializationError':
+        case 'TimeoutError': {
+          if (elasticSearchClientError?.message !== undefined) {
+            Logger.error(
+              `${moduleName} ${documentProcessError}`,
+              traceId,
+              `(${elasticSearchClientError.name}) - ${elasticSearchClientError.message}`,
+            );
+            return {
+              response: 'QUERY_FAILURE_INVALID_QUERY',
+              documents: [],
+              failureMessage: elasticSearchClientError.message,
+            };
+          }
+          break;
+        }
+        case 'ResponseError': {
+          const responseException = err as ResponseError;
+          if (responseException?.message !== undefined) {
+            if (responseException.message !== 'Response Error') {
+              let startPosition = responseException?.message?.indexOf('Reason:');
+              const position = responseException?.message?.indexOf(' Preview');
+              startPosition = startPosition > -1 ? startPosition : 0;
+              if (position > -1) {
+                responseException.message = responseException?.message?.substring(startPosition, position);
+              } else if (startPosition !== 0) {
+                responseException.message = responseException?.message?.substring(startPosition);
+              }
+              Logger.error(
+                `${moduleName} ${documentProcessError}`,
+                traceId,
+                `(${elasticSearchClientError.name}) - ${elasticSearchClientError.message}`,
+              );
+              return { response: 'QUERY_FAILURE_INVALID_QUERY', documents: [], failureMessage: responseException.message };
+            }
+            if (responseException?.body !== undefined) {
+              let responseBody = JSON.parse(responseException?.body?.toString());
+              if (responseBody?.error?.type !== undefined) {
+                switch (responseBody?.error?.type) {
+                  case 'IndexNotFoundException':
+                    // No object has been uploaded for the requested type
+                    Logger.warn(`${moduleName} ${documentProcessError} index not found`, traceId);
+                    return {
+                      response: 'QUERY_FAILURE_INVALID_QUERY',
+                      documents: [],
+                      failureMessage: 'IndexNotFoundException',
+                    };
+                  case 'SemanticAnalysisException':
+                    // The query term is invalid
+                    Logger.error(
+                      `${moduleName} ${documentProcessError} invalid query terms`,
+                      traceId,
+                      `(${elasticSearchClientError.name}) - ${responseBody?.error?.reason}`,
+                    );
+                    return {
+                      response: 'QUERY_FAILURE_INVALID_QUERY',
+                      documents: [],
+                      failureMessage: responseBody?.error?.details,
+                    };
+                  default:
+                    Logger.error(`${moduleName} ${documentProcessError}`, traceId, responseBody ?? err);
+                    return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: responseBody };
+                }
+              } else {
+                responseBody = JSON.parse(JSON.stringify(responseException.body));
+                Logger.error(`${moduleName} ${documentProcessError}`, traceId, responseBody ?? err);
+                return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: responseBody };
+              }
+            }
+          }
+          break;
+        }
+        default: {
+          break;
+        }
+      }
+    }
+    Logger.error(`${moduleName} UNKNOWN_FAILURE`, traceId, err);
+    return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: err.message };
+  } catch {
+    Logger.error(`${moduleName} UNKNOWN_FAILURE`, traceId, err);
+    return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: err.message };
+  }
+}