diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/.npmrc b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/.npmrc new file mode 100644 index 00000000..fd1a1780 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/.npmrc @@ -0,0 +1,2 @@ +@edfi:registry=https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/ +always-auth=false \ No newline at end of file diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/docker-compose.yml b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/docker-compose.yml new file mode 100644 index 00000000..95d5230e --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3.8" + +volumes: + esdata01: + driver: local + +networks: + default: + name: elastic + external: false + +services: + elasticsearch-node1: + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0@sha256:9aaa38551b4d9e655c54d9dc6a1dad24ee568c41952dc8cf1d4808513cfb5f65 + container_name: elasticsearch-node1 + labels: + co.elastic.logs/module: elasticsearch + volumes: + - esdata01:/usr/share/elasticsearch/data + ports: + - 9200:9200 + environment: + - node.name=es01 + - cluster.name=elasticsearch-node1 + - discovery.type=single-node + - xpack.security.enabled=false + mem_limit: 2g diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/readme.md b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/readme.md new file mode 100644 index 00000000..d38b3464 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker/readme.md @@ -0,0 +1,14 @@ +# Elasticsearch Backend for Meadowlark + +:exclamation: This solution should only be used on localhost with proper firewalls around +external network access to the workstation. Not appropriate for production use. + +This Docker Compose file provisions a single node of the ElasticSearch search +engine. + +## Test dependency on older "docker-compose" versus newer "docker compose" + +The integration tests use the testcontainers library to spin up an Elasticsearch instance. As of +Feb 2023, it uses the legacy "docker-compose" command from Compose V1. If tests fail +with "Error: spawn docker-compose ENOENT", you will need to either [install Compose V1 +standalone](https://docs.docker.com/compose/install/other/) or `alias docker-compose='docker compose'`. diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/index.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/index.ts new file mode 100644 index 00000000..fa2bffa5 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/index.ts @@ -0,0 +1,4 @@ +// Reminder - this index.ts is for on-the-fly transpile when there is no dist directory + +// All exports from the "real" index.ts +export * from './src/index'; diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json new file mode 100644 index 00000000..6fe80a68 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/package.json @@ -0,0 +1,35 @@ +{ + "name": "@edfi/meadowlark-elasticsearch-backend", + "main": "dist/index.js", + "version": "0.3.0-pre-36", + "description": "Meadowlark backend plugin for elasticsearch", + "license": "Apache-2.0", + "publishConfig": { + "registry": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/npm/registry/" + }, + "files": [ + "/dist", + "/LICENSE.md", + "/package.json" + ], + "scripts": { + "build": "npm run build:clean && npm run build:copy-non-ts && npm run build:dist", + "build:clean": "rimraf dist", + "build:dist": "tsc", + "build:copy-non-ts": "copyfiles -u 1 -e \"**/*.ts\" \"src/**/*\" dist --verbose" + }, + "dependencies": { + "@edfi/meadowlark-core": "^v0.3.0-pre-35", + "@edfi/meadowlark-utilities": "^v0.3.0-pre-35", + "@edfi/metaed-core": "^4.0.1-dev.13", + "@elastic/elasticsearch": "^8.8.0", + "@elastic/transport": "^8.3.2" + }, + "devDependencies": { + "@elastic/elasticsearch-mock": "^2.0.0", + "copyfiles": "^2.4.1", + "dotenv": "^16.0.3", + "rimraf": "^3.0.2", + "testcontainers": "^9.8.0" + } +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/BackendFacade.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/BackendFacade.ts new file mode 100644 index 00000000..b4c0300f --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/BackendFacade.ts @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { + DeleteRequest, + DeleteResult, + QueryRequest, + QueryResult, + UpdateRequest, + UpdateResult, + UpsertRequest, + UpsertResult, +} from '@edfi/meadowlark-core'; +import * as QueryElasticsearch from './repository/QueryElasticsearch'; +import * as UpdateElasticsearch from './repository/UpdateElasticsearch'; +import { getSharedClient, closeSharedConnection } from './repository/Db'; + +export async function queryDocuments(request: QueryRequest): Promise { + return QueryElasticsearch.queryDocuments(request, await getSharedClient()); +} + +export async function afterDeleteDocumentById(request: DeleteRequest, result: DeleteResult): Promise { + return UpdateElasticsearch.afterDeleteDocumentById(request, result, await getSharedClient()); +} + +export async function afterUpsertDocument(request: UpsertRequest, result: UpsertResult): Promise { + return UpdateElasticsearch.afterUpsertDocument(request, result, await getSharedClient()); +} + +export async function afterUpdateDocumentById(request: UpdateRequest, result: UpdateResult): Promise { + return UpdateElasticsearch.afterUpdateDocumentById(request, result, await getSharedClient()); +} + +export async function closeConnection(): Promise { + return closeSharedConnection(); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/index.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/index.ts new file mode 100644 index 00000000..6d116b44 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/index.ts @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { QueryHandlerPlugin, Subscribe } from '@edfi/meadowlark-core'; +import { + afterDeleteDocumentById, + afterUpdateDocumentById, + afterUpsertDocument, + queryDocuments, + closeConnection, +} from './BackendFacade'; + +export function initializeQueryHandler(): QueryHandlerPlugin { + return { + queryDocuments, + closeConnection, + }; +} + +export function initializeListener(subscribe: typeof Subscribe): void { + subscribe.afterDeleteDocumentById(afterDeleteDocumentById); + subscribe.afterUpsertDocument(afterUpsertDocument); + subscribe.afterUpdateDocumentById(afterUpdateDocumentById); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/Db.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/Db.ts new file mode 100644 index 00000000..f9c57e63 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/Db.ts @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { Client, ClientOptions } from '@elastic/elasticsearch'; +import { Config, Logger } from '@edfi/meadowlark-utilities'; +import { BasicAuth } from '@elastic/transport/lib/types'; + +let singletonClient: Client | null = null; + +const moduleName = 'elasticsearch.repository.Db'; + +export async function getElasticSearchClient(node?: string, auth?: BasicAuth, requestTimeout?: number): Promise { + Logger.debug(`${moduleName}.getElasticSearchClient creating local client`, null); + const clientOpts: ClientOptions = { + node, + auth, + requestTimeout, + /* Might need to setup SSL here in the future */ + }; + try { + return new Client(clientOpts); + } catch (e) { + const masked = { ...clientOpts } as any; + delete masked.auth?.password; + + Logger.error(`${moduleName}.getElasticSearchClient error connecting with options ${JSON.stringify(masked)}`, null, e); + throw e; + } +} + +/** + * Create and return an ElasticSearch connection object + */ +export async function getNewClient(): Promise { + Logger.debug(`${moduleName}.getNewClient creating local client`, null); + const node = Config.get('ELASTICSEARCH_ENDPOINT'); + const auth = { + username: Config.get('ELASTICSEARCH_USERNAME'), + password: Config.get('ELASTICSEARCH_PASSWORD'), + }; + const requestTimeout = Config.get('ELASTICSEARCH_REQUEST_TIMEOUT'); + return getElasticSearchClient(node, auth, requestTimeout); +} + +/** + * Return the shared client + */ +export async function getSharedClient(): Promise { + if (singletonClient == null) { + singletonClient = await getNewClient(); + } + + return singletonClient; +} + +export async function closeSharedConnection(): Promise { + if (singletonClient != null) { + await singletonClient.close(); + } + singletonClient = null; + Logger.info(`Elasticsearch connection: closed`, null); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/ElasticSearchException.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/ElasticSearchException.ts new file mode 100644 index 00000000..f4298686 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/ElasticSearchException.ts @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { ElasticsearchClientError, ResponseError } from '@elastic/transport/lib/errors'; +import { Logger } from '@edfi/meadowlark-utilities'; +import { QueryResult } from '@edfi/meadowlark-core'; + +export async function handleElasticSearchError( + err: ElasticsearchClientError | Error, + moduleName: string = '', + traceId: string = '', + elasticSearchRequestId: string = '', +): Promise { + try { + const elasticSearchClientError = err as ElasticsearchClientError; + const documentProcessError = `ElasticSearch Error${ + elasticSearchRequestId ? ` processing the object '${elasticSearchRequestId}'` : '' + }:`; + if (elasticSearchClientError?.name !== undefined) { + switch (elasticSearchClientError.name) { + case 'ConfigurationError': + case 'ConnectionError': + case 'DeserializationError': + case 'NoLivingConnectionsError': + case 'NotCompatibleError': + case 'ElasticSearchClientError': + case 'RequestAbortedError': + case 'SerializationError': + case 'TimeoutError': { + if (elasticSearchClientError?.message !== undefined) { + Logger.error( + `${moduleName} ${documentProcessError}`, + traceId, + `(${elasticSearchClientError.name}) - ${elasticSearchClientError.message}`, + ); + return { + response: 'QUERY_FAILURE_INVALID_QUERY', + documents: [], + failureMessage: elasticSearchClientError.message, + }; + } + break; + } + case 'ResponseError': { + const responseException = err as ResponseError; + if (responseException?.message !== undefined) { + if (responseException.message !== 'Response Error') { + let startPosition = responseException?.message?.indexOf('Reason:'); + const position = responseException?.message?.indexOf(' Preview'); + startPosition = startPosition > -1 ? startPosition : 0; + if (position > -1) { + responseException.message = responseException?.message?.substring(startPosition, position); + } else if (startPosition !== 0) { + responseException.message = responseException?.message?.substring(startPosition); + } + Logger.error( + `${moduleName} ${documentProcessError}`, + traceId, + `(${elasticSearchClientError.name}) - ${elasticSearchClientError.message}`, + ); + return { response: 'QUERY_FAILURE_INVALID_QUERY', documents: [], failureMessage: responseException.message }; + } + if (responseException?.body !== undefined) { + let responseBody = JSON.parse(responseException?.body?.toString()); + if (responseBody?.error?.type !== undefined) { + switch (responseBody?.error?.type) { + case 'IndexNotFoundException': + // No object has been uploaded for the requested type + Logger.warn(`${moduleName} ${documentProcessError} index not found`, traceId); + return { + response: 'QUERY_FAILURE_INVALID_QUERY', + documents: [], + failureMessage: 'IndexNotFoundException', + }; + case 'SemanticAnalysisException': + // The query term is invalid + Logger.error( + `${moduleName} ${documentProcessError} invalid query terms`, + traceId, + `(${elasticSearchClientError.name}) - ${responseBody?.error?.reason}`, + ); + return { + response: 'QUERY_FAILURE_INVALID_QUERY', + documents: [], + failureMessage: responseBody?.error?.details, + }; + default: + Logger.error(`${moduleName} ${documentProcessError}`, traceId, responseBody ?? err); + return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: responseBody }; + } + } else { + responseBody = JSON.parse(JSON.stringify(responseException.body)); + Logger.error(`${moduleName} ${documentProcessError}`, traceId, responseBody ?? err); + return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: responseBody }; + } + } + } + break; + } + default: { + break; + } + } + } + Logger.error(`${moduleName} UNKNOWN_FAILURE`, traceId, err); + return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: err.message }; + } catch { + Logger.error(`${moduleName} UNKNOWN_FAILURE`, traceId, err); + return { response: 'UNKNOWN_FAILURE', documents: [], failureMessage: err.message }; + } +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/QueryElasticsearch.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/QueryElasticsearch.ts new file mode 100644 index 00000000..cdf28061 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/QueryElasticsearch.ts @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { Client } from '@elastic/elasticsearch'; +import { QueryRequest, QueryResult, ResourceInfo } from '@edfi/meadowlark-core'; +import { isDebugEnabled, Logger } from '@edfi/meadowlark-utilities'; +import { normalizeDescriptorSuffix } from '@edfi/metaed-core'; +import { handleElasticSearchError } from './ElasticSearchException'; + +const moduleName = 'elasticsearch.repository.QueryElasticsearch'; + +/** + * Returns ElasticSearch index name from the given ResourceInfo. + */ +export function indexFromResourceInfo(resourceInfo: ResourceInfo): string { + const adjustedResourceName = resourceInfo.isDescriptor + ? normalizeDescriptorSuffix(resourceInfo.resourceName) + : resourceInfo.resourceName; + + return `${resourceInfo.projectName}$${resourceInfo.resourceVersion}$${adjustedResourceName}` + .toLowerCase() + .replace(/\./g, '-'); +} + +/** + * DSL querying + */ +async function performDslQuery(client: Client, path: string, query: any, size: number, from: number): Promise { + return client.transport.request({ + method: 'POST', + path: `/${path}/_search`, + body: { + from, + query, + size, + sort: [{ _doc: { order: 'asc' } }], + }, + }); +} + +/** + * Entry point for querying with ElasticSearch + */ +export async function queryDocuments(request: QueryRequest, client: Client): Promise { + const { resourceInfo, queryParameters, paginationParameters, traceId } = request; + + Logger.debug(`${moduleName}.queryDocuments Building query`, traceId); + + let documents: any = []; + let recordCount: number; + try { + const matches: any[] = []; + + // API client requested filters + if (Object.entries(queryParameters).length > 0) { + Object.entries(queryParameters).forEach(([key, value]) => { + matches.push({ + match: { [key]: value }, + }); + }); + } + + // Ownership-based security filter - if the resource is a descriptor we will ignore security + if (request.security.authorizationStrategy.type === 'OWNERSHIP_BASED' && !resourceInfo.isDescriptor) { + matches.push({ + match: { + createdBy: `"${request.security.clientId}"`, + }, + }); + } + + const query = { + bool: { + must: matches, + }, + }; + + Logger.debug(`${moduleName}.queryDocuments queryDocuments executing query: ${query}`, traceId); + + const body = await performDslQuery( + client, + indexFromResourceInfo(resourceInfo), + query, + paginationParameters.limit as number, + paginationParameters.offset as number, + ); + + recordCount = body.hits.total.value; + + if (recordCount > 0) { + // eslint-disable-next-line no-underscore-dangle + documents = body.hits.hits.map((datarow) => JSON.parse(datarow._source.info)); + } + + if (isDebugEnabled()) { + const idsForLogging: string[] = documents.map((document) => document.id); + Logger.debug(`${moduleName}.queryDocuments Ids of documents returned: ${JSON.stringify(idsForLogging)}`, traceId); + } + } catch (e) { + return handleElasticSearchError(e, `${moduleName}.queryDocuments`, traceId); + } + + return { response: 'QUERY_SUCCESS', documents, totalCount: recordCount }; +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/UpdateElasticsearch.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/UpdateElasticsearch.ts new file mode 100644 index 00000000..b8d7da24 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/src/repository/UpdateElasticsearch.ts @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { Client } from '@elastic/elasticsearch'; +import { + DeleteRequest, + DeleteResult, + DocumentUuid, + UpdateRequest, + UpdateResult, + UpsertRequest, + UpsertResult, +} from '@edfi/meadowlark-core'; +import { Logger } from '@edfi/meadowlark-utilities'; +import { indexFromResourceInfo } from './QueryElasticsearch'; +import { handleElasticSearchError } from './ElasticSearchException'; + +const moduleName = 'elasticsearch.repository.UpdateElasticsearch'; + +/** + * Parameters for an ElasticSearch request + */ +type ElasticsearchRequest = { index: string; id: DocumentUuid }; + +/** + * Listener for afterDeleteDocumentById events + */ +export async function afterDeleteDocumentById(request: DeleteRequest, result: DeleteResult, client: Client) { + Logger.info(`${moduleName}.afterDeleteDocumentById`, request.traceId); + if (result.response !== 'DELETE_SUCCESS') return; + + const elasticsearchRequest: ElasticsearchRequest = { + id: request.documentUuid, + index: indexFromResourceInfo(request.resourceInfo), + }; + + try { + Logger.debug( + `${moduleName}.afterDeleteDocumentById removing ${elasticsearchRequest.id} from index ${elasticsearchRequest.index}`, + request.traceId, + ); + await client.delete({ ...elasticsearchRequest, refresh: true }); + } catch (err) { + await handleElasticSearchError(err, `${moduleName}.afterDeleteDocumentById`, request.traceId, elasticsearchRequest.id); + } +} + +/** + * Shared elasticsearch upsert logic + */ +async function upsertToElasticsearch(request: UpsertRequest, documentUuid: DocumentUuid, client: Client) { + const elasticsearchRequest: ElasticsearchRequest = { + id: documentUuid, + index: indexFromResourceInfo(request.resourceInfo), + }; + + Logger.debug( + `${moduleName}.upsertToElasticsearch inserting id ${elasticsearchRequest.id} into index ${elasticsearchRequest.index}`, + request.traceId, + ); + + try { + await client.index({ + ...elasticsearchRequest, + body: { + id: elasticsearchRequest.id, + info: JSON.stringify({ id: elasticsearchRequest.id, ...request.edfiDoc }), + ...request.edfiDoc, + createdBy: request.security.clientId, + meadowlarkId: request.meadowlarkId, + documentUuid, + }, + refresh: true, + }); + } catch (err) { + await handleElasticSearchError(err, `${moduleName}.upsertToElasticsearch`, request.traceId, elasticsearchRequest.id); + } +} + +/** + * Listener for afterUpsertDocument events + */ +export async function afterUpsertDocument(request: UpsertRequest, result: UpsertResult, client: Client) { + Logger.info(`${moduleName}.afterUpsertDocument`, request.traceId); + if (result.response !== 'UPDATE_SUCCESS' && result.response !== 'INSERT_SUCCESS') return; + const documentUuid: DocumentUuid = + result.response === 'UPDATE_SUCCESS' ? result.existingDocumentUuid : result.newDocumentUuid; + await upsertToElasticsearch(request, documentUuid, client); +} + +/** + * Listener for afterUpdateDocumentById events + */ +export async function afterUpdateDocumentById(request: UpdateRequest, result: UpdateResult, client: Client) { + Logger.info(`${moduleName}.afterUpdateDocumentById`, request.traceId); + if (result.response !== 'UPDATE_SUCCESS') return; + await upsertToElasticsearch( + { + meadowlarkId: request.meadowlarkId, + resourceInfo: request.resourceInfo, + documentInfo: request.documentInfo, + edfiDoc: request.edfiDoc, + validateDocumentReferencesExist: request.validateDocumentReferencesExist, + security: request.security, + traceId: request.traceId, + }, + request.documentUuid, + client, + ); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/.eslintrc b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/.eslintrc new file mode 100644 index 00000000..436aa9f4 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/.eslintrc @@ -0,0 +1,14 @@ +{ + "env": { + "jest": true + }, + "rules": { + "no-unused-expressions": "off", + "import/no-extraneous-dependencies": [ + "error", + { + "devDependencies": true + } + ] + } +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/QueryOpensearch.test.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/QueryOpensearch.test.ts new file mode 100644 index 00000000..57254c44 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/QueryOpensearch.test.ts @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { Client } from '@elastic/elasticsearch'; +import Mock from '@elastic/elasticsearch-mock'; +import { + PaginationParameters, + QueryRequest, + QueryResult, + AuthorizationStrategy, + TraceId, + ResourceInfo, +} from '@edfi/meadowlark-core'; +import { queryDocuments, indexFromResourceInfo } from '../src/repository/QueryElasticsearch'; + +const mock = new Mock(); + +const resourceInfo: ResourceInfo = { + projectName: 'ed-fi', + resourceName: 'student', + isDescriptor: false, + resourceVersion: '3.3.1-b', + allowIdentityUpdates: false, +}; + +describe('when querying for students', () => { + const setupMockRequestHappyPath = (uniqueIds: string[], matches: any[], size?: number, from?: number): Client => { + const datarows = uniqueIds.map((x) => ({ _source: { studentUniqueId: x, info: `{ "studentUniqueId": "${x}" }` } })); + + const body: any = { + query: { + bool: { + must: matches, + }, + }, + sort: [{ _doc: { order: 'asc' } }], + }; + + if (size) body.size = size; + + if (from) body.from = from; + + mock.add( + { + method: 'POST', + path: `/${indexFromResourceInfo(resourceInfo)}/_search`, + body, + }, + () => ({ + hits: { + total: { value: datarows.length, relation: 'eq' }, + hits: datarows, + }, + }), + ); + + const client = new Client({ + node: 'http://localhost:9200', + Connection: mock.getConnection(), + }); + return client; + }; + + const setupQueryRequest = ( + authorizationStrategy: AuthorizationStrategy, + queryParameters: any, + paginationParameters: PaginationParameters, + clientId = 'hi', + ): QueryRequest => ({ + resourceInfo, + queryParameters, + paginationParameters, + security: { authorizationStrategy, clientId }, + traceId: 'tracer' as TraceId, + }); + + describe('given there are no students', () => { + it('should return an empty array', async () => { + // Arrange + const client = setupMockRequestHappyPath([], []); + const request = setupQueryRequest({ type: 'FULL_ACCESS' }, {}, {}); + + // Act + const result = await queryDocuments(request, client); + + // Assert + expect(result.documents.length).toEqual(0); + }); + + afterAll(() => { + mock.clearAll(); + }); + }); + + describe('given there are students', () => { + describe('given full access authorization', () => { + describe('given no query or pagination', () => { + const authorizationStrategy: AuthorizationStrategy = { type: 'FULL_ACCESS' }; + let queryResult: QueryResult; + const studentUniqueIdOne = 'one'; + const studentUniqueIdTwo = 'two'; + + beforeAll(async () => { + const client = setupMockRequestHappyPath([studentUniqueIdOne, studentUniqueIdTwo], []); + const request = setupQueryRequest(authorizationStrategy, {}, {}); + + queryResult = await queryDocuments(request, client); + }); + + it('should return two students', async () => { + expect(queryResult.documents.length).toBe(2); + }); + + it('should return the first student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdOne)).toBeGreaterThan(-1); + }); + + it('should return the second student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdTwo)).toBeGreaterThan(-1); + }); + + afterAll(() => { + mock.clearAll(); + }); + }); + + describe('given two query terms', () => { + const authorizationStrategy: AuthorizationStrategy = { type: 'FULL_ACCESS' }; + let queryResult: QueryResult; + let spyOnRequest: jest.SpyInstance; + const studentUniqueIdOne = 'one'; + const studentUniqueIdTwo = 'two'; + const birthCity = 'a'; + const birthDate = '2022-07-28'; + const matches = [ + { + match: { birthCity }, + }, + { + match: { birthDate }, + }, + ]; + const expectedQuery = { + from: undefined, + query: { + bool: { + must: matches, + }, + }, + size: undefined, + sort: [{ _doc: { order: 'asc' } }], + }; + + beforeAll(async () => { + const client = setupMockRequestHappyPath([studentUniqueIdOne, studentUniqueIdTwo], matches); + const request = setupQueryRequest(authorizationStrategy, { birthCity, birthDate }, {}); + + spyOnRequest = jest.spyOn(client.transport, 'request'); + + queryResult = await queryDocuments(request, client); + }); + + it('should return two students', async () => { + expect(queryResult.documents.length).toBe(2); + }); + + it('should return the first student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdOne)).toBeGreaterThan(-1); + }); + + it('should return the second student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdTwo)).toBeGreaterThan(-1); + }); + + it('should have used the correct SQL query', async () => { + expect(spyOnRequest.mock.calls.length).toBe(1); + expect(spyOnRequest.mock.calls[0].length).toBe(1); + const { body } = spyOnRequest.mock.calls[0][0]; + expect(body).toEqual(expectedQuery); + }); + + afterAll(() => { + mock.clearAll(); + }); + }); + + describe('given page limits', () => { + const authorizationStrategy: AuthorizationStrategy = { type: 'FULL_ACCESS' }; + let queryResult: QueryResult; + let spyOnRequest: jest.SpyInstance; + const studentUniqueIdOne = 'one'; + const studentUniqueIdTwo = 'two'; + const limit = 1; + const offset = 2; + const expectedQuery = { + from: 2, + query: { bool: { must: [] } }, + size: 1, + sort: [{ _doc: { order: 'asc' } }], + }; + + beforeAll(async () => { + const client = setupMockRequestHappyPath([studentUniqueIdOne, studentUniqueIdTwo], [], limit, offset); + const request = setupQueryRequest(authorizationStrategy, {}, { limit, offset }); + + spyOnRequest = jest.spyOn(client.transport, 'request'); + + queryResult = await queryDocuments(request, client); + }); + + it('should return two students', async () => { + expect(queryResult.documents.length).toBe(2); + }); + + it('should return the first student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdOne)).toBeGreaterThan(-1); + }); + + it('should return the second student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdTwo)).toBeGreaterThan(-1); + }); + + it('should have used the correct SQL query', async () => { + expect(spyOnRequest.mock.calls.length).toBe(1); + expect(spyOnRequest.mock.calls[0].length).toBe(1); + const { body } = spyOnRequest.mock.calls[0][0]; + expect(body).toEqual(expectedQuery); + }); + + afterAll(() => { + mock.clearAll(); + }); + }); + + describe('given both query terms and page limits', () => { + const authorizationStrategy: AuthorizationStrategy = { type: 'FULL_ACCESS' }; + let queryResult: QueryResult; + const studentUniqueIdOne = 'one'; + const studentUniqueIdTwo = 'two'; + const birthCity = 'a'; + const birthDate = '2022-07-28'; + const matches = [ + { + match: { birthCity }, + }, + { + match: { birthDate }, + }, + ]; + let spyOnRequest: jest.SpyInstance; + const limit = 1; + const offset = 1; + const expectedQuery = { + from: offset, + query: { bool: { must: matches } }, + size: limit, + sort: [{ _doc: { order: 'asc' } }], + }; + + beforeAll(async () => { + const client = setupMockRequestHappyPath([studentUniqueIdTwo], matches, limit, offset); + const request = setupQueryRequest(authorizationStrategy, { birthCity, birthDate }, { limit, offset }); + + spyOnRequest = jest.spyOn(client.transport, 'request'); + + queryResult = await queryDocuments(request, client); + }); + + it('should return two students', async () => { + expect(queryResult.documents.length).toBe(1); + }); + + it('should return the first student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdOne)).toBe(-1); + }); + + it('should return the second student', async () => { + expect(queryResult.documents.findIndex((x: any) => x.studentUniqueId === studentUniqueIdTwo)).toBeGreaterThan(-1); + }); + + it('should have used the correct SQL query', async () => { + expect(spyOnRequest.mock.calls.length).toBe(1); + expect(spyOnRequest.mock.calls[0].length).toBe(1); + const { body } = spyOnRequest.mock.calls[0][0]; + expect(body).toEqual(expectedQuery); + }); + + afterAll(() => { + mock.clearAll(); + }); + }); + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/config/integration/jest.config.js b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/config/integration/jest.config.js new file mode 100644 index 00000000..533125e3 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/config/integration/jest.config.js @@ -0,0 +1,21 @@ +const rootDir = '../../../../../'; +// eslint-disable-next-line import/no-extraneous-dependencies +const defaultConfig = require(`${rootDir}/tests/config/jest.config`); + +module.exports = { + displayName: 'Integration Tests: ElasticSearch', + globalSetup: '/backends/meadowlark-elasticsearch-backend/test/setup/Setup.ts', + globalTeardown: '/backends/meadowlark-elasticsearch-backend/test/setup/Teardown.ts', + ...defaultConfig, + testMatch: ['**/meadowlark-elasticsearch-backend/test/integration/**/*.(spec|test).[jt]s?(x)'], + coverageThreshold: { + global: { + branches: 52, + functions: 58, + lines: 60, + statements: 60, + }, + }, + rootDir, + workerIdleMemoryLimit: '200MB', +}; diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/QueryElasticsearch.test.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/QueryElasticsearch.test.ts new file mode 100644 index 00000000..416ddeef --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/QueryElasticsearch.test.ts @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { + ResourceInfo, + UpsertRequest, + NoDocumentInfo, + UpsertResult, + PaginationParameters, + QueryRequest, + Security, + AuthorizationStrategy, + DeleteRequest, + DeleteResult, +} from '@edfi/meadowlark-core'; +import { DocumentUuid, MeadowlarkId, TraceId } from '@edfi/meadowlark-core/src/model/BrandedTypes'; +import { generateDocumentUuid } from '@edfi/meadowlark-core/src/model/DocumentIdentity'; +import { Client } from '@elastic/elasticsearch'; +import { queryDocuments } from '../../src/repository/QueryElasticsearch'; +import { afterDeleteDocumentById, afterUpsertDocument } from '../../src/repository/UpdateElasticsearch'; +import { getNewTestClient } from '../setup/ElasticSearchSetupEnvironment'; + +jest.setTimeout(120_000); + +const resourceInfo: ResourceInfo = { + projectName: 'ed-fi', + resourceName: 'student', + isDescriptor: false, + resourceVersion: '3.3.1-b', + allowIdentityUpdates: false, +}; + +const student1 = { + studentUniqueId: '123fer58', + firstName: 'First', + lastSurname: 'Student', + birthDate: '2001-01-01', + birthCountryDescriptor: 'uri://ed-fi.org/CountryDescriptor#US', +}; + +const student1DocumentUuid: DocumentUuid = generateDocumentUuid(); +const student1MeadowlarkId: MeadowlarkId = 'student1-123' as MeadowlarkId; + +const student2 = { + studentUniqueId: '123fer58', + firstName: 'Second', + lastSurname: 'Student', + birthDate: '2001-01-01', + birthCountryDescriptor: 'uri://ed-fi.org/CountryDescriptor#US', +}; + +const student2DocumentUuid: DocumentUuid = generateDocumentUuid(); +const student2MeadowlarkId: MeadowlarkId = 'student2-456' as MeadowlarkId; + +const security: Security = { + authorizationStrategy: { type: 'FULL_ACCESS' } as AuthorizationStrategy, + clientId: '1', +}; + +const setupUpsertRequest = ( + meadowlarkId: MeadowlarkId, + edfiDoc = {}, + newResourceInfo = resourceInfo, + documentInfo = NoDocumentInfo, +): UpsertRequest => ({ + meadowlarkId, + resourceInfo: newResourceInfo, + documentInfo, + edfiDoc, + validateDocumentReferencesExist: false, + security, + traceId: 'traceId' as TraceId, +}); + +const setupQueryRequest = ( + queryParameters: any, + paginationParameters: PaginationParameters, + newResourceInfo = resourceInfo, +): QueryRequest => ({ + resourceInfo: newResourceInfo, + queryParameters, + paginationParameters, + security, + traceId: 'tracer' as TraceId, +}); + +describe('When querying for documents', () => { + let client: Client; + + beforeAll(async () => { + client = await getNewTestClient(); + + await afterUpsertDocument( + setupUpsertRequest(student1MeadowlarkId, student1), + { + response: 'INSERT_SUCCESS', + newDocumentUuid: student1DocumentUuid, + } as UpsertResult, + client, + ); + + await afterUpsertDocument( + setupUpsertRequest(student2MeadowlarkId, student2), + { + response: 'INSERT_SUCCESS', + newDocumentUuid: student2DocumentUuid, + } as UpsertResult, + client, + ); + }); + + afterAll(async () => { + client = await getNewTestClient(); + await afterDeleteDocumentById( + { documentUuid: student1DocumentUuid, resourceInfo } as DeleteRequest, + { response: 'DELETE_SUCCESS' } as DeleteResult, + client, + ); + + await afterDeleteDocumentById( + { documentUuid: student2DocumentUuid, resourceInfo } as DeleteRequest, + { response: 'DELETE_SUCCESS' } as DeleteResult, + client, + ); + }); + + describe('when querying with parameters', () => { + describe('when querying with wrong resource info', () => { + it('should return invalid query', async () => { + const invalidResourceInfo = { ...resourceInfo }; + invalidResourceInfo.projectName = 'wrong-project'; + const result = await queryDocuments(setupQueryRequest({}, {}, invalidResourceInfo), client); + + expect(result.response).toEqual('QUERY_FAILURE_INVALID_QUERY'); + expect(result.documents).toHaveLength(0); + expect(result.totalCount).toBeUndefined(); + }); + }); + + describe('when querying with wrong property', () => { + it('should return error message', async () => { + const result = await queryDocuments(setupQueryRequest({ lastSure: 'Last' }, {}), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(0); + expect(result.documents).toHaveLength(0); + }); + }); + + describe('when querying with non existent data', () => { + it('should return empty results', async () => { + const result = await queryDocuments(setupQueryRequest({ firstName: 'Last' }, {}), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(0); + expect(result.documents).toHaveLength(0); + }); + }); + + describe('when querying without parameters', () => { + it('should return all values', async () => { + const result = await queryDocuments(setupQueryRequest({}, {}), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(2); + expect(result.documents[0]).toEqual({ ...student1, id: student1DocumentUuid }); + expect(result.documents[1]).toEqual({ ...student2, id: student2DocumentUuid }); + }); + }); + + describe('when querying with valid parameters', () => { + it('should return value', async () => { + const result = await queryDocuments(setupQueryRequest({ firstName: student1.firstName }, {}), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(1); + expect(result.documents[0]).toEqual({ ...student1, id: student1DocumentUuid }); + }); + }); + + describe('when querying with limit', () => { + it('should return value', async () => { + const result = await queryDocuments(setupQueryRequest({}, { limit: 1 }), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(2); + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toEqual({ ...student1, id: student1DocumentUuid }); + }); + }); + + describe('when querying with limit and offset', () => { + it('should return value', async () => { + const result = await queryDocuments(setupQueryRequest({}, { limit: 1, offset: 1 }), client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(2); + expect(result.documents).toHaveLength(1); + expect(result.documents[0]).toEqual({ ...student2, id: student2DocumentUuid }); + }); + }); + + describe('when querying with parameters and offset', () => { + it('should return value', async () => { + const result = await queryDocuments( + setupQueryRequest({ firstName: student1.firstName }, { limit: 2, offset: 1 }), + client, + ); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(1); + expect(result.documents).toHaveLength(0); + }); + }); + + describe('when querying with extra characters', () => { + it("shouldn't return values", async () => { + const result = await queryDocuments( + setupQueryRequest({ firstName: "student1.firstName'%20or%20studentUniqueId%20is%20not%20null%20%23" }, {}), + client, + ); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(0); + expect(result.documents).toHaveLength(0); + }); + }); + }); + + describe('when querying with ownership', () => { + const ownershipSecurity: Security = { + authorizationStrategy: { type: 'OWNERSHIP_BASED' } as AuthorizationStrategy, + clientId: '2', + }; + + describe('when querying for descriptor', () => { + const descriptorResourceInfo: ResourceInfo = { + projectName: 'ed-fi', + resourceName: 'countryDescriptor', + isDescriptor: true, + resourceVersion: '3.3.1-b', + allowIdentityUpdates: false, + }; + + const queryRequest: QueryRequest = { + resourceInfo: descriptorResourceInfo, + queryParameters: {}, + paginationParameters: {}, + security: ownershipSecurity, + traceId: 'tracer' as TraceId, + }; + + const descriptorDocumentUuid: DocumentUuid = generateDocumentUuid(); + const descriptorMeadowlarkId: MeadowlarkId = 'desc-123' as MeadowlarkId; + + const descriptorUpsertRequest: UpsertRequest = { + meadowlarkId: descriptorMeadowlarkId, + resourceInfo: descriptorResourceInfo, + documentInfo: NoDocumentInfo, + edfiDoc: {}, + validateDocumentReferencesExist: false, + security: ownershipSecurity, + traceId: 'traceId' as TraceId, + }; + + beforeAll(async () => { + await afterUpsertDocument( + descriptorUpsertRequest, + { + response: 'INSERT_SUCCESS', + newDocumentUuid: descriptorDocumentUuid, + } as UpsertResult, + client, + ); + }); + + afterAll(async () => { + await afterDeleteDocumentById( + { documentUuid: descriptorDocumentUuid, resourceInfo: descriptorResourceInfo } as DeleteRequest, + { response: 'DELETE_SUCCESS' } as DeleteResult, + client, + ); + }); + + it('should ignore security', async () => { + const result = await queryDocuments(queryRequest, client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(1); + expect(result.documents[0]).toEqual({ id: descriptorDocumentUuid }); + }); + }); + + describe('when querying with ownership', () => { + const queryRequest: QueryRequest = { + resourceInfo, + queryParameters: {}, + paginationParameters: {}, + security: ownershipSecurity, + traceId: 'tracer' as TraceId, + }; + + it('should return empty array for different client', async () => { + const result = await queryDocuments(queryRequest, client); + + expect(result.response).toEqual('QUERY_SUCCESS'); + expect(result.totalCount).toEqual(0); + }); + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/UpdateElasticsearch.test.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/UpdateElasticsearch.test.ts new file mode 100644 index 00000000..2b4434b8 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/integration/UpdateElasticsearch.test.ts @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { + DeleteRequest, + DeleteResult, + DocumentUuid, + MeadowlarkId, + newSecurity, + NoDocumentInfo, + PaginationParameters, + QueryRequest, + ResourceInfo, + TraceId, + UpdateRequest, + UpdateResult, + UpsertRequest, + UpsertResult, +} from '@edfi/meadowlark-core'; +import { Client } from '@elastic/elasticsearch'; +import { queryDocuments } from '../../src/repository/QueryElasticsearch'; +import { + afterDeleteDocumentById, + afterUpdateDocumentById, + afterUpsertDocument, +} from '../../src/repository/UpdateElasticsearch'; +import { getNewTestClient } from '../setup/ElasticSearchSetupEnvironment'; + +jest.setTimeout(120_000); + +const resourceInfo: ResourceInfo = { + projectName: 'ed-fi', + resourceName: 'student', + isDescriptor: false, + resourceVersion: '3.3.1-b', + allowIdentityUpdates: false, +}; + +const resourceIndex: string = 'ed-fi$3-3-1-b$student'; + +const meadowlarkId = '1234a-5678b' as MeadowlarkId; +const documentUuid: DocumentUuid = 'e6ce70aa-8216-46e9-b6a1-a3be70f72f36' as DocumentUuid; + +const newUpsertRequest: UpsertRequest = { + meadowlarkId, + resourceInfo, + documentInfo: NoDocumentInfo, + edfiDoc: { upsert: true }, + validateDocumentReferencesExist: false, + security: { ...newSecurity() }, + traceId: 'traceId' as TraceId, +}; + +const newUpdateRequest: UpdateRequest = { + meadowlarkId, + documentUuid, + resourceInfo, + documentInfo: NoDocumentInfo, + edfiDoc: { update: true }, + validateDocumentReferencesExist: false, + security: { ...newSecurity() }, + traceId: 'traceId' as TraceId, +}; + +const setupQueryRequest = (queryParameters: any, paginationParameters: PaginationParameters): QueryRequest => ({ + resourceInfo, + queryParameters, + paginationParameters, + security: { ...newSecurity() }, + traceId: 'tracer' as TraceId, +}); + +describe('given the upsert of a new document', () => { + let client: Client; + + beforeAll(async () => { + client = await getNewTestClient(); + }); + + describe('when insert was successful', () => { + beforeEach(async () => { + await afterUpsertDocument( + newUpsertRequest, + { + response: 'INSERT_SUCCESS', + newDocumentUuid: documentUuid, + } as UpsertResult, + client, + ); + }); + + afterEach(async () => { + await client.indices.delete({ index: resourceIndex }); + }); + + it('should be created', async () => { + const response = await queryDocuments(setupQueryRequest({}, {}), client); + expect(response.documents).toHaveLength(1); + expect((response.documents[0] as any).upsert).toBe(true); + }); + }); + + describe('when update was successful', () => { + beforeEach(async () => { + newUpsertRequest.traceId = 'tracer2' as TraceId; + await afterUpsertDocument( + newUpsertRequest, + { + response: 'UPDATE_SUCCESS', + } as UpsertResult, + client, + ); + }); + + afterEach(async () => { + await client.indices.delete({ index: resourceIndex }); + }); + + it('should be updated', async () => { + const response = await queryDocuments(setupQueryRequest({}, {}), client); + expect(response.documents).toHaveLength(1); + expect((response.documents[0] as any).upsert).toBe(true); + }); + }); + + describe('when response was not successful', () => { + it.each([ + 'INSERT_FAILURE_REFERENCE', + 'INSERT_FAILURE_CONFLICT', + 'UPDATE_FAILURE_REFERENCE', + 'UPSERT_FAILURE_AUTHORIZATION', + 'UNKNOWN_FAILURE', + ])('should not insert when result is %s', async (response) => { + await afterUpsertDocument( + newUpsertRequest, + { + response, + } as UpsertResult, + client, + ); + + const result = await queryDocuments(setupQueryRequest({}, {}), client); + + expect(result.documents).toHaveLength(0); + }); + }); + + describe('when updating by meadowlarkId is successful', () => { + beforeEach(async () => { + await afterUpdateDocumentById( + newUpdateRequest, + { + response: 'UPDATE_SUCCESS', + } as UpdateResult, + client, + ); + }); + + afterEach(async () => { + await client.indices.delete({ index: resourceIndex }); + }); + + it('should be updated', async () => { + const response = await queryDocuments(setupQueryRequest({}, {}), client); + expect(response.documents).toHaveLength(1); + expect((response.documents[0] as any).update).toBe(true); + }); + }); + + describe('when updating should not be saved', () => { + it.each(['UPDATE_FAILURE_REFERENCE', 'UPDATE_FAILURE_NOT_EXISTS', 'UPDATE_FAILURE_AUTHORIZATION', 'UNKNOWN_FAILURE'])( + 'should not update when result is %s', + async (response) => { + await afterUpdateDocumentById( + newUpdateRequest, + { + response, + } as UpdateResult, + client, + ); + + const result = await queryDocuments(setupQueryRequest({}, {}), client); + + expect(result.documents).toHaveLength(0); + }, + ); + }); + + describe('when deleting by meadowlarkId', () => { + beforeEach(async () => { + await afterUpdateDocumentById( + newUpdateRequest, + { + response: 'UPDATE_SUCCESS', + } as UpdateResult, + client, + ); + }); + + afterEach(async () => { + await client.indices.delete({ index: resourceIndex }); + }); + + it('should be able to delete document', async () => { + await afterDeleteDocumentById( + { documentUuid, resourceInfo } as DeleteRequest, + { response: 'DELETE_SUCCESS' } as DeleteResult, + client, + ); + + const response = await queryDocuments(setupQueryRequest({}, {}), client); + + expect(response.documents).toHaveLength(0); + }); + }); +}); diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchContainer.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchContainer.ts new file mode 100644 index 00000000..4eb72e35 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchContainer.ts @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { GenericContainer, StartedTestContainer } from 'testcontainers'; + +let startedContainer: StartedTestContainer; + +export async function setup() { + const elasticSearchPort = parseInt(process.env.ELASTICSEARCH_PORT ?? '9200', 10); + startedContainer = await new GenericContainer( + 'docker.elastic.co/elasticsearch/elasticsearch:8.8.0@sha256:9aaa38551b4d9e655c54d9dc6a1dad24ee568c41952dc8cf1d4808513cfb5f65', + ) + .withName('elasticsearch-node-test') + .withExposedPorts({ + container: elasticSearchPort, + host: elasticSearchPort, + }) + .withEnvironment({ + 'node.name': 'es01', + 'cluster.name': 'elasticsearch-node1', + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + }) + .withStartupTimeout(120_000) + .start(); + + process.env.ELASTICSEARCH_ENDPOINT = `http://localhost:${elasticSearchPort}`; + process.env.ELASTICSEARCH_REQUEST_TIMEOUT = '10000'; +} + +export async function stop(): Promise { + await startedContainer.stop(); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchSetupEnvironment.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchSetupEnvironment.ts new file mode 100644 index 00000000..c9a57c2e --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/ElasticSearchSetupEnvironment.ts @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +import { Logger } from '@edfi//meadowlark-utilities'; +import { Client } from '@elastic/elasticsearch'; +import * as ElasticSearchContainer from './ElasticSearchContainer'; +import { getElasticSearchClient } from '../../src/repository/Db'; + +const moduleName = 'elasticsearch.repository.Db'; + +export async function setupElasticSearch() { + Logger.info('-- Setup ElasticSearch environment --', null); + await ElasticSearchContainer.setup(); +} + +export async function teardownElasticSearch() { + Logger.info('-- Tearing down ElasticSearch environment --', null); + await ElasticSearchContainer.stop(); +} + +/** + * Create and return an ElasticSearch connection object + */ +export async function getNewTestClient(): Promise { + Logger.debug(`${moduleName}.getNewClient creating local client`, null); + const node = process.env.ELASTICSEARCH_ENDPOINT; + const auth = { username: '', password: '' }; + const requestTimeout = 10000; + return getElasticSearchClient(node, auth, requestTimeout); +} diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Setup.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Setup.ts new file mode 100644 index 00000000..cae5c793 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Setup.ts @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. + +const elasticSearchEnvironmentSetup = require('./ElasticSearchSetupEnvironment'); + +async function setupElasticSearchIntegrationTestEnvironment() { + try { + // Setup elasticSearch environment for integration tests. + await elasticSearchEnvironmentSetup.setupElasticSearch(); + } catch (error) { + throw new Error(`Error setting up integration test environment: ${error}`); + } +} + +module.exports = async () => setupElasticSearchIntegrationTestEnvironment(); diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Teardown.ts b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Teardown.ts new file mode 100644 index 00000000..fcacf976 --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/test/setup/Teardown.ts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// Licensed to the Ed-Fi Alliance under one or more agreements. +// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. +// See the LICENSE and NOTICES files in the project root for more information. +const elasticSearchEnvironmentTeardown = require('./ElasticSearchSetupEnvironment'); + +module.exports = async () => { + try { + // Setup elasticSearch environment for integration tests. + await elasticSearchEnvironmentTeardown.teardownElasticSearch(); + } catch (error) { + throw new Error(`Error Teardown: ${error}`); + } +}; diff --git a/Meadowlark-js/backends/meadowlark-elasticsearch-backend/tsconfig.json b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/tsconfig.json new file mode 100644 index 00000000..77fb804f --- /dev/null +++ b/Meadowlark-js/backends/meadowlark-elasticsearch-backend/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "./src", + "outDir": "./dist" + }, + "include": [ + "./src" + ] +} diff --git a/Meadowlark-js/package-lock.json b/Meadowlark-js/package-lock.json index 3c7fc3a3..0269b4b8 100644 --- a/Meadowlark-js/package-lock.json +++ b/Meadowlark-js/package-lock.json @@ -44,6 +44,25 @@ "typescript": "4.8.4" } }, + "backends/meadowlark-elasticsearch-backend": { + "name": "@edfi/meadowlark-elasticsearch-backend", + "version": "0.3.0-pre-36", + "license": "Apache-2.0", + "dependencies": { + "@edfi/meadowlark-core": "^v0.3.0-pre-35", + "@edfi/meadowlark-utilities": "^v0.3.0-pre-35", + "@edfi/metaed-core": "^4.0.1-dev.13", + "@elastic/elasticsearch": "^8.8.0", + "@elastic/transport": "^8.3.2" + }, + "devDependencies": { + "@elastic/elasticsearch-mock": "^2.0.0", + "copyfiles": "^2.4.1", + "dotenv": "^16.0.3", + "rimraf": "^3.0.2", + "testcontainers": "^9.8.0" + } + }, "backends/meadowlark-mongodb-backend": { "name": "@edfi/meadowlark-mongodb-backend", "version": "0.3.0-pre-36", @@ -1753,6 +1772,10 @@ "resolved": "tests/e2e", "link": true }, + "node_modules/@edfi/meadowlark-elasticsearch-backend": { + "resolved": "backends/meadowlark-elasticsearch-backend", + "link": true + }, "node_modules/@edfi/meadowlark-fastify": { "resolved": "services/meadowlark-fastify", "link": true @@ -1862,6 +1885,72 @@ "url": "https://opencollective.com/ramda" } }, + "node_modules/@elastic/elasticsearch": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.8.1.tgz", + "integrity": "sha512-ibArPKHEmak3jao7xts2gROATiwPQo9aOrWWdix5mJcX1gnjm/UeJBVO901ROmaxFVPKxVnjC9Op3gJYkqagjg==", + "dependencies": { + "@elastic/transport": "^8.3.2", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@elastic/elasticsearch-mock": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch-mock/-/elasticsearch-mock-2.0.0.tgz", + "integrity": "sha512-VACQF7GStt8DetY91aJhXCYog6zXM0Vyb62k592EEt3aB4plrOLot+JvlLMC4URjh2jt9qYfER9hn4AI+ULTSw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "find-my-way": "^5.2.0", + "into-stream": "^6.0.0" + } + }, + "node_modules/@elastic/elasticsearch-mock/node_modules/find-my-way": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-5.6.0.tgz", + "integrity": "sha512-pFTzbl2u+iSrvVOGtfKenvDmNIhNtEcwbzvRMfx3TGO69fbO5udgTKtAZAaUfIUrHQWLkkWvhNafNz179kaCIw==", + "dev": true, + "dependencies": { + "fast-decode-uri-component": "^1.0.1", + "fast-deep-equal": "^3.1.3", + "safe-regex2": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@elastic/transport": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.3.2.tgz", + "integrity": "sha512-ZiBYRVPj6pwYW99fueyNU4notDf7ZPs7Ix+4T1btIJsKJmeaORIItIfs+0O7KV4vV+DcvyMhkY1FXQx7kQOODw==", + "dependencies": { + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^5.22.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@elastic/transport/node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@elastic/transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, "node_modules/@eslint/eslintrc": { "version": "1.4.1", "dev": true, @@ -7178,6 +7267,17 @@ "semver": "^7.0.0" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/byline": { "version": "5.0.0", "dev": true, @@ -15812,6 +15912,14 @@ "node": ">=8" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", @@ -16554,6 +16662,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici": { + "version": "5.22.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.1.tgz", + "integrity": "sha512-Ji2IJhFXZY0x/0tVBXeQwgPlLWw13GVzpsWPQ3rV50IFMMof2I55PZZxtm4P6iNq+L5znYN9nSTAq0ZyE6lSJw==", + "dependencies": { + "busboy": "^1.6.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/unique-filename": { "version": "2.0.1", "dev": true, diff --git a/Meadowlark-js/package.json b/Meadowlark-js/package.json index d94a7403..87018cab 100644 --- a/Meadowlark-js/package.json +++ b/Meadowlark-js/package.json @@ -48,9 +48,10 @@ "test:lint:eslint": "eslint --max-warnings 0 --ext .js,.ts .", "test:lint:ts": "tsc -p . --noEmit", "test:unit": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects tests/config/unit", - "test:integration": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration backends/meadowlark-opensearch-backend/test/config/integration backends/meadowlark-postgresql-backend/test/config/integration --runInBand", + "test:integration": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration backends/meadowlark-opensearch-backend/test/config/integration backends/meadowlark-elasticsearch-backend/test/config/integration backends/meadowlark-postgresql-backend/test/config/integration --runInBand", "test:integration:mongodb": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-mongodb-backend/test/config/integration --runInBand", "test:integration:opensearch": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-opensearch-backend/test/config/integration --runInBand", + "test:integration:elasticsearch": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-elasticsearch-backend/test/config/integration --runInBand", "test:integration:postgresql": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects backends/meadowlark-postgresql-backend/test/config/integration --runInBand", "test:unit:coverage": "rimraf coverage.unit/ && npm run test:unit", "test:unit:coverage:ci": "rimraf coverage.unit/ && npm run test:unit -- --maxWorkers=2 --ci", diff --git a/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts b/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts index ffc6600c..0343954d 100644 --- a/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts +++ b/Meadowlark-js/packages/meadowlark-utilities/src/Config.ts @@ -30,6 +30,10 @@ export type ConfigKeys = | 'OPENSEARCH_USERNAME' | 'OPENSEARCH_PASSWORD' | 'OPENSEARCH_REQUEST_TIMEOUT' + | 'ELASTICSEARCH_ENDPOINT' + | 'ELASTICSEARCH_USERNAME' + | 'ELASTICSEARCH_PASSWORD' + | 'ELASTICSEARCH_REQUEST_TIMEOUT' | 'LISTENER1_PLUGIN' | 'LISTENER2_PLUGIN' | 'QUERY_HANDLER_PLUGIN' @@ -129,6 +133,18 @@ export async function initializeConfig(provider: ConfigPlugin) { set('OPENSEARCH_REQUEST_TIMEOUT', await provider.getInt('OPENSEARCH_REQUEST_TIMEOUT', 30000)); } + // should only be required if enabled... + if ( + get('LISTENER1_PLUGIN') === '@edfi/meadowlark-elasticsearch-backend' || + get('LISTENER2_PLUGIN') === '@edfi/meadowlark-elasticsearch-backend' || + get('QUERY_HANDLER_PLUGIN') === '@edfi/meadowlark-elasticsearch-backend' + ) { + set('ELASTICSEARCH_ENDPOINT', await provider.getString('ELASTICSEARCH_ENDPOINT', ThrowIfNotFound)); + set('ELASTICSEARCH_USERNAME', await provider.getString('ELASTICSEARCH_USERNAME', 'x')); + set('ELASTICSEARCH_PASSWORD', await provider.getString('ELASTICSEARCH_PASSWORD', 'y')); + set('ELASTICSEARCH_REQUEST_TIMEOUT', await provider.getInt('ELASTICSEARCH_REQUEST_TIMEOUT', 30000)); + } + set('ALLOW_TYPE_COERCION', await provider.getBool('ALLOW_TYPE_COERCION', false)); set('ALLOW__EXT_PROPERTY', await provider.getBool('ALLOW__EXT_PROPERTY', false)); set('FASTIFY_NUM_THREADS', await provider.getInt('FASTIFY_NUM_THREADS', CpuCount)); diff --git a/Meadowlark-js/services/meadowlark-fastify/.env.example b/Meadowlark-js/services/meadowlark-fastify/.env.example index 87a461ff..75e1db2c 100644 --- a/Meadowlark-js/services/meadowlark-fastify/.env.example +++ b/Meadowlark-js/services/meadowlark-fastify/.env.example @@ -19,10 +19,15 @@ OPENSEARCH_ENDPOINT=http://localhost:9200 # Timeout in ms OPENSEARCH_REQUEST_TIMEOUT=10000 +# ElasticSearch endpoint and timeout. +# Required fields when QUERY_HANDLER_PLUGIN is set to use Elasticsearch. +ELASTICSEARCH_ENDPOINT=http://localhost:9200 +ELASTICSEARCH_REQUEST_TIMEOUT=10000 + #### Configurable backend plugin PoC - set to an npm package name DOCUMENT_STORE_PLUGIN=@edfi/meadowlark-mongodb-backend -QUERY_HANDLER_PLUGIN=@edfi/meadowlark-opensearch-backend +QUERY_HANDLER_PLUGIN=@edfi/meadowlark-opensearch-backend # or @edfi/meadowlark-elasticsearch-backend #LISTENER1_PLUGIN=@edfi/meadowlark-opensearch-backend diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ba258fd7..3dc98cf3 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -5,9 +5,8 @@ As of release 0.3.0, "Meadowlark" provides a Fastify-based API that implements (imperfectly) an Ed-Fi API compliant with Data Standard 3.3, and implements an OAuth2 API supporting the client-credentials flow. These services are backed by -MongoDB for primary data storage. The Ed-Fi API also OpenSearch to handle `GET` -all items and `GET` by querystring filters; in theory Elasticsearch should work -as well, though the development team has not tested it. +MongoDB for primary data storage. The Ed-Fi API also includes OpenSearch and +ElasticSearch to handle `GET` all items and `GET` by querystring filters. The MongoDB configuration needs to have a replica set to support atomic transactions. Technically the replica set can contain only a single node, though @@ -19,22 +18,22 @@ communicate through unencrypted channels. Indeed, the development team has not tested any alternative. However, only the API port should be open to outside traffic, except as needed for debugging. - ┌──────────────────────────────────────────────┐ - │ │ - │ ┌────────┐ │ - │ ┌──────►MongoDB ├───────┐ │ - │ │ └───┬────┘ │ │ - ┌───────────┐ │ ┌────────┐ │ │ │ │ - │API Clients├──────────┼─►Fastify ├────┤ ┌───▼────┐ ┌────▼───┐ │ - └───────────┘ │ └────────┘ │ │MongoDB │ │MongoDB │ │ - │ │ └────────┘ └────────┘ │ - │ │ │ - │ │ ┌───────────┐ │ - │ └──────►OpenSearch │ │ - │ └───────────┘ │ - │ │ - │ Network Boundary │ - └──────────────────────────────────────────────┘ + ┌────────────────────────────────────────────────────────┐ + │ │ + │ ┌────────┐ │ + │ ┌──────►MongoDB ├───────┐ │ + │ │ └───┬────┘ │ │ + ┌───────────┐ │ ┌────────┐ │ │ │ │ + │API Clients├──────┼─►Fastify ├────┤ ┌───▼────┐ ┌────▼───┐ │ + └───────────┘ │ └────────┘ │ │MongoDB │ │MongoDB │ │ + │ │ └────────┘ └────────┘ │ + │ │ │ + │ │ ┌───────────┐ ┌──────────────┐│ + │ └──────►OpenSearch │ --- │ElasticSearch ││ + │ └───────────┘ └──────────────┘│ + │ │ + │ Network Boundary │ + └────────────────────────────────────────────────────────┘ ## Environment Variables @@ -62,6 +61,8 @@ release 0.3.0. | OPENSEARCH_USERNAME | x | Username for connecting to OpenSearch | | OPENSEARCH_PASSWORD | y | Password for connecting to OpenSearch | | OPENSEARCH_REQUEST_TIMEOUT | 30000 | In milliseconds | +| **ELASTICSEARCH_ENDPOINT** | (none) | Only required when the ElasticSearch listener is configured for Fastify. | +| ELASTICSEARCH_REQUEST_TIMEOUT | 30000 | In milliseconds | | POSTGRES_HOST | localhost | Server/host name for PostgreSQL | | POSTGRES_PORT | 5432 | Port number for PostgreSQL | | POSTGRES_USER | (none) | Username for accessing PostgreSQL | @@ -90,9 +91,9 @@ To create a new key: | HTTP_PROTOCOL_AND_SERVER | http://localhost | The base URL for the site | | MEADOWLARK_STAGE | local | Used in the URL | | DISABLE_LOG_ANONYMIZATION | false | When true, request and response logs will contain complete payloads instead of anonymized payloads | -| LISTENER1_PLUGIN | (none) | Only one option at this time: "@edfi/meadowlark-opensearch-backend"; if not set, `GET` queries will fail | +| LISTENER1_PLUGIN | (none) | "@edfi/meadowlark-opensearch-backend" or "@edfi/meadowlark-elasticsearch-backend"; if not set, `GET` queries will fail | | LISTENER2_PLUGIN | (none) | No options at this time | -| QUERY_HANDLER_PLUGIN | (none) | Only one option at this time: "@edfi/meadowlark-opensearch-backend"; if not set, `GET` queries will fail | +| QUERY_HANDLER_PLUGIN | (none) | "@edfi/meadowlark-opensearch-backend" or "@edfi/meadowlark-elasticsearch-backend"; if not set, `GET` queries will fail | | DOCUMENT_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | Future alternative: "@edfi/meadowlark-postgresql-back | | AUTHORIZATION_STORE_PLUGIN | @edfi/meadowlark-mongodb-backend | No alternative at this time. | | **OAUTH_SERVER_ENDPOINT_FOR_OWN_TOKEN_REQUEST** | (none) | Ex: http://localhost:3000/local/oauth/token | diff --git a/docs/DOCKER-LOCAL-DEV.md b/docs/DOCKER-LOCAL-DEV.md index 63df9d6a..93381fef 100644 --- a/docs/DOCKER-LOCAL-DEV.md +++ b/docs/DOCKER-LOCAL-DEV.md @@ -17,6 +17,7 @@ to for permanent data storage: :exclamation: be sure to read for critical one-time manual setup instructions. * [PostgreSQL](../backends/meadowlark-postgresql-backend/docker/readme.md) * [OpenSearch](../backends/meadowlark-opensearch-backend/docker/readme.md) +* [ElasticSearch](../backends/meadowlark-elasticsearch-backend/docker/readme.md) ## Global Docker Configuration diff --git a/docs/LOCALHOST.md b/docs/LOCALHOST.md index 3fb08639..6b65d219 100644 --- a/docs/LOCALHOST.md +++ b/docs/LOCALHOST.md @@ -18,6 +18,7 @@ Instructions for running a local "developer" environment on localhost: * [MongoDB](../Meadowlark-js/backends/meadowlark-mongodb-backend/docker) * [PostgreSQL](../Meadowlark-js/backends/meadowlark-postgresql-backend/docker) * [OpenSearch](../Meadowlark-js/backends/meadowlark-opensearch-backend/docker) + * [ElasticSearch](../Meadowlark-js/backends/meadowlark-elasticsearch-backend/docker) 5. Open a command prompt and navigate to the `/Meadowlark-js` folder 6. Run `npm install` 7. Run `npm run build` @@ -35,7 +36,7 @@ PostgreSQL, and OpenSearch. One mechanism is to stop the Docker containers and then delete the volumes they were using, then restart Docker. If you do not want to delete the volumes, then you can manually delete records. Examples: -### OpenSearch +### OpenSearch and ElasticSearch (Same commands work for both) Open the [DevTools console](http://localhost:5601/app/dev_tools#/console) in a browser and run these dangerous commands: