[RND-511] Refactor configuration or containers for testing to have a …

…better handling and control (#228)

* Fix error logged when there's an error creating resources

* Update test containers

* Adding method to build docker image

* Moving API Configuration to coded setup

* Creating code for mongo connection

* Change port for default mongo connection

* Removing action step related with mongo permissions

* Adding open search configuration

* Changing jest config to use default ts-jest setup

* [RND-511] Refactor container usage to improve management

* Using code based configuration for opensearch tests

* Filename casing issue

* Restoring ts-jest import

* Fixing compilation issue on Mongo backend integration tests

* Improving error message handling on setup for e2e

* Adding debug option for e2e tests

* Refactor to workaround issue with building Dockerfile

* Updating documentation

* Removing flag no longer used

* Update script names

---------

Co-authored-by: Stephen Fuqua <[email protected]>

.github/workflows/on-pullrequest.yml

@@ -269,11 +269,8 @@ jobs:
           echo OAUTH_SIGNING_KEY="$( openssl rand -base64 256 | tr -d '\n' )" >> .env
         working-directory: Meadowlark-js/tests/e2e/setup/
 
-      - name: Give permission for scripts
-        run: chmod -R +x ./tests/e2e/setup/scripts/
-
       - name: End to End tests for ${{matrix.config.db}}
-        run: npm run test:e2e -- --ci
+        run: npm run test:e2e:build -- --ci
         env:
           DOCUMENT_STORE_PLUGIN: ${{ matrix.config.plugin }}
 

Meadowlark-js/.vscode/settings.json

@@ -33,6 +33,7 @@
     "FASTIFY",
     "metaed",
     "opensearch",
+    "opensearchproject",
     "teamcity",
     "testcontainers",
     "uncapitalize"

Meadowlark-js/backends/meadowlark-mongodb-backend/test/config/integration/jest.config.js

@@ -2,7 +2,7 @@
 const defaultConfig = require('../../../../../tests/config/jest.config');
 
 module.exports = {
-  displayName: 'Integration Tests: Mongodb ',
+  displayName: 'Integration Tests: Mongodb',
   ...defaultConfig,
   preset: '@shelf/jest-mongodb',
   testMatch: ['**/meadowlark-mongodb-backend/test/integration/**/*.(spec|test).[jt]s?(x)'],

Meadowlark-js/backends/meadowlark-opensearch-backend/package.json

@@ -29,6 +29,6 @@
     "copyfiles": "^2.4.1",
     "dotenv": "^16.0.3",
     "rimraf": "^3.0.2",
-    "testcontainers": "^9.1.2"
+    "testcontainers": "^9.3.0"
   }
 }

Meadowlark-js/backends/meadowlark-opensearch-backend/test/setup/OpenSearchContainer.ts

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+import { GenericContainer, StartedTestContainer } from 'testcontainers';
+
+let startedContainer: StartedTestContainer;
+
+export async function setup() {
+  const openSearchPort = parseInt(process.env.OPENSEARCH_PORT ?? '8201', 10);
+  startedContainer = await new GenericContainer(
+    'opensearchproject/opensearch:2.5.0@sha256:f077efb452be64d3df56d74fe99fd63244704896edf6ead73a0f5decb95a40bf',
+  )
+    .withName('opensearch-test')
+    .withExposedPorts({
+      container: openSearchPort,
+      host: openSearchPort,
+    })
+    .withEnvironment({
+      'discovery.type': 'single-node',
+      DISABLE_INSTALL_DEMO_CONFIG: 'true',
+      DISABLE_SECURITY_PLUGIN: 'true',
+      'http.port': `${openSearchPort}`,
+    })
+    .withStartupTimeout(120_000)
+    .start();
+
+  process.env.OPENSEARCH_ENDPOINT = `http://localhost:${openSearchPort}`;
+  process.env.OPENSEARCH_USERNAME = 'admin';
+  process.env.OPENSEARCH_PASSWORD = 'admin';
+  process.env.OPENSEARCH_REQUEST_TIMEOUT = '10000';
+}
+
+export async function stop(): Promise<void> {
+  await startedContainer.stop();
+}

Meadowlark-js/backends/meadowlark-opensearch-backend/test/setup/OpenSearchSetupEnvironment.ts

@@ -2,40 +2,22 @@
 // Licensed to the Ed-Fi Alliance under one or more agreements.
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
-import path from 'path';
-import { DockerComposeEnvironment, StartedDockerComposeEnvironment } from 'testcontainers';
-import dotenv from 'dotenv';
+
 import { Logger } from '@edfi//meadowlark-utilities';
 import { Client } from '@opensearch-project/opensearch/.';
+import * as OpenSearchContainer from './OpenSearchContainer';
 import { getOpenSearchClient } from '../../src/repository/Db';
 
-const containerName = 'opensearch-integration-test';
-const envFilePath = path.join(__dirname, '.env');
 const moduleName = 'opensearch.repository.Db';
-const port = 8200;
-let environment: StartedDockerComposeEnvironment;
-let host: string;
-dotenv.config({ path: envFilePath });
 
 export async function setupOpenSearch() {
-  const composeFile = 'docker-compose.yml';
-  const composeFilePath = path.resolve(__dirname, './');
   Logger.info('-- Setup OpenSearch environment --', null);
-  environment = await new DockerComposeEnvironment(composeFilePath, composeFile)
-    .withNoRecreate()
-    .withStartupTimeout(120_000)
-    .up();
-  const container = await environment.getContainer(containerName);
-  host = await container.getHost();
-  process.env.OPENSEARCH_ENDPOINT = `http://${host}:${port}`;
-  process.env.OPENSEARCH_USERNAME = 'admin';
-  process.env.OPENSEARCH_PASSWORD = 'admin';
-  process.env.OPENSEARCH_REQUEST_TIMEOUT = '10000';
+  await OpenSearchContainer.setup();
 }
 
 export async function teardownOpenSearch() {
   Logger.info('-- Tearing down OpenSearch environment --', null);
-  await environment.getContainer(containerName).stop();
+  await OpenSearchContainer.stop();
 }
 
 /**

Meadowlark-js/package.json

@@ -34,7 +34,7 @@
   },
   "workspaces": [
     "packages/meadowlark-utilities",
-    "packages/meadowlark-core",    
+    "packages/meadowlark-core",
     "packages/meadowlark-authz-server",
     "backends/*",
     "services/*",
@@ -55,7 +55,10 @@
     "test:unit:coverage": "rimraf coverage.unit/ && npm run test:unit",
     "test:unit:coverage:ci": "rimraf coverage.unit/ && npm run test:unit -- --maxWorkers=2 --ci",
     "test:integration:coverage": "rimraf coverage.integration/ && npm run test:integration -- --collectCoverage",
-    "test:e2e": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects tests/config/e2e --runInBand",
+    "test:e2e:jest": "cross-env NODE_OPTIONS=--max-old-space-size=6144 LOG_LEVEL=warn jest --projects tests/config/e2e --runInBand",
+    "test:e2e:debug": "cross-env DEBUG=testcontainers* npm run test:e2e:jest",
+    "test:e2e:build": "npm run docker:build && npm run test:e2e:jest",
+    "test:e2e:build:wsl1": "npm run docker:build:wsl1 && npm run test:e2e:jest",
     "start:local": "lerna run start:local",
     "docker:lint": "cat ./Dockerfile | docker run --rm -i hadolint/hadolint",
     "docker:install": "npm i lerna@^6.0.2 rimraf@^3.0.2 copyfiles@^2.4.1 [email protected] -g && npm ci --only=production",