[DMS-454] Add E2E OpenSearch and ElasticSearch to scheduled job (#379)

.github/workflows/scheduled-build.yml

@@ -49,6 +49,14 @@ jobs:
         id: unit_test_step
         run: ./build-dms.ps1 UnitTest -Configuration Debug
 
+      - name: Run ElasticSearch End to End Tests
+        if: success()
+        run: ./build-dms.ps1 E2ETest -EnableElasticSearch
+
+      - name: Run OpenSearch End to End Tests
+        if: success()
+        run: ./build-dms.ps1 E2ETest -EnableOpenSearch
+
       - name: Notify Slack on success
         if: success()
         uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e #v1.26.0

build-dms.ps1

@@ -82,7 +82,11 @@ param(
 
     # Only required with E2E testing.
     [switch]
-    $EnableOpenSearch
+    $EnableOpenSearch,
+
+    # Only required with E2E testing.
+    [switch]
+    $EnableElasticSearch
 )
 
 $solutionRoot = "$PSScriptRoot/src/dms"
@@ -163,7 +167,7 @@ function SetQueryHandler {
 
     $appSettingsPath = Join-Path -Path $E2EDirectory -ChildPath "appsettings.json"
     $json = Get-Content $appSettingsPath -Raw | ConvertFrom-Json
-    if ($EnableOpenSearch) {
+    if ($EnableOpenSearch -or $EnableElasticSearch) {
         $json.QueryHandler = "opensearch"
     }
     else {
@@ -244,6 +248,18 @@ function RunE2E {
 function E2ETests {
     Invoke-Step { DockerBuild }
 
+    # Clean up all the containers and volumes
+    Invoke-Execute {
+        try {
+            Push-Location eng/docker-compose/
+            ./start-local-dms.ps1 -d -v
+            ./start-local-dms.ps1 -SearchEngine "ElasticSearch" -d -v
+        }
+        finally {
+            Pop-Location
+        }
+    }
+
     if ($EnableOpenSearch) {
         Invoke-Execute {
             try {
@@ -255,6 +271,17 @@ function E2ETests {
             }
         }
     }
+    elseif ($EnableElasticSearch) {
+        Invoke-Execute {
+            try {
+                Push-Location eng/docker-compose/
+                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e" -SearchEngine "ElasticSearch"
+            }
+            finally {
+                Pop-Location
+            }
+        }
+    }
     else {
         Invoke-Step { DockerRun }
     }
@@ -327,13 +354,16 @@ function Invoke-TestExecution {
         $Filter,
 
         [switch]
-        $EnableOpenSearch
+        $EnableOpenSearch,
+
+        [switch]
+        $EnableElasticSearch
     )
     switch ($Filter) {
-        E2ETests { Invoke-Step { E2ETests -EnableOpenSearch:$EnableOpenSearch } }
+        E2ETests { Invoke-Step { E2ETests -EnableOpenSearch:$EnableOpenSearch -EnableElasticSearch:$EnableElasticSearch } }
         UnitTests { Invoke-Step { UnitTests } }
         IntegrationTests { Invoke-Step { IntegrationTests } }
-        Default { "Unknow Test Type" }
+        Default { "Unknown Test Type" }
     }
 }
 
@@ -411,7 +441,7 @@ Invoke-Main {
             Invoke-Publish
         }
         UnitTest { Invoke-TestExecution UnitTests }
-        E2ETest { Invoke-TestExecution E2ETests -EnableOpenSearch:$EnableOpenSearch }
+        E2ETest { Invoke-TestExecution E2ETests -EnableOpenSearch:$EnableOpenSearch -EnableElasticSearch:$EnableElasticSearch }
         IntegrationTest { Invoke-TestExecution IntegrationTests }
         Coverage { Invoke-Coverage }
         Package { Invoke-BuildPackage }

eng/docker-compose/kafka-elasticsearch.yml

@@ -60,6 +60,9 @@ services:
     image: edfialliance/ed-fi-kafka-connect:pre
     ports:
       - ${CONNECT_SINK_PORT:-8084}:8083
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
     links:
       - kafka
       - zookeeper
@@ -93,6 +96,11 @@ services:
       - esdata:/usr/share/elasticsearch/data
     networks:
       - dms
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:${ELASTICSEARCH_HTTP_PORT:-9200}"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
 
 volumes:
   zookeeper-logs:

eng/docker-compose/kafka-opensearch.yml

@@ -60,6 +60,9 @@ services:
     image: edfialliance/ed-fi-kafka-connect:pre
     ports:
       - ${CONNECT_SINK_PORT:-8084}:8083
+    depends_on:
+      opensearch:
+        condition: service_healthy
     links:
       - kafka
       - zookeeper