From 3e8e6f60103da895c133316788e18fcd080a7010 Mon Sep 17 00:00:00 2001
From: Johnny Brenes <103782746+JBrenesSimpat@users.noreply.github.com>
Date: Mon, 9 Dec 2024 09:36:56 -0600
Subject: [PATCH] [DMS-437] E2E testing for Elasticsearch (#364)

* Recreate indices after delete for elasticsearch

* return the original port for search

* adding netwoks to steps just like kafka-opensearch

* missing elasticsearch step network part

* adding netwerk to kafka ui for elasticsearch
---
 eng/docker-compose/kafka-elasticsearch-ui.yml | 26 ++++++++++++-------
 eng/docker-compose/kafka-elasticsearch.yml    | 14 ++++++++++
 src/config/.config/dotnet-tools.json          |  2 +-
 src/dms/.config/dotnet-tools.json             |  2 +-
 .../Management/OpenSearchContainerSetup.cs    | 10 ++++++-
 5 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/eng/docker-compose/kafka-elasticsearch-ui.yml b/eng/docker-compose/kafka-elasticsearch-ui.yml
index b033b36c3..07c9f0628 100644
--- a/eng/docker-compose/kafka-elasticsearch-ui.yml
+++ b/eng/docker-compose/kafka-elasticsearch-ui.yml
@@ -15,14 +15,22 @@ services:
       DYNAMIC_CONFIG_ENABLED: 'true'
       KAFKA_CLUSTERS_0_NAME: kafka1
       KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+    networks:
+      - dms
 
   kibana:
-      image: docker.elastic.co/kibana/kibana:8.5.1
-      environment:
-        ELASTICSEARCH_HOSTS: http://dms-search:9200
-        ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_ADMIN_USERNAME:-admin}
-        ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_ADMIN_PASSWORD:-abcdefgh1!}
-      ports:
-        - "${ELASTICSEARCH_DASHBOARD_PORT:-5601}:5601"
-      depends_on:
-        - elasticsearch
+    image: docker.elastic.co/kibana/kibana:8.5.1
+    environment:
+      ELASTICSEARCH_HOSTS: http://dms-search:9200
+      ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_ADMIN_USERNAME:-admin}
+      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_ADMIN_PASSWORD:-abcdefgh1!}
+    ports:
+      - "${ELASTICSEARCH_DASHBOARD_PORT:-5601}:5601"
+    depends_on:
+      - elasticsearch
+    networks:
+      - dms
+
+networks:
+  dms:
+    external: true
diff --git a/eng/docker-compose/kafka-elasticsearch.yml b/eng/docker-compose/kafka-elasticsearch.yml
index 7ae7ddd76..b270728c6 100644
--- a/eng/docker-compose/kafka-elasticsearch.yml
+++ b/eng/docker-compose/kafka-elasticsearch.yml
@@ -13,6 +13,8 @@ services:
       - zookeeper-data:/zookeeper/data
       - zookeeper-txns:/zookeeper/txns
       - zookeeper-conf:/zookeeper/conf
+    networks:
+      - dms
 
   kafka:
     hostname: dms-kafka1
@@ -28,6 +30,8 @@ services:
     volumes:
       - kafka-data:/kafka/data
       - kafka-logs:/kafka/logs
+    networks:
+      - dms
 
   kafka-postgresql-source:
     hostname: kafka-postgresql-source
@@ -47,6 +51,8 @@ services:
     volumes:
       - kafka-postgresql-source-logs:/kafka/logs
       - kafka-postgresql-source-config:/kafka/config
+    networks:
+      - dms
 
   kafka-elasticsearch-sink:
     hostname: kafka-elasticsearch-sink
@@ -66,6 +72,8 @@ services:
     volumes:
       - kafka-elasticsearch-sink-logs:/kafka/logs
       - kafka-elasticsearch-sink-config:/kafka/config
+    networks:
+      - dms
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.5.1
@@ -83,6 +91,8 @@ services:
       - "${ELASTICSEARCH_ANALYZER_PORT:-9300}:9300"
     volumes:
       - esdata:/usr/share/elasticsearch/data
+    networks:
+      - dms
 
 volumes:
   zookeeper-logs:
@@ -96,3 +106,7 @@ volumes:
   kafka-elasticsearch-sink-logs:
   kafka-elasticsearch-sink-config:
   esdata:
+
+networks:
+  dms:
+    external: true
diff --git a/src/config/.config/dotnet-tools.json b/src/config/.config/dotnet-tools.json
index e08b45e94..1bb47fc64 100644
--- a/src/config/.config/dotnet-tools.json
+++ b/src/config/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "csharpier": {
-      "version": "0.30.0",
+      "version": "0.30.1",
       "commands": [
         "dotnet-csharpier"
       ]
diff --git a/src/dms/.config/dotnet-tools.json b/src/dms/.config/dotnet-tools.json
index e08b45e94..1bb47fc64 100644
--- a/src/dms/.config/dotnet-tools.json
+++ b/src/dms/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "csharpier": {
-      "version": "0.30.0",
+      "version": "0.30.1",
       "commands": [
         "dotnet-csharpier"
       ]
diff --git a/src/dms/tests/EdFi.DataManagementService.Tests.E2E/Management/OpenSearchContainerSetup.cs b/src/dms/tests/EdFi.DataManagementService.Tests.E2E/Management/OpenSearchContainerSetup.cs
index 7dd150432..844f04b1b 100644
--- a/src/dms/tests/EdFi.DataManagementService.Tests.E2E/Management/OpenSearchContainerSetup.cs
+++ b/src/dms/tests/EdFi.DataManagementService.Tests.E2E/Management/OpenSearchContainerSetup.cs
@@ -35,9 +35,17 @@ private static async Task ResetOpenSearch()
         OpenSearchClient openSearchClient = new();
         var indices = await openSearchClient.Cat.IndicesAsync();
 
-        foreach (var index in indices.Records.Where(x => x.Index.Contains("ed-fi")))
+        foreach (var index in indices.Records.Where(x => x.Index.Contains("ed-fi$")))
         {
             await openSearchClient.Indices.DeleteAsync(index.Index);
+
+            // Recreate the index with default or custom settings if needed
+            await openSearchClient.Indices.CreateAsync(index.Index, c => c
+                .Settings(s => s
+                    .NumberOfShards(1)
+                    .NumberOfReplicas(1)
+                )
+            );
         }
     }
 }