[DMS-437] E2E testing for Elasticsearch (#364)

* Recreate indices after delete for elasticsearch

* return the original port for search

* adding netwoks to steps just like kafka-opensearch

* missing elasticsearch step network part

* adding netwerk to kafka ui for elasticsearch

eng/docker-compose/kafka-elasticsearch-ui.yml

@@ -15,14 +15,22 @@ services:
       DYNAMIC_CONFIG_ENABLED: 'true'
       KAFKA_CLUSTERS_0_NAME: kafka1
       KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+    networks:
+      - dms
 
   kibana:
-      image: docker.elastic.co/kibana/kibana:8.5.1
-      environment:
-        ELASTICSEARCH_HOSTS: http://dms-search:9200
-        ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_ADMIN_USERNAME:-admin}
-        ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_ADMIN_PASSWORD:-abcdefgh1!}
-      ports:
-        - "${ELASTICSEARCH_DASHBOARD_PORT:-5601}:5601"
-      depends_on:
-        - elasticsearch
+    image: docker.elastic.co/kibana/kibana:8.5.1
+    environment:
+      ELASTICSEARCH_HOSTS: http://dms-search:9200
+      ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_ADMIN_USERNAME:-admin}
+      ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_ADMIN_PASSWORD:-abcdefgh1!}
+    ports:
+      - "${ELASTICSEARCH_DASHBOARD_PORT:-5601}:5601"
+    depends_on:
+      - elasticsearch
+    networks:
+      - dms
+
+networks:
+  dms:
+    external: true

eng/docker-compose/kafka-elasticsearch.yml

@@ -13,6 +13,8 @@ services:
       - zookeeper-data:/zookeeper/data
       - zookeeper-txns:/zookeeper/txns
       - zookeeper-conf:/zookeeper/conf
+    networks:
+      - dms
 
   kafka:
     hostname: dms-kafka1
@@ -28,6 +30,8 @@ services:
     volumes:
       - kafka-data:/kafka/data
       - kafka-logs:/kafka/logs
+    networks:
+      - dms
 
   kafka-postgresql-source:
     hostname: kafka-postgresql-source
@@ -47,6 +51,8 @@ services:
     volumes:
       - kafka-postgresql-source-logs:/kafka/logs
       - kafka-postgresql-source-config:/kafka/config
+    networks:
+      - dms
 
   kafka-elasticsearch-sink:
     hostname: kafka-elasticsearch-sink
@@ -66,6 +72,8 @@ services:
     volumes:
       - kafka-elasticsearch-sink-logs:/kafka/logs
       - kafka-elasticsearch-sink-config:/kafka/config
+    networks:
+      - dms
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.5.1
@@ -83,6 +91,8 @@ services:
       - "${ELASTICSEARCH_ANALYZER_PORT:-9300}:9300"
     volumes:
       - esdata:/usr/share/elasticsearch/data
+    networks:
+      - dms
 
 volumes:
   zookeeper-logs:
@@ -96,3 +106,7 @@ volumes:
   kafka-elasticsearch-sink-logs:
   kafka-elasticsearch-sink-config:
   esdata:
+
+networks:
+  dms:
+    external: true

src/config/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "csharpier": {
-      "version": "0.30.0",
+      "version": "0.30.1",
       "commands": [
         "dotnet-csharpier"
       ]

src/dms/.config/dotnet-tools.json

@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "csharpier": {
-      "version": "0.30.0",
+      "version": "0.30.1",
       "commands": [
         "dotnet-csharpier"
       ]

src/dms/tests/EdFi.DataManagementService.Tests.E2E/Management/OpenSearchContainerSetup.cs

@@ -35,9 +35,17 @@ private static async Task ResetOpenSearch()
         OpenSearchClient openSearchClient = new();
         var indices = await openSearchClient.Cat.IndicesAsync();
 
-        foreach (var index in indices.Records.Where(x => x.Index.Contains("ed-fi")))
+        foreach (var index in indices.Records.Where(x => x.Index.Contains("ed-fi$")))
         {
             await openSearchClient.Indices.DeleteAsync(index.Index);
+
+            // Recreate the index with default or custom settings if needed
+            await openSearchClient.Indices.CreateAsync(index.Index, c => c
+                .Settings(s => s
+                    .NumberOfShards(1)
+                    .NumberOfReplicas(1)
+                )
+            );
         }
     }
 }