Adding a unit test for OS2x to OS2x migration

Signed-off-by: Jugal Chauhan <[email protected]>

Author: jugal-chauhan

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test/integ_test/test_cases/backfill_tests.py

@@ -182,8 +182,85 @@ def __init__(self, console_config_path: str, console_link_env: Environment, uniq
     def test_before(self):
         # Create a single index
         self.source_operations.create_index(cluster=self.source_cluster, index_name=self.index_name)
-        self.source_operations.disable_bloom(cluster=self.source_cluster, index_name=self.index_name)
-        self.source_operations.refresh(cluster=self.source_cluster, index_name=self.index_name)
+        self.source_operations.get_index(cluster=self.source_cluster, index_name=self.index_name)
+
+        # Create a single document
+        self.source_operations.create_document(
+            cluster=self.source_cluster,
+            index_name=self.index_name,
+            doc_id=self.doc_id1
+        )
+        self.source_operations.get_document(
+            cluster=self.source_cluster,
+            index_name=self.index_name,
+            doc_id=self.doc_id1
+        )
+
+        # Create a single document
+        self.source_operations.create_document(
+            cluster=self.source_cluster,
+            index_name=self.index_name,
+            doc_id=self.doc_id2
+        )
+        self.source_operations.get_document(
+            cluster=self.source_cluster,
+            index_name=self.index_name,
+            doc_id=self.doc_id2
+        )
+
+        # Use no-op transformer
+        noop_transform = self.source_operations.get_noop_transformation()
+        self.source_operations.create_transformation_json_file(
+            transform_config_data=[noop_transform],
+            file_path_to_create=self.transform_config_file
+        )
+
+    def metadata_migrate(self):
+        metadata_result: CommandResult = self.metadata.migrate(
+            extra_args=[
+                "--transformer-config-file", self.transform_config_file,
+                "--index-allowlist", self.index_name,
+                "--component-template-allowlist", "none",
+                "--index-template-allowlist", "none"
+            ]
+        )
+        assert metadata_result.success
+
+    def metadata_after(self):
+        self.target_operations.get_index(
+            cluster=self.target_cluster,
+            index_name=self.index_name,
+            max_attempts=5,
+            delay=2.0
+        )
+
+    def backfill_wait_for_stop(self):
+        return super().backfill_wait_for_stop()
+
+
+
+class Test0009OS2TestsWithBestCompression(MATestBase):
+    def __init__(self, console_config_path: str, console_link_env: Environment, unique_id: str):
+        allow_combinations = [
+            (ElasticsearchV2_X, OpensearchV2_X)
+        ]
+        run_isolated = True
+        description = "For ES 8x test using 2 simple docs in 1 index to perform metadata and backfill."
+        super().__init__(console_config_path=console_config_path,
+                         console_link_env=console_link_env,
+                         unique_id=unique_id,
+                         description=description,
+                         allow_source_target_combinations=allow_combinations,
+                         migrations_required=[MigrationType.BACKFILL, MigrationType.METADATA],
+                         run_isolated=run_isolated)
+        self.transform_config_file = "/shared-logs-output/test-transformations/transformation.json"
+        self.index_name = f"test_0010_{self.unique_id}"
+        self.doc_id1 = "test_1000_doc"
+        self.doc_id2 = "test_2000_doc"
+
+    def test_before(self):
+        # Create a single index
+        self.source_operations.create_index(cluster=self.source_cluster, index_name=self.index_name)
         self.source_operations.get_index(cluster=self.source_cluster, index_name=self.index_name)
 
         # Create a single document

libraries/testAutomation/os-2-to-os-2-values.yaml

@@ -0,0 +1,189 @@
+conditionalPackageInstalls:
+  bulkLoader: true
+  grafana: false
+  jaeger: false
+  kafka: true
+  migrationConsole: true
+  prometheus: false
+  proxy: true
+  replayer: true
+  sharedConfigs: true
+  snapshotVolume: true
+  logsVolume: true
+
+shared-configs:
+  globalParameters:
+    sourceCluster:
+      allowRuntimeOverride: true
+      object:
+        endpoint: "http://opensearch-source:9200"
+        directEndpoint: "http://opensearch-source:9200"
+        allowInsecure: true
+        authType: "basic_auth"
+        basicAuthUsername: "admin"
+        basicAuthPassword: "myStrongPassword123!"
+        basicAuthHeader: "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE="
+        version: "OS_2.19"
+    targetCluster:
+      allowRuntimeOverride: true
+      object:
+        endpoint: "https://opensearch-cluster-master:9200"
+        allowInsecure: true
+        authType: "basic_auth"
+        basicAuthUsername: "admin"
+        basicAuthPassword: "myStrongPassword123!"
+        basicAuthHeader: "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE="
+        version: "OS_2.19"
+    kafkaBrokers:
+      allowRuntimeOverride: false
+      object:
+        brokers: "captured-traffic-kafka-bootstrap.ma.svc:9092"
+        kafkaType: "standard"
+    snapshot:
+      allowRuntimeOverride: true
+      object:
+        repoPath: "/snapshot"
+        snapshotName: "migration-assistant-snapshot"
+        snapshotType: "fs"
+    metadataMigration:
+      allowRuntimeOverride: true
+      object:
+        clusterAwarenessAttributes: 1
+        metadataType: "from_snapshot"
+        sourceClusterVersion: "OS_2.19"
+
+capture-proxy:
+  parameters:
+    destinationUri:
+      source: otherConfig
+      configMapName: "source-cluster"
+      configMapKey: "directEndpoint"
+    insecureDestination:
+      source: otherConfig
+      configMapName: "source-cluster"
+      configMapKey: "allowInsecure"
+    kafkaConnection:
+      source: otherConfig
+      configMapName: "kafka-brokers"
+      configMapKey: "brokers"
+
+migration-console:
+  snapshotVolumeEnabled: true
+  snapshotVolumePvc: "snapshot-volume-pvc"
+  sharedLogsVolumeEnabled: true
+  sharedLogsPvc: "shared-logs-pvc"
+
+bulk-document-loader:
+  snapshotVolumeEnabled: true
+  snapshotVolumePvc: "snapshot-volume-pvc"
+  sharedLogsVolumeEnabled: true
+  sharedLogsPvc: "shared-logs-pvc"
+  parameters:
+    snapshotName:
+      source: otherConfig
+      configMapName: "snapshot"
+      configMapKey: "snapshotName"
+    targetHost:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "endpoint"
+    targetInsecure:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "allowInsecure"
+      parameterType: booleanFlag
+    targetUsername:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "basicAuthUsername"
+    targetPassword:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "basicAuthPassword"
+    sourceVersion:
+      source: otherConfig
+      configMapName: "source-cluster"
+      configMapKey: "version"
+    docTransformerConfigFile:
+      source: parameterConfig
+      value: "/shared-logs-output/test-transformations/transformation.json"
+      allowRuntimeOverride: true
+
+replayer:
+  sharedLogsVolumeEnabled: true
+  sharedLogsPvc: "shared-logs-pvc"
+  parameters:
+    kafkaTrafficBrokers:
+      source: otherConfig
+      configMapName: "kafka-brokers"
+      configMapKey: "brokers"
+    targetUri:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "endpoint"
+    insecure:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "allowInsecure"
+      parameterType: booleanFlag
+    authHeaderValue:
+      source: otherConfig
+      configMapName: "target-cluster"
+      configMapKey: "basicAuthHeader"
+    speedupFactor:
+      source: parameterConfig
+      value: 10
+      allowRuntimeOverride: true
+    transformerConfigFile:
+      source: parameterConfig
+      value: "/shared-logs-output/test-transformations/transformation.json"
+      allowRuntimeOverride: true
+
+captured-traffic-kafka-cluster:
+  environment: test
+  clusterName: captured-traffic
+  replicas: 1
+  storageType: ephemeral
+  storageSize: 100Gi
+  storageDeleteClaim: true
+  dedicatedController:
+    replicas: 1
+    storageSize: 10Gi
+
+jaeger:
+  allInOne:
+    enabled: true
+  provisionDataStore:
+    cassandra: false
+  storage:
+    type: memory
+  agent:
+    enabled: false
+  collector:
+    enabled: false
+  query:
+    enabled: false
+
+grafana:
+  datasources:
+    datasources.yaml:
+      apiVersion: 1
+      datasources:
+        - name: Prometheus
+          type: prometheus
+          access: proxy
+          url: http://prometheus-server.prometheus.svc.cluster.local:9090
+          isDefault: true
+          editable: true
+        - name: Jaeger
+          type: jaeger
+          access: proxy
+          url: http://jaeger-query.jaeger.svc.cluster.local:16686
+          isDefault: false
+          editable: true
+  sidecar:
+    datasources:
+      enabled: true
+    dashboards:
+      enabled: true
+      label: grafana_dashboard

libraries/testAutomation/testAutomation/test_runner.py

@@ -13,7 +13,7 @@
 logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-VALID_SOURCE_VERSIONS = ["ES_5.6", "ES_8.x"]
+VALID_SOURCE_VERSIONS = ["ES_5.6", "ES_8.x", "OS_2.x"]
 VALID_TARGET_VERSIONS = ["OS_2.x"]
 SOURCE_RELEASE_NAME = "source"
 TARGET_RELEASE_NAME = "target"