Fix tests for 2.17 (#279)

* remove langbind tests

* bump minimum supported version to 2.17.1-cloud

* switch to supported models

* fix version check

* fix more tests

* rerank depth test fix

* skip image chunking test

* fix recommend doc fields

* Temporarily disable flaky test

---------

Co-authored-by: yihanzhao <[email protected]>
Co-authored-by: danyilq <[email protected]>

Author: 3 people

.github/workflows/open-source-unit-tests.yml

@@ -99,7 +99,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           export PYTHONPATH=$(pwd):$(pwd)/src:$PYTHONPATH
-          SUPPORTED_MQ_VERSION=$(python -c 'from marqo import version; print(version.__minimum_supported_marqo_version__)') || exit 1
+          SUPPORTED_MQ_VERSION=$(python -c 'from marqo import version; print(f"{version.__minimum_supported_marqo_version__}-cloud")') || exit 1
 
           # error out if version is empty:
           if [ -z "$SUPPORTED_MQ_VERSION" ]; then exit 1; fi

src/marqo/version.py

@@ -1,4 +1,4 @@
-__minimum_supported_marqo_version__ = "2.15.0"
+__minimum_supported_marqo_version__ = "2.17.1"
 
 # NOTE: This isn't used anywhere
 def supported_marqo_version() -> str:

tests/cloud_test_logic/cloud_test_index.py

@@ -14,7 +14,6 @@ class CloudTestIndex(str, Enum):
     3) unstructured_no_model: 512-dimension custom vectors, 1 shard, no replicas, CPU, basic storage.
     4) structured_text: Structured text index with hf/e5-base-v2, lexical search, 2 shards, 1 replica, CPU, balanced storage.
     5) structured_image: Structured image-text index with open_clip/ViT-B-32, 2 shards, 1 replica, CPU, balanced storage, with image preprocessing.
-    6) structured_languagebind_model: a structured index using the LanguageBind model for multi-modal support.
     For more information on the settings of each index, please refer to index_name_to_settings_mappings.
 
     FOR CLOUD REPLICAS AND SHARDS:
@@ -35,7 +34,6 @@ class CloudTestIndex(str, Enum):
     structured_image_custom = "pymarqo_str_img_custom"
     structured_text = "pymarqo_str_txt"
     structured_image = "pymarqo_str_img"
-    structured_languagebind_model = "pymarqo_str_langbind_model"
 
 
 index_name_to_settings_mappings = {
@@ -123,40 +121,4 @@ class CloudTestIndex(str, Enum):
             "patchMethod": "simple",
         }
     },
-    CloudTestIndex.structured_languagebind_model: {
-        "type": "structured",
-        "model": "LanguageBind/Video_V1.5_FT_Audio_FT_Image",
-        "inferenceType": "marqo.GPU",
-        "storageClass": "marqo.balanced",
-        "allFields": [
-            {"name": "text_field_1", "type": "text"},
-            {"name": "text_field_2", "type": "text"},
-            {"name": "text_field_3", "type": "text"},
-            {"name": "video_field_1", "type": "video_pointer"},
-            {"name": "video_field_2", "type": "video_pointer"},
-            {"name": "video_field_3", "type": "video_pointer"},
-            {"name": "audio_field_1", "type": "audio_pointer"},
-            {"name": "audio_field_2", "type": "audio_pointer"},
-            {"name": "image_field_1", "type": "image_pointer"},
-            {"name": "image_field_2", "type": "image_pointer"},
-            {
-                "name": "multimodal_field",
-                "type": "multimodal_combination",
-                "dependentFields": {
-                    "text_field_1": 0.1,
-                    "text_field_2": 0.1,
-                    "image_field_1": 0.5,
-                    "video_field_1": 0.1,
-                    "video_field_2": 0.1,
-                    "audio_field_1": 0.1
-                }
-            },
-            {"name": "map_int_score_modifier_field", "type": "map<text, int>", "features": ["score_modifier"]},
-            {"name": "map_double_score_modifier_field", "type": "map<text, double>", "features": ["score_modifier"]},
-            {"name": "map_float_score_modifier_field", "type": "map<text, float>", "features": ["score_modifier"]},
-            {"name": "map_long_score_modifier_field", "type": "map<text, long>", "features": ["score_modifier"]},
-        ],
-        "tensorFields": ["multimodal_field", "text_field_3", "video_field_3", "audio_field_2", "image_field_2"],
-        "normalizeEmbeddings": True,
-    },
 }

tests/marqo_test.py

@@ -209,15 +209,11 @@ def setUpClass(cls) -> None:
         cls.unstructured_no_model_index_name = "unstructured_no_model_index"
         cls.structured_image_index_name_simple_preprocessing_method = \
             "structured_image_index_simple_preprocessing_method"
-        cls.structured_languagebind_index_name = "structured_languagebind_index"
 
         # TODO: include structured when boolean_field bug for structured is fixed
         cls.test_cases = [
             (CloudTestIndex.unstructured_image, cls.unstructured_index_name),
         ]
-        cls.test_cases_multimodal = [
-            (CloudTestIndex.structured_languagebind_model, cls.structured_languagebind_index_name)
-        ]
 
         # class property to indicate if test is being run on multi
         cls.IS_MULTI_INSTANCE = (True if os.environ.get("IS_MULTI_INSTANCE", False) in ["True", "TRUE", "true", True] else False)
@@ -245,7 +241,7 @@ def setUpClass(cls) -> None:
                         "indexName": cls.unstructured_image_index_name,
                         "type": "unstructured",
                         "treatUrlsAndPointersAsImages": True,
-                        "model": "ViT-B/32",
+                        "model": "open_clip/ViT-B-32/laion400m_e32",
                     },
                     {
                         "indexName": cls.structured_image_index_name,
@@ -256,7 +252,7 @@ def setUpClass(cls) -> None:
                                       {"name": "image_field_1", "type": "image_pointer"},
                                       ],
                         "tensorFields": ["text_field_1", "text_field_2", "text_field_3", "image_field_1"],
-                        "model": "ViT-B/32",
+                        "model": "open_clip/ViT-B-32/laion400m_e32",
                     },
                     {
                         "indexName": cls.unstructured_no_model_index_name,
@@ -268,36 +264,6 @@ def setUpClass(cls) -> None:
                             "dimensions": 512
                         }
                     },
-                    {
-                        "indexName": cls.structured_languagebind_index_name,
-                        "type": "structured",
-                        "model": "LanguageBind/Video_V1.5_FT_Audio_FT_Image",
-                        "allFields": [
-                            {"name": "text_field_1", "type": "text"},
-                            {"name": "text_field_2", "type": "text"},
-                            {"name": "text_field_3", "type": "text"},
-                            {"name": "video_field_1", "type": "video_pointer"},
-                            {"name": "video_field_2", "type": "video_pointer"},
-                            {"name": "video_field_3", "type": "video_pointer"},
-                            {"name": "audio_field_1", "type": "audio_pointer"},
-                            {"name": "audio_field_2", "type": "audio_pointer"},
-                            {"name": "image_field_1", "type": "image_pointer"},
-                            {"name": "image_field_2", "type": "image_pointer"},
-                            {
-                                "name": "multimodal_field", 
-                                "type": "multimodal_combination",
-                                "dependentFields": {
-                                    "text_field_1": 0.1,
-                                    "text_field_2": 0.1,
-                                    "image_field_1": 0.5,
-                                    "video_field_1": 0.1,
-                                    "video_field_2": 0.1,
-                                    "audio_field_1": 0.1
-                                }
-                            },
-                        ],
-                        "tensorFields": ["multimodal_field", "text_field_3", "video_field_3", "audio_field_2", "image_field_2"]
-                    }
                 ])
             except Exception as e:
                 print("Error creating indexes: ", e)

tests/v2_tests/test_add_documents.py

@@ -588,9 +588,6 @@ def test_no_model_custom_vector_doc(self):
         Note: `no_model` is not yet supported on Cloud.
         """
         self.test_cases = [(CloudTestIndex.unstructured_no_model, self.unstructured_no_model_index_name)]
-        self.test_cases_multimodal = [
-            (CloudTestIndex.structured_languagebind_model, self.structured_languagebind_index_name)
-        ]
 
         for cloud_test_index_to_use, open_source_test_index_name in self.test_cases:
             test_index_name = self.get_test_index_name(
@@ -707,103 +704,6 @@ def test_add_empty_docs_batched(self):
                                                                    tensor_fields="field a")
             assert res == []
 
-    def test_add_multimodal_single_documents(self):
-        documents = [
-            {
-                "video_field_3": "https://marqo-k400-video-test-dataset.s3.amazonaws.com/videos/---QUuC4vJs_000084_000094.mp4",
-                "_id": "1"
-            },
-            {
-                "audio_field_2": "https://marqo-ecs-50-audio-test-dataset.s3.amazonaws.com/audios/marqo-audio-test.mp3",
-                "_id": "2"
-            },
-            {
-                "image_field_2": "https://raw.githubusercontent.com/marqo-ai/marqo-api-tests/mainline/assets/ai_hippo_realistic.png",
-                "_id": "3"
-            },
-            {
-                "text_field_3": "hello there padawan. Today you will begin your training to be a Jedi",
-                "_id": "4"
-            },
-        ]
-        for cloud_test_index_to_use, open_source_test_index_name in self.test_cases_multimodal:
-            if "languagebind" not in str(cloud_test_index_to_use):
-                continue
-            test_index_name = self.get_test_index_name(
-                cloud_test_index_to_use=cloud_test_index_to_use,
-                open_source_test_index_name=open_source_test_index_name
-            )
-            with self.subTest(test_index_name):
-                tensor_fields = ["text_field_3", "image_field_2", "video_field_3", "audio_field_2"] if "unstructured" in test_index_name else None
-                res = self.client.index(test_index_name).add_documents(documents, tensor_fields=tensor_fields)
-                print(res)
-                
-                for item in res['items']:
-                    self.assertEqual(200, item['status'])
-
-                get_res = self.client.index(test_index_name).get_documents(
-                    document_ids=["1", "2", "3", "4"],
-                    expose_facets=True
-                )
-                print(get_res)
-
-                for i, doc in enumerate(get_res['results']):
-                    i += 1
-                    tensor_facets = doc['_tensor_facets']
-                    self.assertIn('_embedding', tensor_facets[0])
-                    self.assertEqual(len(tensor_facets[0]['_embedding']), 768)
-
-    def test_add_multimodal_field_document(self):
-        multimodal_document = [{
-            "_id": "1_multimodal",
-            "text_field_1": "New York",
-            "text_field_2": "Los Angeles",
-            "image_field_1": "https://raw.githubusercontent.com/marqo-ai/marqo-api-tests/mainline/assets/ai_hippo_realistic.png",
-            "video_field_1": "https://marqo-k400-video-test-dataset.s3.amazonaws.com/videos/---QUuC4vJs_000084_000094.mp4",
-            "video_field_2": "https://marqo-k400-video-test-dataset.s3.amazonaws.com/videos/---QUuC4vJs_000084_000094.mp4",
-            "audio_field_1": "https://marqo-ecs-50-audio-test-dataset.s3.amazonaws.com/audios/marqo-audio-test.mp3",
-        }]
-        for cloud_test_index_to_use, open_source_test_index_name in self.test_cases_multimodal:
-            if "languagebind" not in str(cloud_test_index_to_use):
-                continue
-            test_index_name = self.get_test_index_name(
-                cloud_test_index_to_use=cloud_test_index_to_use,
-                open_source_test_index_name=open_source_test_index_name
-            )
-            with self.subTest(test_index_name):
-                mappings = {
-                    "multimodal_field": {
-                        "type": "multimodal_combination",
-                        "weights": {
-                            "text_field_1": 0.1,
-                            "text_field_2": 0.1,
-                            "image_field_1": 0.5,
-                            "video_field_1": 0.1,
-                            "video_field_2": 0.1,
-                            "audio_field_1": 0.1
-                        },
-                    }
-                } if "unstructured" in test_index_name else None
-                tensor_fields = ["multimodal_field"] if "unstructured" in test_index_name else None
-                res = self.client.index(test_index_name).add_documents(
-                    multimodal_document,
-                    tensor_fields=tensor_fields,
-                    mappings=mappings
-                )
-                print(res)
-
-                for item in res['items']:
-                    self.assertEqual(200, item['status'])
-
-                doc = self.client.index(test_index_name).get_documents(
-                    document_ids=["1_multimodal"],
-                    expose_facets=True
-                )
-                print(doc)
-
-                self.assertIn('_tensor_facets', doc['results'][0])
-                self.assertIn('_embedding', doc['results'][0]['_tensor_facets'][0])
-                self.assertEqual(len(doc['results'][0]['_tensor_facets'][0]['_embedding']), 768)
 
     def test_media_download_headers_is_not_included(self):
         """Ensure newly added attributes mediaDownloadHeaders is not included in the request body."""
@@ -835,4 +735,4 @@ def run():
             self.assertEqual({"key": "value-2"}, kwargs["body"]["imageDownloadHeaders"])
             self.assertEqual({"key": "value-1"}, kwargs["body"]["mediaDownloadHeaders"])
             return True
-        run()
+        run()