[benchmark][MOAT] Shuffle choices for MOAT (#852)

* [benchmark][MOAT] Shuffle choices for MOAT

* [benchmark] [MOAT] set random seed for choice shuffle

Author: waltsun

vlmeval/dataset/moat.py

@@ -6,8 +6,11 @@
 from .utils import DEBUG_MESSAGE
 
 import zipfile
+from random import shuffle, seed
 
 
+RANDOM_SEED = 0
+
 VQA_SYSTEM_PROMPT = json.dumps({
     'task': 'Answer the question presented to you truthfully.',
     'requirements': [
@@ -42,6 +45,10 @@ class MOAT(ImageBaseDataset):
         'MOAT': '803b5a176a5b01aa1b8094fae73532a2',
     }
 
+    def __init__(self, dataset, **kwargs):
+        super().__init__(dataset, **kwargs)
+        seed(RANDOM_SEED) # seed the random number generator to ensure reproducibility
+
     def post_build(self, dataset):
         assert dataset == "MOAT", f"Wrong dataset name {dataset}"
         ROOT = LMUDataRoot()
@@ -70,16 +77,19 @@ def build_prompt(self, line):
         question, choices, images, outside_knowledge_text, outside_knowledge_images = line['question'], line['choices'], line['images'], line['outside_knowledge_text'], line['outside_knowledge_images']  # noqa: E501
         choices, images, outside_knowledge_images = toliststr(choices), toliststr(images), toliststr(outside_knowledge_images)  # noqa: E501
 
+        if len(choices):
+            shuffle(choices)        # shuffle the choices to avoid bias
+            question += f'\nThe choices are: {choices}'
         msgs = [
             {
                 'type': 'text',
-                'value': VQA_SYSTEM_PROMPT + '\n' + question + (f'\nThe choices are: {choices}' if choices else ''),
+                'value': VQA_SYSTEM_PROMPT + '\n' + question,
             },
         ]
         for img in images:
             msgs.append({'type': 'image', 'value': osp.join(self.img_root, img)})
         if not pd.isna(outside_knowledge_text):
-            msgs.append({'type': 'text', 'value': outside_knowledge_text})
+            msgs.append({'type': 'text', 'value': 'Hint:\n' + outside_knowledge_text})
         for img in outside_knowledge_images:
             msgs.append({'type': 'image', 'value': osp.join(self.img_root, img)})
         return msgs