Bug fixes for base config

Author: mschoenb97

GLIP

@@ -42,14 +42,14 @@ gpt3:                                               # GPT-3 configuration
     n_votes: 1                                      # Number of tries to use for GPT-3. Use with temperature > 0
     qa_prompt: ./prompts/gpt3/gpt3_qa.txt
     temperature: 0.                                 # Temperature for GPT-3. Almost deterministic if 0
-    model: text-davinci-003                         # Can replace with code-davinci-002 (which is free for now) but will have worse performance as it's meant for code
+    model: text-davinci-003                         # See openai.Model.list() for available models
 
 codex:
     temperature: 0.                                 # Temperature for Codex. (Almost) deterministic if 0
     best_of: 1                                      # Number of tries to choose from. Use when temperature > 0
     max_tokens: 512                                 # Maximum number of tokens to generate for Codex
-    prompt: ./prompts/api.prompt                    # Codex prompt file, which defines the API. If you use a Chat-based model (3.5/4) try ./prompts/chatapi.prompt (doesn't support video for now due to token limits)
-    model: code-davinci-002                         # Codex model to use. [code-davinci-002, gpt-3.5-turbo, gpt-4]
+    prompt: ./prompts/chatapi.prompt                # Codex prompt file, which defines the API. (doesn't support video for now due to token limits)
+    model: gpt-3.5-turbo                            # Codex model to use. [gpt-3.5-turbo, gpt-4]. See openai.Model.list()
 
 # Saving and loading parameters
 save: True                                          # Save the results to a file
@@ -60,6 +60,7 @@ clear_cache: False                                  # Clear stored cache
 use_cached_codex: False                             # Use previously-computed Codex results
 cached_codex_path: ''                               # Path to the csv results file from which to load Codex results
 log_every: 20                                       # Log accuracy every n batches
+wandb: False                                        # Use Weights and Biases
 
 blip_half_precision: True                           # Use 8bit (Faster but slightly less accurate) for BLIP if True
 blip_v2_model_type: blip2-flan-t5-xxl               # Which model to use for BLIP-2

configs/base_config.yaml

@@ -5,4 +5,27 @@ dataset:
     data_path: 'data'
 blip_v2_model_type: blip2-flan-t5-xxl  # Change to blip2-flan-t5-xl for smaller GPUs
 blip_half_precision: True
-# Add more changes here, following the same format as base_config.yaml
+# Add more changes here, following the same format as base_config.yaml
+
+load_models:                                        # Which pretrained models to load
+    maskrcnn: False
+    clip: False
+    glip: False
+    owlvit: False
+    tcl: False
+    gpt3_qa: True
+    gpt3_general: True
+    depth: False
+    blip: False
+    saliency: False
+    xvlm: False
+    codex: True
+    object_detector: False
+
+# wandb: False
+# 
+
+# codex:
+#     model: gpt-3.5-turbo
+
+# execute_code: True

configs/my_config.yaml

@@ -1,3 +1,2 @@
-query,answer,image_name
-What color do you get if you combine the colors of the viper and the flower?,purple,viper_flower.png
-Tell me about the competition between the two skyscrapers in the image.,,skyscrapers.png
+index,sample_id,possible_answers,query_type,info_to_prompt,query,answer,image_name,img,
+0,0,How many cookies are there?,seven,,How many cookies are there?,seven,cookies.png,cookies.png

data/queries.csv

@@ -69,14 +69,17 @@ def get_video(self, video_path):
         return video
 
     def __getitem__(self, index):
-        query = self.df.iloc[index]["query"]
-        answer = self.df.iloc[index]["answer"]
+
+        out_dict = self.df.iloc[index].to_dict()
+
         sample_path = self.get_sample_path(index)
 
         # Load and transform image
         image = self.get_image(sample_path) if self.input_type == "image" else self.get_video(sample_path)
 
-        out_dict = {"query": query, "answer": answer, "image": image, 'index': index}
+        out_dict["image"] = image
+        out_dict["index"] = index
+        
         return out_dict
 
     def __len__(self):

datasets/dataset.py

@@ -141,6 +141,7 @@ def find(self, object_name: str) -> list[ImagePatch]:
             if object_name == 'person':
                 object_name = 'people'  # GLIP does better at people than person
 
+            # all_object_coordinates = self.forward('object_detector', self.cropped_image)
             all_object_coordinates = self.forward('glip', self.cropped_image, object_name)
         if len(all_object_coordinates) == 0:
             return []
@@ -155,6 +156,7 @@ def find(self, object_name: str) -> list[ImagePatch]:
             #     mask = all_areas == all_areas.max()  # At least return one element
             all_object_coordinates = all_object_coordinates[mask]
 
+
         return [self.crop(*coordinates) for coordinates in all_object_coordinates]
 
     def exists(self, object_name) -> bool:

image_patch.py

@@ -4,6 +4,8 @@
 import pathlib
 from functools import partial
 import warnings
+import traceback
+
 
 import pandas as pd
 import torch.multiprocessing as mp
@@ -79,6 +81,8 @@ def run_program(parameters, queues_in_, input_type_, retrying=False):
             # Functions to be used
             llm_query_partial, bool_to_yesno, distance, best_image_match)
     except Exception as e:
+        # print full traceback
+        traceback.print_exc()
         if retrying:
             return None, code
         print(f'Sample {sample_id} failed with error: {e}. Next you will see an "expected an indented block" error. ')
@@ -110,6 +114,7 @@ def main():
     batch_size = config.dataset.batch_size
     num_processes = min(batch_size, 50)
 
+
     if config.multiprocessing:
         queue_results_main = manager.Queue()
         queues_results = [manager.Queue() for _ in range(batch_size)]
@@ -126,11 +131,11 @@ def main():
         import wandb
         wandb.init(project="viper", config=OmegaConf.to_container(config))
         # log the prompt file
-        wandb.save(config.prompt)
+        wandb.save(config.codex.prompt)
 
     dataset = MyDataset(**config.dataset)
 
-    with open(config.prompt) as f:
+    with open(config.codex.prompt) as f:
         base_prompt = f.read().strip()
 
     codes_all = None
@@ -155,12 +160,15 @@ def main():
             if config.multiprocessing else open(os.devnull, "w") as pool:
         try:
             n_batches = len(dataloader)
+
             for i, batch in tqdm(enumerate(dataloader), total=n_batches):
 
                 # Combine all querys and get Codex predictions for them
                 # TODO compute Codex for next batch as current batch is being processed
+
                 if not config.use_cached_codex:
-                    codes = codex(prompt=batch['info_to_prompt'], base_prompt=base_prompt)
+                    # codes = codex(prompt=batch['info_to_qprompt'], base_prompt=base_prompt)
+                    codes = codex(prompt=batch['query'], base_prompt=base_prompt)
 
                 else:
                     codes = codes_all[i * batch_size:(i + 1) * batch_size]  # If cache
@@ -171,13 +179,13 @@ def main():
                         # Otherwise, we would create a new model for every process
                         results = []
                         for c, sample_id, img, possible_answers, query in \
-                                zip(codes, batch['sample_id'], batch['img'], batch['possible_answers'], batch['query']):
+                                zip(codes, batch['sample_id'], batch['image'], batch['possible_answers'], batch['query']):
                             result = run_program([c, sample_id, img, possible_answers, query], queues_in, input_type)
                             results.append(result)
                     else:
                         results = list(pool.imap(partial(
                             run_program, queues_in_=queues_in, input_type_=input_type),
-                            zip(codes, batch['sample_id'], batch['img'], batch['possible_answers'], batch['query'])))
+                            zip(codes, batch['sample_id'], batch['image'], batch['possible_answers'], batch['query'])))
                 else:
                     results = [(None, c) for c in codes]
                     warnings.warn("Not executing code! This is only generating the code. We set the flag "
@@ -192,7 +200,7 @@ def main():
                 all_possible_answers += batch['possible_answers']
                 all_query_types += batch['query_type']
                 all_querys += batch['query']
-                all_img_paths += [dataset.get_img_path(idx) for idx in batch['index']]
+                all_img_paths += [dataset.get_sample_path(idx) for idx in batch['index']]
                 if i % config.log_every == 0:
                     try:
                         accuracy = datasets.accuracy(all_results, all_answers, all_possible_answers, all_query_types)
@@ -201,6 +209,8 @@ def main():
                         console.print(f'Error computing accuracy: {e}')
 
         except Exception as e:
+            # print full stack trace
+            traceback.print_exc()
             console.print(f'Exception: {e}')
             console.print("Completing logging and exiting...")
 

main_batch.py

@@ -940,6 +940,7 @@ def codex_helper(extended_prompt):
 #         if len(resp) == 1:
 #             resp = resp[0]
     else:
+        warnings.warn('OpenAI Codex is deprecated. Please use GPT-4 or GPT-3.5-turbo.')
         response = openai.Completion.create(
             model="code-davinci-002",
             temperature=config.codex.temperature,