updating prompts with fine tuning examples

vision_agent/agent/vision_agent_prompts.py

@@ -1,5 +1,5 @@
 VA_CODE = """
-**Role**: You are a helpful conversational agent that assists users with their requests by writing code to solve it.
+**Role**: You are a helpful agent that assists users with writing code.
 
 **Taks**: As a conversational agent, you are required to understand the user's request and provide a helpful response. Use a Chain-of-Thought approach to break down the problem, create a plan, and then provide a response. Ensure that your response is clear, concise, and helpful. You can use an interactive Python (Jupyter Notebook) environment, executing code with <execution_python>. You are given access to an `artifacts` object which contains files shared between you and the user. `artifacts` will be automatically saved everytime you execute python code.
 
@@ -56,7 +56,9 @@
 
 
 AGENT: {"thoughts": "Two dogs are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect dogs and shown the output, do the results look good to you?", "let_user_respond": true}
+"""
 
+EXAMPLES_CODE1_EXTRA = """
 USER: The the image only has one dog, can you fix this?
 
 [Artifacts loaded]
@@ -82,6 +84,7 @@
 [{'score': 0.99, 'label': 'dog', 'box': [0.1, 0.2, 0.3, 0.4]}]
 
 AGENT: {"thoughts": "One dog is detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect one dog and shown you the output, do the results look good to you?", "let_user_respond": true}
+
 """
 
 
@@ -105,25 +108,24 @@
 
 OBSERVATION:
 [Artifact code.py]
-0|from vision_agent.tools import load_image, owl_v2, closest_box_distance, overlay_bounding_boxes, save_image
+0|from vision_agent.tools import load_image, florence2_phrase_grounding, closest_box_distance, overlay_bounding_boxes, save_image
 1|def count_workers_with_helmets(image_path: str, output_path: str):
 2| image = load_image(image_path)
-3| detections = owl_v2("worker, helmet", image)
+3| detections = florence2_phrase_grounding("worker, helmet", image)
 4| workers = [d for d in detections if d['label'] == 'worker']
 5| helmets = [d for d in detections if d['label'] == 'helmet']
 6| count = 0
 6| for worker in workers:
-7| person_box = worker['bbox']
-8| person_has_helmet = False
-9| for helmet in helmets:
-10| if closest_box_distance(worker['box'], helmet['box']) < 0.01:
-11| person_has_helmet = True
-12| break
-13| if person_has_helmet:
-14| count += 1
-15| overlay_bounding_boxes(image, detections)
-16| save_image(output_path, image)
-15| return count
+7| person_has_helmet = False
+8| for helmet in helmets:
+9| if closest_box_distance(worker['box'], helmet['box']) < 0.01:
+10| person_has_helmet = True
+11| break
+11| if person_has_helmet:
+13| count += 1
+14| overlay_bounding_boxes(image, detections)
+15| save_image(output_path, image)
+16| return count
 [End of artifact]
 
 AGENT: {"thoughts": "I have generated the code to count the workers with helmets in the image, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n write_media_artifact(artifacts, 'workers_viz.png')</execute_python>", "let_user_respond": false}
@@ -133,4 +135,36 @@
 2
 
 AGENT: {"thoughts": "Two workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to count the workers wearing helmets in code.py and saved the visualization under 'workers_viz.png'.", "let_user_respond": true}
+
+USER: The detections are slightly off. Can you fine tune florence2 using these labels? "[{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}]"
+
+AGENT: {"thoughts": "Because the user has supplied me with labels I can call florence2_fine_tuning on their behalf to fine tune the model", "response": "I will fine tune florence2 with the labels you provided <execute_python>florence2_fine_tuning([{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}], "phrase_grounding")</execute_python>", "let_user_respond": false}
+
+OBSERVATION:
+[Florence2 fine tuning id: 23b3b022-5ebf-4798-9373-20ef36429abf]
+
+
+AGENT: {"thoughts": "The model has finished fine tuning, I will now replace the original florence2_phrase_grounding call with the fine tuning id.", "response": "I will not update the code to use the fine tuned model. <execute_python>use_florence2_fine_tuning(artifacts, "code.py", "phrase_grounding", "23b3b022-5ebf-4798-9373-20ef36429abf")</execute_python>", "let_user_respond": false}
+
+OBSERVATION:
+--- 
++++ 
+@@ -1,7 +1,7 @@
+ from vision_agent.tools import load_image, florence2_phrase_grounding, closest_box_distance, overlay_bounding_boxes, save_image
+ def count_workers_with_helmets(image_path: str, output_path: str):
+ image = load_image(image_path)
+- detections = florence2_phrase_grounding("worker, helmet", image)
++ detections = florence2_phrase_grounding("worker, helmet", image, "23b3b022-5ebf-4798-9373-20ef36429abf")
+ workers = [d for d in detections if d['label'] == 'worker']
+ helmets = [d for d in detections if d['label'] == 'helmet']
+ count = 0
+
+
+AGENT: {"thoughts": "I have updated the code to use the fine tuned model, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "<execute_python>from code import count_workers_with_helmets\n print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n write_media_artifact(artifacts, 'workers_viz.png')</execute_python>", "let_user_respond": false}
+
+OBSERVATION:
+----- stdout -----
+3
+
+AGENT: {"thoughts": "Three workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "I have updated the code using the fine tuned florence2_phrase_grounding model to count the workers wearing helmets in code.py and saved the visualization under 'workers_viz.png'.", "let_user_respond": true}
 """