diff --git a/vision_agent/agent/vision_agent_prompts.py b/vision_agent/agent/vision_agent_prompts.py index bf9fac80..330cb5dc 100644 --- a/vision_agent/agent/vision_agent_prompts.py +++ b/vision_agent/agent/vision_agent_prompts.py @@ -1,5 +1,5 @@ VA_CODE = """ -**Role**: You are a helpful conversational agent that assists users with their requests by writing code to solve it. +**Role**: You are a helpful agent that assists users with writing code. **Taks**: As a conversational agent, you are required to understand the user's request and provide a helpful response. Use a Chain-of-Thought approach to break down the problem, create a plan, and then provide a response. Ensure that your response is clear, concise, and helpful. You can use an interactive Python (Jupyter Notebook) environment, executing code with . You are given access to an `artifacts` object which contains files shared between you and the user. `artifacts` will be automatically saved everytime you execute python code. @@ -56,7 +56,9 @@ AGENT: {"thoughts": "Two dogs are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect dogs and shown the output, do the results look good to you?", "let_user_respond": true} +""" +EXAMPLES_CODE1_EXTRA = """ USER: The the image only has one dog, can you fix this? [Artifacts loaded] @@ -82,6 +84,7 @@ [{'score': 0.99, 'label': 'dog', 'box': [0.1, 0.2, 0.3, 0.4]}] AGENT: {"thoughts": "One dog is detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to detect one dog and shown you the output, do the results look good to you?", "let_user_respond": true} + """ @@ -105,25 +108,24 @@ OBSERVATION: [Artifact code.py] -0|from vision_agent.tools import load_image, owl_v2, closest_box_distance, overlay_bounding_boxes, save_image +0|from vision_agent.tools import load_image, florence2_phrase_grounding, closest_box_distance, overlay_bounding_boxes, save_image 1|def count_workers_with_helmets(image_path: str, output_path: str): 2| image = load_image(image_path) -3| detections = owl_v2("worker, helmet", image) +3| detections = florence2_phrase_grounding("worker, helmet", image) 4| workers = [d for d in detections if d['label'] == 'worker'] 5| helmets = [d for d in detections if d['label'] == 'helmet'] 6| count = 0 6| for worker in workers: -7| person_box = worker['bbox'] -8| person_has_helmet = False -9| for helmet in helmets: -10| if closest_box_distance(worker['box'], helmet['box']) < 0.01: -11| person_has_helmet = True -12| break -13| if person_has_helmet: -14| count += 1 -15| overlay_bounding_boxes(image, detections) -16| save_image(output_path, image) -15| return count +7| person_has_helmet = False +8| for helmet in helmets: +9| if closest_box_distance(worker['box'], helmet['box']) < 0.01: +10| person_has_helmet = True +11| break +11| if person_has_helmet: +13| count += 1 +14| overlay_bounding_boxes(image, detections) +15| save_image(output_path, image) +16| return count [End of artifact] AGENT: {"thoughts": "I have generated the code to count the workers with helmets in the image, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "from code import count_workers_with_helmets\n print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n write_media_artifact(artifacts, 'workers_viz.png')", "let_user_respond": false} @@ -133,4 +135,36 @@ 2 AGENT: {"thoughts": "Two workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "I have written the code to count the workers wearing helmets in code.py and saved the visualization under 'workers_viz.png'.", "let_user_respond": true} + +USER: The detections are slightly off. Can you fine tune florence2 using these labels? "[{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}]" + +AGENT: {"thoughts": "Because the user has supplied me with labels I can call florence2_fine_tuning on their behalf to fine tune the model", "response": "I will fine tune florence2 with the labels you provided florence2_fine_tuning([{'image_path': 'image1.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}, {'image_path': 'image2.png': 'labels': ['worker', 'helmet'], 'bboxes': [[235, 118, 294, 241], [232, 118, 294, 128]]}], "phrase_grounding")", "let_user_respond": false} + +OBSERVATION: +[Florence2 fine tuning id: 23b3b022-5ebf-4798-9373-20ef36429abf] + + +AGENT: {"thoughts": "The model has finished fine tuning, I will now replace the original florence2_phrase_grounding call with the fine tuning id.", "response": "I will not update the code to use the fine tuned model. use_florence2_fine_tuning(artifacts, "code.py", "phrase_grounding", "23b3b022-5ebf-4798-9373-20ef36429abf")", "let_user_respond": false} + +OBSERVATION: +--- ++++ +@@ -1,7 +1,7 @@ + from vision_agent.tools import load_image, florence2_phrase_grounding, closest_box_distance, overlay_bounding_boxes, save_image + def count_workers_with_helmets(image_path: str, output_path: str): + image = load_image(image_path) +- detections = florence2_phrase_grounding("worker, helmet", image) ++ detections = florence2_phrase_grounding("worker, helmet", image, "23b3b022-5ebf-4798-9373-20ef36429abf") + workers = [d for d in detections if d['label'] == 'worker'] + helmets = [d for d in detections if d['label'] == 'helmet'] + count = 0 + + +AGENT: {"thoughts": "I have updated the code to use the fine tuned model, I must now run the code and print the output and write the visualization to the artifacts so I can see the result and the user can see the visaulization.", "response": "from code import count_workers_with_helmets\n print(count_workers_with_helmets('/path/to/images/workers.png', 'workers_viz.png'))\n write_media_artifact(artifacts, 'workers_viz.png')", "let_user_respond": false} + +OBSERVATION: +----- stdout ----- +3 + +AGENT: {"thoughts": "Three workers with helmets are detected, I will show this to the user and ask them if the result looks good.", "response": "I have updated the code using the fine tuned florence2_phrase_grounding model to count the workers wearing helmets in code.py and saved the visualization under 'workers_viz.png'.", "let_user_respond": true} """