From 3fa91ee2f368196989d0438e1d03f72935874e61 Mon Sep 17 00:00:00 2001 From: Dillon Laird Date: Fri, 19 Jul 2024 14:28:30 -0700 Subject: [PATCH] multi plan --- vision_agent/agent/vision_agent_coder.py | 25 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/vision_agent/agent/vision_agent_coder.py b/vision_agent/agent/vision_agent_coder.py index 29113074..341f0eb3 100644 --- a/vision_agent/agent/vision_agent_coder.py +++ b/vision_agent/agent/vision_agent_coder.py @@ -584,6 +584,7 @@ def __call__( def chat_with_workflow( self, chat: List[Message], + test_multi_plan: bool = True, display_visualization: bool = False, ) -> Dict[str, Any]: """Chat with VisionAgentCoder and return intermediate information regarding the @@ -595,6 +596,9 @@ def chat_with_workflow( [{"role": "user", "content": "describe your task here..."}] or if it contains media files, it should be in the format of: [{"role": "user", "content": "describe your task here...", "media": ["image1.jpg", "image2.jpg"]}] + test_multi_plan (bool): If True, it will test tools for multiple plans and + pick the best one based off of the tool results. If False, it will go + with the first plan. display_visualization (bool): If True, it opens a new window locally to show the image(s) created by visualization code (if there is any). @@ -666,14 +670,19 @@ def chat_with_workflow( self.log_progress, self.verbosity, ) - best_plan, tool_output_str = pick_plan( - int_chat, - plans, - tool_infos["all"], - self.coder, - code_interpreter, - verbosity=self.verbosity, - ) + + if test_multi_plan: + best_plan, tool_output_str = pick_plan( + int_chat, + plans, + tool_infos["all"], + self.coder, + code_interpreter, + verbosity=self.verbosity, + ) + else: + best_plan = list(plans.keys())[0] + tool_output_str = "" if best_plan in plans and best_plan in tool_infos: plan_i = plans[best_plan]