From 0e35dc2e49338c4d869c8bfdd0db4c6ccbfdbbc3 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Fri, 26 Jul 2024 10:12:50 +0800 Subject: [PATCH 1/5] feat: new log with pick plan --- vision_agent/agent/vision_agent.py | 138 +++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 35 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index d8fc079a..f100ff25 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -176,13 +176,29 @@ def pick_plan( model: LMM, code_interpreter: CodeInterpreter, test_multi_plan: bool, + log_progress: Callable[[Dict[str, Any]], None], verbosity: int = 0, max_retries: int = 3, ) -> Tuple[Any, str, str]: if not test_multi_plan: k = list(plans.keys())[0] + log_progress( + { + "type": "logs", + "log_content": "Plans created", + "status": "completed", + "payload": plans[k], + } + ) return plans[k], tool_infos[k], "" + log_progress( + { + "type": "logs", + "log_content": "Picking plan", + "status": "started", + } + ) all_tool_info = tool_infos["all"] chat = copy.deepcopy(chat) if chat[-1]["role"] != "user": @@ -194,6 +210,14 @@ def pick_plan( ) code = extract_code(model(prompt)) + log_progress( + { + "type": "logs", + "log_content": "Running code", + "code": code, + "status": "running", + } + ) tool_output = code_interpreter.exec_isolation(DefaultImports.prepend_imports(code)) tool_output_str = "" if len(tool_output.logs.stdout) > 0: @@ -203,6 +227,18 @@ def pick_plan( _print_code("Initial code and tests:", code) _LOGGER.info(f"Initial code execution result:\n{tool_output.text()}") + log_progress( + { + "type": "logs", + "log_content": ( + "Code exeuction succeed" + if tool_output.success + else "Code execution failed" + ), + "payload": tool_output.to_json(), + "status": "completed" if tool_output.success else "failed", + } + ) # retry if the tool output is empty or code fails count = 0 while (not tool_output.success or tool_output_str == "") and count < max_retries: @@ -213,10 +249,33 @@ def pick_plan( code=code, error=tool_output.text() ), ) + log_progress( + { + "type": "logs", + "log_content": "Retry running code", + "code": code, + "status": "running", + } + ) code = extract_code(model(prompt)) tool_output = code_interpreter.exec_isolation( DefaultImports.prepend_imports(code) ) + log_progress( + { + "type": "logs", + "log_content": ( + "Code exeuction succeed" + if tool_output.success + else "Code execution failed" + ), + "code": code, + "payload": { + "result": tool_output.to_json(), + } + "status": "completed" if tool_output.success else "failed", + } + ) tool_output_str = "" if len(tool_output.logs.stdout) > 0: tool_output_str = tool_output.logs.stdout[0] @@ -246,14 +305,26 @@ def pick_plan( plan = best_plan["best_plan"] if plan in plans and plan in tool_infos: - return plans[plan], tool_infos[plan], tool_output_str + best_plans = plans[plan] + best_tool_infos = tool_infos[plan] else: if verbosity >= 1: _LOGGER.warning( f"Best plan {plan} not found in plans or tool_infos. Using the first plan and tool info." ) k = list(plans.keys())[0] - return plans[k], tool_infos[k], tool_output_str + best_plans = plans[k] + best_tool_infos = tool_infos[k] + + log_progress( + { + "type": "logs", + "log_content": "Picked best plan", + "status": "complete", + "payload": best_plans, + } + ) + return best_plans, best_tool_infos, tool_output_str @traceable @@ -323,7 +394,8 @@ def write_and_test_code( ) -> Dict[str, Any]: log_progress( { - "type": "code", + "type": "log", + "log_content": "Generating code", "status": "started", } ) @@ -341,10 +413,11 @@ def write_and_test_code( log_progress( { - "type": "code", + "type": "log", + "log_content": "Running code", "status": "running", + "code": DefaultImports.prepend_imports(code), "payload": { - "code": DefaultImports.prepend_imports(code), "test": test, }, } @@ -354,10 +427,11 @@ def write_and_test_code( ) log_progress( { - "type": "code", + "type": "log", + "log_content": "Code exeuction succeed" if result.success else "Code execution failed", "status": "completed" if result.success else "failed", + "code": DefaultImports.prepend_imports(code), "payload": { - "code": DefaultImports.prepend_imports(code), "test": test, "result": result.to_json(), }, @@ -507,15 +581,8 @@ def _print_code(title: str, code: str, test: Optional[str] = None) -> None: def retrieve_tools( plans: Dict[str, List[Dict[str, str]]], tool_recommender: Sim, - log_progress: Callable[[Dict[str, Any]], None], verbosity: int = 0, -) -> Dict[str, str]: - log_progress( - { - "type": "tools", - "status": "started", - } - ) +) -> Tuple[Dict[str, str], Dict[str, List[str, str]]]: tool_info = [] tool_desc = [] tool_lists: Dict[str, List[Dict[str, str]]] = {} @@ -526,7 +593,12 @@ def retrieve_tools( tool_info.extend([e["doc"] for e in tools]) tool_desc.extend([e["desc"] for e in tools]) tool_lists[k].extend( - {"description": e["desc"], "documentation": e["doc"]} for e in tools + { + "plan": task["instructions"] if index == 0 else "", + "tool": e["desc"].strip().split()[0], + "documentation": e["doc"], + } + for index, e in enumerate(tools) ) if verbosity == 2: @@ -540,14 +612,7 @@ def retrieve_tools( ) all_tools = "\n\n".join(set(tool_info)) tool_lists_unique["all"] = all_tools - log_progress( - { - "type": "tools", - "status": "completed", - "payload": tool_lists[list(plans.keys())[0]], - } - ) - return tool_lists_unique + return tool_lists_unique, tool_lists class VisionAgent(Agent): @@ -704,7 +769,8 @@ def chat_with_workflow( self.log_progress( { - "type": "plans", + "type": "logs", + "log_content": "Creating plans", "status": "started", } ) @@ -715,27 +781,29 @@ def chat_with_workflow( self.planner, ) - self.log_progress( - { - "type": "plans", - "status": "completed", - "payload": plans[list(plans.keys())[0]], - } - ) - if self.verbosity >= 1 and test_multi_plan: for p in plans: _LOGGER.info( f"\n{tabulate(tabular_data=plans[p], headers='keys', tablefmt='mixed_grid', maxcolwidths=_MAX_TABULATE_COL_WIDTH)}" ) - tool_infos = retrieve_tools( + tool_infos, tool_lists = retrieve_tools( plans, self.tool_recommender, self.log_progress, self.verbosity, ) + if test_multi_plan: + self.log_progress( + { + "type": "logs", + "log_content": "Creating plans", + "status": "completed", + "payload": tool_lists, + } + ) + best_plan, best_tool_info, tool_output_str = pick_plan( int_chat, plans, @@ -777,8 +845,8 @@ def chat_with_workflow( { "type": "final_code", "status": "completed" if success else "failed", + "code": DefaultImports.prepend_imports(code), "payload": { - "code": DefaultImports.prepend_imports(code), "test": test, "result": execution_result.to_json(), }, From 648e6676d338cd3440d807fd5e01313315a7bf9b Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Fri, 26 Jul 2024 11:30:14 +0800 Subject: [PATCH 2/5] fix lint --- vision_agent/agent/vision_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index f100ff25..26f4a235 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -272,7 +272,7 @@ def pick_plan( "code": code, "payload": { "result": tool_output.to_json(), - } + }, "status": "completed" if tool_output.success else "failed", } ) From de1b11ec8beef151660544df10306e1d80bfb58c Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Fri, 26 Jul 2024 11:56:29 +0800 Subject: [PATCH 3/5] format --- vision_agent/agent/vision_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 26f4a235..b25ba6c5 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -428,7 +428,9 @@ def write_and_test_code( log_progress( { "type": "log", - "log_content": "Code exeuction succeed" if result.success else "Code execution failed", + "log_content": ( + "Code exeuction succeed" if result.success else "Code execution failed" + ), "status": "completed" if result.success else "failed", "code": DefaultImports.prepend_imports(code), "payload": { From 5ee4453ca06d93cba95fe5cd00e18ef3388f7084 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Fri, 26 Jul 2024 12:24:08 +0800 Subject: [PATCH 4/5] fix lint --- vision_agent/agent/vision_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index b25ba6c5..a8133173 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -792,7 +792,6 @@ def chat_with_workflow( tool_infos, tool_lists = retrieve_tools( plans, self.tool_recommender, - self.log_progress, self.verbosity, ) From a7cf080e59959e55e26bacf7ec836e283059a56d Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Fri, 26 Jul 2024 14:28:33 +0800 Subject: [PATCH 5/5] address comment --- vision_agent/agent/vision_agent.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index a8133173..d525962e 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -195,7 +195,7 @@ def pick_plan( log_progress( { "type": "logs", - "log_content": "Picking plan", + "log_content": "Generating code to pick best plan", "status": "started", } ) @@ -213,7 +213,7 @@ def pick_plan( log_progress( { "type": "logs", - "log_content": "Running code", + "log_content": "Executing code to test plan", "code": code, "status": "running", } @@ -231,7 +231,7 @@ def pick_plan( { "type": "logs", "log_content": ( - "Code exeuction succeed" + "Code execution succeed" if tool_output.success else "Code execution failed" ), @@ -265,7 +265,7 @@ def pick_plan( { "type": "logs", "log_content": ( - "Code exeuction succeed" + "Code execution succeed" if tool_output.success else "Code execution failed" ), @@ -429,7 +429,7 @@ def write_and_test_code( { "type": "log", "log_content": ( - "Code exeuction succeed" if result.success else "Code execution failed" + "Code execution succeed" if result.success else "Code execution failed" ), "status": "completed" if result.success else "failed", "code": DefaultImports.prepend_imports(code), @@ -584,7 +584,7 @@ def retrieve_tools( plans: Dict[str, List[Dict[str, str]]], tool_recommender: Sim, verbosity: int = 0, -) -> Tuple[Dict[str, str], Dict[str, List[str, str]]]: +) -> Tuple[Dict[str, str], Dict[str, List[Dict[str, str]]]]: tool_info = [] tool_desc = [] tool_lists: Dict[str, List[Dict[str, str]]] = {}