diff --git a/vision_agent/agent/vision_agent_v2.py b/vision_agent/agent/vision_agent_v2.py index 340ba42e..c6d1e423 100644 --- a/vision_agent/agent/vision_agent_v2.py +++ b/vision_agent/agent/vision_agent_v2.py @@ -165,7 +165,7 @@ def write_and_exec_code( tool_info: str, exec: Execute, retrieved_ltm: str, - log_progress: Callable[..., str], + log_progress: Callable[[Dict[str, Any]], None], max_retry: int = 3, verbosity: int = 0, ) -> Tuple[bool, str, str, Dict[str, List[str]]]: @@ -179,7 +179,17 @@ def write_and_exec_code( success, result = exec.run_isolation(code) if verbosity == 2: _CONSOLE.print(Syntax(code, "python", theme="gruvbox-dark", line_numbers=True)) - log_progress(f"\tCode success: {success}\n\tResult: {str(result)}", code) + log_progress({ + "log": f"Code success: {success}", + }) + log_progress({ + "log": "Code:", + "code": code, + }) + log_progress({ + "log": "Result:", + "result": str(result), + }) _LOGGER.info(f"\tCode success: {success}, result: {str(result)}") working_memory: Dict[str, List[str]] = {} while not success and counter < max_retry: @@ -206,7 +216,14 @@ def write_and_exec_code( _CONSOLE.print( Syntax(code, "python", theme="gruvbox-dark", line_numbers=True) ) - log_progress(f"\tDebugging reflection: {reflection}\n\tResult: {result}") + log_progress({ + "log": "Debugging reflection:", + "reflection": reflection, + }) + log_progress({ + "log": "Result:", + "result": result, + }) _LOGGER.info(f"\tDebugging reflection: {reflection}, result: {result}") if success: @@ -227,7 +244,7 @@ def run_plan( exec: Execute, code: str, tool_recommender: Sim, - log_progress: Callable[..., str], + log_progress: Callable[[Dict[str, str]], None], long_term_memory: Optional[Sim] = None, verbosity: int = 0, ) -> Tuple[str, str, List[Dict[str, Any]], Dict[str, List[str]]]: @@ -238,10 +255,10 @@ def run_plan( working_memory: Dict[str, List[str]] = {} for task in active_plan: - log_progress( - f"""Going to run the following task(s) in sequence: -{tabulate(tabular_data=[task], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}""" - ) + log_progress({ + "log": f"Going to run the following task(s) in sequence:\n", + "task": task + }) _LOGGER.info( f""" {tabulate(tabular_data=[task], headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}""" @@ -250,7 +267,9 @@ def run_plan( tool_info = "\n".join([e["doc"] for e in tools]) if verbosity == 2: - log_progress(f"Tools retrieved: {[e['desc'] for e in tools]}") + log_progress({ + "log": f"Tools retrieved: {[e['desc'] for e in tools]}" + }) _LOGGER.info(f"Tools retrieved: {[e['desc'] for e in tools]}") if long_term_memory is not None: @@ -282,7 +301,13 @@ def run_plan( Syntax(code, "python", theme="gruvbox-dark", line_numbers=True) ) - log_progress(f"\tCode success: {success}\n\tResult: {str(result)}") + log_progress({ + "log": f"Code success: {success}", + }) + log_progress({ + "log": "Result:", + "result": str(result), + }) _LOGGER.info(f"\tCode success: {success} result: {str(result)}") task["success"] = success @@ -320,7 +345,7 @@ def __init__( tool_recommender: Optional[Sim] = None, long_term_memory: Optional[Sim] = None, verbosity: int = 0, - report_progress_callback: Optional[Callable[..., Any]] = None, + report_progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> None: self.planner = OpenAILLM(temperature=0.0, json_mode=True) self.coder = OpenAILLM(temperature=0.0) @@ -375,10 +400,10 @@ def chat_with_workflow( working_code = task["code"] user_req, plan = write_plan(chat, plan, TOOL_DESCRIPTIONS, self.planner) - self.log_progress( - f"""Plan: -{tabulate(tabular_data=plan, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}""" - ) + self.log_progress({ + "log": "Plans:", + "plan": plan, + }) _LOGGER.info( f"""Plan: {tabulate(tabular_data=plan, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}""" @@ -412,8 +437,10 @@ def chat_with_workflow( retries += 1 - self.log_progress("The Vision Agent V2 has concluded this chat.") - self.log_progress(f"Plan success: {success}") + self.log_progress({ + "log": f"The Vision Agent V2 has concluded this chat.\nSuccess: {success}", + "finished": True, + }) return { "code": working_code, @@ -423,7 +450,7 @@ def chat_with_workflow( "plan": plan, } - def log_progress(self, description: str, code: Optional[str] = "") -> None: + def log_progress(self, data: Dict[str, any]) -> None: if self.report_progress_callback is not None: - self.report_progress_callback(description, code) + self.report_progress_callback(data) pass diff --git a/vision_agent/agent/vision_agent_v3.py b/vision_agent/agent/vision_agent_v3.py index d8de28c6..6e5e735e 100644 --- a/vision_agent/agent/vision_agent_v3.py +++ b/vision_agent/agent/vision_agent_v3.py @@ -3,7 +3,7 @@ import logging import sys from pathlib import Path -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Optional, Union, cast, Callable from rich.console import Console from rich.syntax import Syntax @@ -114,6 +114,7 @@ def write_and_test_code( coder: LLM, tester: LLM, debugger: LLM, + log_progress: Callable[[Dict[str, str]], None], verbosity: int = 0, max_retries: int = 3, ) -> Dict[str, Any]: @@ -131,9 +132,17 @@ def write_and_test_code( success, result = _EXECUTE.run_isolation(f"{code}\n{test}") if verbosity == 2: _LOGGER.info("First code and tests:") + log_progress({ + "log": "First code and tests:", + "code": f"{code}\n{test}", + }) _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) + log_progress({ + "log": "First result:", + "result": result, + }) _LOGGER.info(f"First result: {result}") count = 0 @@ -156,6 +165,10 @@ def write_and_test_code( success, result = _EXECUTE.run_isolation(f"{code}\n{test}") if verbosity == 2: + log_progress({ + "log": f"Debug attempt {count + 1}, reflection:", + "reflection": fixed_code_and_test['reflections'], + }) _LOGGER.info( f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}" ) @@ -164,6 +177,10 @@ def write_and_test_code( f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True ) ) + log_progress({ + "log": "Debug result:", + "result": result, + }) _LOGGER.info(f"Debug result: {result}") count += 1 @@ -182,7 +199,10 @@ def write_and_test_code( def retrieve_tools( - plan: List[Dict[str, str]], tool_recommender: Sim, verbosity: int = 0 + plan: List[Dict[str, str]], + tool_recommender: Sim, + log_progress: Callable[[Dict[str, str]], None], + verbosity: int = 0 ) -> str: tool_info = [] tool_desc = [] @@ -191,6 +211,10 @@ def retrieve_tools( tool_info.extend([e["doc"] for e in tools]) tool_desc.extend([e["desc"] for e in tools]) if verbosity == 2: + log_progress({ + "log": f"Retrieved tools:", + "tools": tool_desc, + }) _LOGGER.info(f"Tools: {tool_desc}") tool_info_set = set(tool_info) return "\n\n".join(tool_info_set) @@ -206,6 +230,7 @@ def __init__( debugger: Optional[LLM] = None, tool_recommender: Optional[Sim] = None, verbosity: int = 0, + report_progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, ) -> None: self.planner = ( OpenAILLM(temperature=0.0, json_mode=True) if planner is None else planner @@ -223,6 +248,7 @@ def __init__( ) self.verbosity = verbosity self.max_retries = 3 + self.report_progress_callback = report_progress_callback def __call__( self, @@ -261,6 +287,10 @@ def chat_with_workflow( ) plan_i_str = "\n-".join([e["instructions"] for e in plan_i]) if self.verbosity == 1 or self.verbosity == 2: + self.log_progress({ + "log": f"Going to run the following plan(s) in sequence:\n", + "plan": plan_i, + }) _LOGGER.info( f""" {tabulate(tabular_data=plan_i, headers="keys", tablefmt="mixed_grid", maxcolwidths=_MAX_TABULATE_COL_WIDTH)}""" @@ -269,6 +299,7 @@ def chat_with_workflow( tool_info = retrieve_tools( plan_i, self.tool_recommender, + self.log_progress, self.verbosity, ) results = write_and_test_code( @@ -279,6 +310,7 @@ def chat_with_workflow( self.coder, self.tester, self.debugger, + self.log_progress, verbosity=self.verbosity, ) success = cast(bool, results["success"]) @@ -294,6 +326,11 @@ def chat_with_workflow( success = cast(bool, reflection["success"]) working_memory.append({"code": f"{code}\n{test}", "feedback": feedback}) + self.log_progress({ + "log": f"The Vision Agent V3 has concluded this chat.\nSuccess: {success}", + "finished": True, + }) + return { "code": code, "test": test, @@ -301,5 +338,7 @@ def chat_with_workflow( "working_memory": working_memory, } - def log_progress(self, description: str) -> None: + def log_progress(self, data: Dict[str, any]) -> None: + if self.report_progress_callback is not None: + self.report_progress_callback(data) pass