From 6ff57ed05ce762fc99a6b8d7cd7561ab139475d2 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Mon, 27 May 2024 17:16:28 +0800 Subject: [PATCH 1/5] feat: add log for debug code and test --- vision_agent/agent/vision_agent_v3.py | 40 ++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/vision_agent/agent/vision_agent_v3.py b/vision_agent/agent/vision_agent_v3.py index 6d194d9c..27f0cdd6 100644 --- a/vision_agent/agent/vision_agent_v3.py +++ b/vision_agent/agent/vision_agent_v3.py @@ -186,6 +186,18 @@ def write_and_test_code( "result": fixed_code_and_test["reflections"], } ) + log_progress( + { + "log": "Debug code:", + "code": code, + } + ) + log_progress( + { + "log": "Debug test:", + "code": test, + } + ) _LOGGER.info( f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}" ) @@ -208,6 +220,32 @@ def write_and_test_code( _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) + + if count == max_retries: + log_progress( + { + "log": f"{max_retries} max retries reached.", + } + ) + + log_progress( + { + "log": "Final code:", + "code": code, + } + ) + log_progress( + { + "log": "Final test:", + "code": test, + } + ) + log_progress( + { + "log": "Final result:", + "result": result, + } + ) _LOGGER.info(f"Final Result: {result}") return { @@ -367,7 +405,7 @@ def chat_with_workflow( self.log_progress( { - "log": f"The Vision Agent V3 has concluded this chat.\nSuccess: {success}", + "log": f"The Vision Agent V3 has concluded this chat.", "finished": True, } ) From 13bbdcb98fbfc7e3ef371be5112c3f25ae5baafd Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Wed, 29 May 2024 20:08:53 +0800 Subject: [PATCH 2/5] feat: change the format --- vision_agent/agent/vision_agent_v3.py | 91 +++++++++------------------ 1 file changed, 28 insertions(+), 63 deletions(-) diff --git a/vision_agent/agent/vision_agent_v3.py b/vision_agent/agent/vision_agent_v3.py index faeb3890..787e0a2c 100644 --- a/vision_agent/agent/vision_agent_v3.py +++ b/vision_agent/agent/vision_agent_v3.py @@ -140,29 +140,23 @@ def write_and_test_code( _LOGGER.info("Initial code and tests:") log_progress( { - "log": "Code:", - "code": code, - } - ) - log_progress( - { - "log": "Test:", - "code": test, + "type": "initial_attempt", + "payload": { + "code": code, + "test": test, + "result": result, + "success": success, + }, } ) _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) - log_progress( - { - "log": "Result:", - "result": result, - } - ) _LOGGER.info(f"Initial result: {result}") count = 0 new_working_memory = [] + while not success and count < max_retries: fixed_code_and_test = extract_json( debugger( @@ -183,20 +177,15 @@ def write_and_test_code( if verbosity == 2: log_progress( { - "log": f"Debug attempt {count + 1}, reflection:", - "result": fixed_code_and_test["reflections"], - } - ) - log_progress( - { - "log": "Debug code:", - "code": code, - } - ) - log_progress( - { - "log": "Debug test:", - "code": test, + "type": "debug_attempt", + "payload": { + "count": count + 1, + "code": code, + "test": test, + "result": result, + "success": success, + "reflection": fixed_code_and_test["reflections"], + }, } ) _LOGGER.info( @@ -207,12 +196,6 @@ def write_and_test_code( f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True ) ) - log_progress( - { - "log": "Debug result:", - "result": result, - } - ) _LOGGER.info(f"Debug result: {result}") count += 1 @@ -221,30 +204,13 @@ def write_and_test_code( _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) - - if count == max_retries: - log_progress( - { - "log": f"{max_retries} max retries reached.", - } - ) - - log_progress( - { - "log": "Final code:", - "code": code, - } - ) - log_progress( - { - "log": "Final test:", - "code": test, - } - ) log_progress( { - "log": "Final result:", - "result": result, + "type": "final_result", + "payload": { + "success": success, + "reach_max_retries": count == max_retries, + }, } ) _LOGGER.info(f"Final Result: {result}") @@ -273,8 +239,8 @@ def retrieve_tools( if verbosity == 2: log_progress( { - "log": "Retrieved tools:", - "tools": tool_desc, + "type": "tools", + "payload": tool_desc, } ) _LOGGER.info(f"Tools: {tool_desc}") @@ -354,8 +320,8 @@ def chat_with_workflow( if self.verbosity >= 1: self.log_progress( { - "log": "Going to run the following plan(s) in sequence:\n", - "plan": plan_i, + "type": "plans", + "payload": plan_i, } ) @@ -400,7 +366,7 @@ def chat_with_workflow( if self.verbosity > 0: self.log_progress( { - "log": "Reflection:", + "type": "final_reflection", "reflection": reflection, } ) @@ -413,8 +379,7 @@ def chat_with_workflow( self.log_progress( { - "log": f"The Vision Agent V3 has concluded this chat.", - "finished": True, + "type": "agent_conclusion", } ) From 520f3c3c9680e541e1dcda4be17c2d9539ef2691 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Thu, 30 May 2024 16:06:10 +0800 Subject: [PATCH 3/5] address feedback --- vision_agent/agent/vision_agent_v3.py | 149 ++++++++++++++++++-------- 1 file changed, 103 insertions(+), 46 deletions(-) diff --git a/vision_agent/agent/vision_agent_v3.py b/vision_agent/agent/vision_agent_v3.py index 787e0a2c..c65b6c45 100644 --- a/vision_agent/agent/vision_agent_v3.py +++ b/vision_agent/agent/vision_agent_v3.py @@ -120,6 +120,12 @@ def write_and_test_code( max_retries: int = 3, input_media: Optional[Union[str, Path]] = None, ) -> Dict[str, Any]: + log_progress( + { + "type": "code_execution", + "status": "started", + } + ) code = extract_code( coder(CODE.format(docstring=tool_info, question=task, feedback=working_memory)) ) @@ -135,20 +141,30 @@ def write_and_test_code( ) ) + log_progress( + { + "type": "code_execution", + "status": "code_generated", + "payload": { + "code": code, + "test": test, + }, + } + ) success, result = _EXECUTE.run_isolation(f"{code}\n{test}") + log_progress( + { + "type": "code_execution", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": result, + }, + } + ) if verbosity == 2: _LOGGER.info("Initial code and tests:") - log_progress( - { - "type": "initial_attempt", - "payload": { - "code": code, - "test": test, - "result": result, - "success": success, - }, - } - ) _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) @@ -158,6 +174,12 @@ def write_and_test_code( new_working_memory = [] while not success and count < max_retries: + log_progress( + { + "type": "code_execution", + "status": "started", + } + ) fixed_code_and_test = extract_json( debugger( FIX_BUG.format( @@ -169,25 +191,34 @@ def write_and_test_code( code = extract_code(fixed_code_and_test["code"]) if fixed_code_and_test["test"].strip() != "": test = extract_code(fixed_code_and_test["test"]) + log_progress( + { + "type": "code_execution", + "status": "code_generated", + "payload": { + "code": code, + "test": test, + }, + } + ) new_working_memory.append( {"code": f"{code}\n{test}", "feedback": fixed_code_and_test["reflections"]} ) success, result = _EXECUTE.run_isolation(f"{code}\n{test}") + + log_progress( + { + "type": "code_execution", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": result, + }, + } + ) if verbosity == 2: - log_progress( - { - "type": "debug_attempt", - "payload": { - "count": count + 1, - "code": code, - "test": test, - "result": result, - "success": success, - "reflection": fixed_code_and_test["reflections"], - }, - } - ) _LOGGER.info( f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}" ) @@ -204,17 +235,20 @@ def write_and_test_code( _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) - log_progress( - { - "type": "final_result", - "payload": { - "success": success, - "reach_max_retries": count == max_retries, - }, - } - ) _LOGGER.info(f"Final Result: {result}") + log_progress( + { + "type": "final_code", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": result, + }, + } + ) + return { "code": code, "test": test, @@ -230,19 +264,27 @@ def retrieve_tools( log_progress: Callable[[Dict[str, Any]], None], verbosity: int = 0, ) -> str: + log_progress( + { + "type": "tools", + "status": "started", + } + ) tool_info = [] tool_desc = [] for task in plan: tools = tool_recommender.top_k(task["instructions"], k=2, thresh=0.3) tool_info.extend([e["doc"] for e in tools]) tool_desc.extend([e["desc"] for e in tools]) + + log_progress( + { + "type": "tools", + "status": "completed", + "payload": tool_desc, + } + ) if verbosity == 2: - log_progress( - { - "type": "tools", - "payload": tool_desc, - } - ) _LOGGER.info(f"Tools: {tool_desc}") tool_info_set = set(tool_info) return "\n\n".join(tool_info_set) @@ -313,6 +355,12 @@ def chat_with_workflow( retries = 0 while not success and retries < self.max_retries: + self.log_progress( + { + "type": "plans", + "status": "started", + } + ) plan_i = write_plan( chat, TOOL_DESCRIPTIONS, format_memory(working_memory), self.planner ) @@ -321,6 +369,7 @@ def chat_with_workflow( self.log_progress( { "type": "plans", + "status": "completed", "payload": plan_i, } ) @@ -355,6 +404,12 @@ def chat_with_workflow( plan.append({"code": code, "test": test, "plan": plan_i}) if self_reflection: + self.log_progress( + { + "type": "self_reflection", + "status": "started", + } + ) reflection = reflect( chat, FULL_TASK.format( @@ -364,22 +419,24 @@ def chat_with_workflow( self.planner, ) if self.verbosity > 0: - self.log_progress( - { - "type": "final_reflection", - "reflection": reflection, - } - ) _LOGGER.info(f"Reflection: {reflection}") feedback = cast(str, reflection["feedback"]) success = cast(bool, reflection["success"]) + self.log_progress( + { + "type": "self_reflection", + "status": "completed" if success else "failed", + "payload": reflection, + } + ) working_memory.append({"code": f"{code}\n{test}", "feedback": feedback}) retries += 1 self.log_progress( { - "type": "agent_conclusion", + "type": "chat_with_workflow", + "status": "completed" if success else "failed", } ) From 0bb50fa04e50572c60d962a4cbdc348983e14da7 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Thu, 30 May 2024 17:27:45 +0800 Subject: [PATCH 4/5] address comment --- vision_agent/agent/vision_agent_v3.py | 35 +++++++++++---------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/vision_agent/agent/vision_agent_v3.py b/vision_agent/agent/vision_agent_v3.py index c65b6c45..c9a33491 100644 --- a/vision_agent/agent/vision_agent_v3.py +++ b/vision_agent/agent/vision_agent_v3.py @@ -122,7 +122,7 @@ def write_and_test_code( ) -> Dict[str, Any]: log_progress( { - "type": "code_execution", + "type": "code", "status": "started", } ) @@ -143,8 +143,8 @@ def write_and_test_code( log_progress( { - "type": "code_execution", - "status": "code_generated", + "type": "code", + "status": "running", "payload": { "code": code, "test": test, @@ -154,7 +154,7 @@ def write_and_test_code( success, result = _EXECUTE.run_isolation(f"{code}\n{test}") log_progress( { - "type": "code_execution", + "type": "code", "status": "completed" if success else "failed", "payload": { "code": code, @@ -176,7 +176,7 @@ def write_and_test_code( while not success and count < max_retries: log_progress( { - "type": "code_execution", + "type": "code", "status": "started", } ) @@ -193,8 +193,8 @@ def write_and_test_code( test = extract_code(fixed_code_and_test["test"]) log_progress( { - "type": "code_execution", - "status": "code_generated", + "type": "code", + "status": "running", "payload": { "code": code, "test": test, @@ -209,7 +209,7 @@ def write_and_test_code( log_progress( { - "type": "code_execution", + "type": "code", "status": "completed" if success else "failed", "payload": { "code": code, @@ -237,18 +237,6 @@ def write_and_test_code( ) _LOGGER.info(f"Final Result: {result}") - log_progress( - { - "type": "final_code", - "status": "completed" if success else "failed", - "payload": { - "code": code, - "test": test, - "result": result, - }, - } - ) - return { "code": code, "test": test, @@ -435,8 +423,13 @@ def chat_with_workflow( self.log_progress( { - "type": "chat_with_workflow", + "type": "final_code", "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": results["test_result"], + }, } ) From dd6a239ee87d4c7228ba0c02f53d3bb452639279 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Thu, 30 May 2024 18:07:53 +0800 Subject: [PATCH 5/5] move log to v1 --- vision_agent/agent/vision_agent.py | 151 +++++++++++++++++++---------- 1 file changed, 101 insertions(+), 50 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 1bba6446..1fef0852 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -126,6 +126,12 @@ def write_and_test_code( max_retries: int = 3, input_media: Optional[Union[str, Path]] = None, ) -> Dict[str, Any]: + log_progress( + { + "type": "code", + "status": "started", + } + ) code = extract_code( coder(CODE.format(docstring=tool_info, question=task, feedback=working_memory)) ) @@ -141,35 +147,44 @@ def write_and_test_code( ) ) + log_progress( + { + "type": "code", + "status": "running", + "payload": { + "code": code, + "test": test, + }, + } + ) success, result = _EXECUTE.run_isolation(f"{_DEFAULT_IMPORT}\n{code}\n{test}") + log_progress( + { + "type": "code", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": result, + }, + } + ) if verbosity == 2: _LOGGER.info("Initial code and tests:") - log_progress( - { - "log": "Code:", - "code": code, - } - ) - log_progress( - { - "log": "Test:", - "code": test, - } - ) _CONSOLE.print( Syntax(f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True) ) - log_progress( - { - "log": "Result:", - "result": result, - } - ) _LOGGER.info(f"Initial result: {result}") count = 0 new_working_memory = [] while not success and count < max_retries: + log_progress( + { + "type": "code", + "status": "started", + } + ) fixed_code_and_test = extract_json( debugger( FIX_BUG.format( @@ -181,18 +196,33 @@ def write_and_test_code( code = extract_code(fixed_code_and_test["code"]) if fixed_code_and_test["test"].strip() != "": test = extract_code(fixed_code_and_test["test"]) + log_progress( + { + "type": "code", + "status": "running", + "payload": { + "code": code, + "test": test, + }, + } + ) new_working_memory.append( {"code": f"{code}\n{test}", "feedback": fixed_code_and_test["reflections"]} ) success, result = _EXECUTE.run_isolation(f"{_DEFAULT_IMPORT}\n{code}\n{test}") + log_progress( + { + "type": "code", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": result, + }, + } + ) if verbosity == 2: - log_progress( - { - "log": f"Debug attempt {count + 1}, reflection:", - "result": fixed_code_and_test["reflections"], - } - ) _LOGGER.info( f"Debug attempt {count + 1}, reflection: {fixed_code_and_test['reflections']}" ) @@ -201,12 +231,6 @@ def write_and_test_code( f"{code}\n{test}", "python", theme="gruvbox-dark", line_numbers=True ) ) - log_progress( - { - "log": "Debug result:", - "result": result, - } - ) _LOGGER.info(f"Debug result: {result}") count += 1 @@ -232,19 +256,26 @@ def retrieve_tools( log_progress: Callable[[Dict[str, Any]], None], verbosity: int = 0, ) -> str: + log_progress( + { + "type": "tools", + "status": "started", + } + ) tool_info = [] tool_desc = [] for task in plan: tools = tool_recommender.top_k(task["instructions"], k=2, thresh=0.3) tool_info.extend([e["doc"] for e in tools]) tool_desc.extend([e["desc"] for e in tools]) + log_progress( + { + "type": "tools", + "status": "completed", + "payload": tools, + } + ) if verbosity == 2: - log_progress( - { - "log": "Retrieved tools:", - "tools": tool_desc, - } - ) _LOGGER.info(f"Tools: {tool_desc}") tool_info_set = set(tool_info) return "\n\n".join(tool_info_set) @@ -372,6 +403,12 @@ def chat_with_workflow( retries = 0 while not success and retries < self.max_retries: + self.log_progress( + { + "type": "plans", + "status": "started", + } + ) plan_i = write_plan( chat, T.TOOL_DESCRIPTIONS, @@ -380,13 +417,15 @@ def chat_with_workflow( media=[media] if media else None, ) plan_i_str = "\n-".join([e["instructions"] for e in plan_i]) + + self.log_progress( + { + "type": "plans", + "status": "completed", + "payload": plan_i, + } + ) if self.verbosity >= 1: - self.log_progress( - { - "log": "Going to run the following plan(s) in sequence:\n", - "plan": plan_i, - } - ) _LOGGER.info( f""" @@ -418,6 +457,12 @@ def chat_with_workflow( plan.append({"code": code, "test": test, "plan": plan_i}) if self_reflection: + self.log_progress( + { + "type": "self_reflection", + "status": "started", + } + ) reflection = reflect( chat, FULL_TASK.format( @@ -427,23 +472,29 @@ def chat_with_workflow( self.planner, ) if self.verbosity > 0: - self.log_progress( - { - "log": "Reflection:", - "reflection": reflection, - } - ) _LOGGER.info(f"Reflection: {reflection}") feedback = cast(str, reflection["feedback"]) success = cast(bool, reflection["success"]) + self.log_progress( + { + "type": "self_reflection", + "status": "completed" if success else "failed", + "payload": reflection, + } + ) working_memory.append({"code": f"{code}\n{test}", "feedback": feedback}) retries += 1 self.log_progress( { - "log": f"Vision Agent has concluded this chat.\nSuccess: {success}", - "finished": True, + "type": "final_code", + "status": "completed" if success else "failed", + "payload": { + "code": code, + "test": test, + "result": results["test_result"], + }, } )