From 40347973a597bbc05a7776a68f6464fe50770d0e Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Tue, 24 Sep 2024 15:40:32 +0800 Subject: [PATCH 1/4] feat: error handling for conversation agent --- vision_agent/agent/vision_agent.py | 21 +++++++++++++++++++-- vision_agent/tools/meta_tools.py | 1 + 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index c64390d5..53827876 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -314,15 +314,32 @@ def chat_with_code( # sometimes it gets stuck in a loop, so we force it to exit if last_response == response: - response["let_user_respond"] = True self.streaming_message( - {"role": "assistant", "error": "Stuck in loop"} + { + "role": "assistant", + "finished": True, + "content": "{}", + "error": { + "name": "Error when running conversation agent", + "value": "Agent is stuck in conversation loop, exited", + "traceback_raw": [], + }, + } + ) + break + elif response["let_user_respond"]: + self.streaming_message( + {"role": "assistant", "content": response, "finished": True} ) + break else: self.streaming_message({"role": "assistant", "content": response}) +<<<<<<< Updated upstream finished = response["let_user_respond"] +======= +>>>>>>> Stashed changes code_action = parse_execution( response["response"], test_multi_plan, customized_tool_names ) diff --git a/vision_agent/tools/meta_tools.py b/vision_agent/tools/meta_tools.py index 52d732f7..8c32e7fc 100644 --- a/vision_agent/tools/meta_tools.py +++ b/vision_agent/tools/meta_tools.py @@ -425,6 +425,7 @@ def detect_dogs(image_path: str): agent = va.agent.VisionAgentCoder() if name not in artifacts: + print(f"[Artifact {name} does not exist]") return f"[Artifact {name} does not exist]" code = artifacts[name] From 6f409c17004dfda3171d91b867d3936a166ae9a1 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Tue, 24 Sep 2024 18:24:53 +0800 Subject: [PATCH 2/4] address comment --- vision_agent/agent/vision_agent.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 53827876..918c9357 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -335,11 +335,8 @@ def chat_with_code( else: self.streaming_message({"role": "assistant", "content": response}) -<<<<<<< Updated upstream finished = response["let_user_respond"] -======= ->>>>>>> Stashed changes code_action = parse_execution( response["response"], test_multi_plan, customized_tool_names ) From a5731d7680f126316ab79bfdedcdfa2f2506d4d2 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Tue, 24 Sep 2024 19:22:44 +0800 Subject: [PATCH 3/4] fix the issue last user message is not legit --- vision_agent/agent/vision_agent.py | 42 +++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 918c9357..1300ca2d 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -229,7 +229,7 @@ def chat_with_code( ) as code_interpreter: orig_chat = copy.deepcopy(chat) int_chat = copy.deepcopy(chat) - last_user_message_content = chat[-1].get("content") + last_user_message = chat[-1] media_list = [] for chat_i in int_chat: if "media" in chat_i: @@ -278,9 +278,10 @@ def chat_with_code( orig_chat.append({"role": "observation", "content": artifacts_loaded}) self.streaming_message({"role": "observation", "content": artifacts_loaded}) - if int_chat[-1]["role"] == "user": - last_user_message_content = cast(str, int_chat[-1].get("content", "")) - user_code_action = parse_execution(last_user_message_content, False) + if last_user_message["role"] == "user": + user_code_action = parse_execution( + cast(str, last_user_message.get("content", "")), False + ) if user_code_action is not None: user_result, user_obs = run_code_action( user_code_action, code_interpreter, str(remote_artifacts_path) @@ -314,10 +315,10 @@ def chat_with_code( # sometimes it gets stuck in a loop, so we force it to exit if last_response == response: + response["let_user_respond"] = True self.streaming_message( { "role": "assistant", - "finished": True, "content": "{}", "error": { "name": "Error when running conversation agent", @@ -326,14 +327,6 @@ def chat_with_code( }, } ) - break - elif response["let_user_respond"]: - self.streaming_message( - {"role": "assistant", "content": response, "finished": True} - ) - break - else: - self.streaming_message({"role": "assistant", "content": response}) finished = response["let_user_respond"] @@ -341,6 +334,28 @@ def chat_with_code( response["response"], test_multi_plan, customized_tool_names ) + if last_response == response: + self.streaming_message( + { + "role": "assistant", + "content": "{}", + "error": { + "name": "Error when running conversation agent", + "value": "Agent is stuck in conversation loop, exited", + "traceback_raw": [], + }, + "finished": finished and code_action is None, + } + ) + else: + self.streaming_message( + { + "role": "assistant", + "content": response, + "finished": finished and code_action is None, + } + ) + if code_action is not None: result, obs = run_code_action( code_action, code_interpreter, str(remote_artifacts_path) @@ -367,6 +382,7 @@ def chat_with_code( "role": "observation", "content": obs, "execution": result, + "finished": finished, } ) From 636bd8cff7823c930e769355375481cc57467503 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Tue, 24 Sep 2024 19:38:46 +0800 Subject: [PATCH 4/4] fix lint --- vision_agent/agent/vision_agent.py | 76 +++++++++++++++--------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 1300ca2d..b54d08b8 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -30,6 +30,12 @@ if str(WORKSPACE) != "": os.environ["PYTHONPATH"] = f"{WORKSPACE}:{os.getenv('PYTHONPATH', '')}" +STUCK_IN_LOOP_ERROR_MESSAGE = { + "name": "Error when running conversation agent", + "value": "Agent is stuck in conversation loop, exited", + "traceback_raw": [], +} + class BoilerplateCode: pre_code = [ @@ -278,33 +284,9 @@ def chat_with_code( orig_chat.append({"role": "observation", "content": artifacts_loaded}) self.streaming_message({"role": "observation", "content": artifacts_loaded}) - if last_user_message["role"] == "user": - user_code_action = parse_execution( - cast(str, last_user_message.get("content", "")), False - ) - if user_code_action is not None: - user_result, user_obs = run_code_action( - user_code_action, code_interpreter, str(remote_artifacts_path) - ) - if self.verbosity >= 1: - _LOGGER.info(user_obs) - int_chat.append({"role": "observation", "content": user_obs}) - orig_chat.append( - { - "role": "observation", - "content": user_obs, - "execution": user_result, - } - ) - self.streaming_message( - { - "role": "observation", - "content": user_obs, - "execution": user_result, - "finished": True, - } - ) - finished = True + finished = self.execute_user_code_action( + last_user_message, code_interpreter, remote_artifacts_path + ) while not finished and iterations < self.max_iterations: response = run_conversation(self.agent, int_chat) @@ -320,11 +302,7 @@ def chat_with_code( { "role": "assistant", "content": "{}", - "error": { - "name": "Error when running conversation agent", - "value": "Agent is stuck in conversation loop, exited", - "traceback_raw": [], - }, + "error": STUCK_IN_LOOP_ERROR_MESSAGE, } ) @@ -339,11 +317,7 @@ def chat_with_code( { "role": "assistant", "content": "{}", - "error": { - "name": "Error when running conversation agent", - "value": "Agent is stuck in conversation loop, exited", - "traceback_raw": [], - }, + "error": STUCK_IN_LOOP_ERROR_MESSAGE, "finished": finished and code_action is None, } ) @@ -397,6 +371,34 @@ def chat_with_code( artifacts.save() return orig_chat, artifacts + def execute_user_code_action( + self, + last_user_message: Message, + code_interpreter: CodeInterpreter, + remote_artifacts_path: Path, + ) -> bool: + if last_user_message["role"] != "user": + return False + user_code_action = parse_execution( + cast(str, last_user_message.get("content", "")), False + ) + if user_code_action is not None: + user_result, user_obs = run_code_action( + user_code_action, code_interpreter, str(remote_artifacts_path) + ) + if self.verbosity >= 1: + _LOGGER.info(user_obs) + self.streaming_message( + { + "role": "observation", + "content": user_obs, + "execution": user_result, + "finished": True, + } + ) + return True + return False + def streaming_message(self, message: Dict[str, Any]) -> None: if self.callback_message: self.callback_message(message)