From f82a6ec672ea747590f1186fe7109e602dcc8db6 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Sat, 21 Sep 2024 21:27:36 +0800 Subject: [PATCH 1/4] feat: when code edited by code executed directly --- vision_agent/agent/vision_agent.py | 19 +++++++++++++++++++ vision_agent/tools/meta_tools.py | 1 + 2 files changed, 20 insertions(+) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 1e1abbe6..72727f49 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -218,6 +218,7 @@ def chat_with_code( ) as code_interpreter: orig_chat = copy.deepcopy(chat) int_chat = copy.deepcopy(chat) + last_user_message = chat[-1] media_list = [] for chat_i in int_chat: if "media" in chat_i: @@ -266,6 +267,24 @@ def chat_with_code( orig_chat.append({"role": "observation", "content": artifacts_loaded}) self.streaming_message({"role": "observation", "content": artifacts_loaded}) + user_code_action = parse_execution(last_user_message.get("content"), False) + + if user_code_action is not None: + user_result, user_obs = run_code_action( + user_code_action, code_interpreter, str(remote_artifacts_path) + ) + if self.verbosity >= 1: + _LOGGER.info(user_obs) + self.streaming_message( + { + "role": "observation", + "content": user_obs, + "execution": user_result, + "finished": True, + } + ) + finished = True + while not finished and iterations < self.max_iterations: response = run_conversation(self.agent, int_chat) if self.verbosity >= 1: diff --git a/vision_agent/tools/meta_tools.py b/vision_agent/tools/meta_tools.py index 9012e9d4..0d20cb28 100644 --- a/vision_agent/tools/meta_tools.py +++ b/vision_agent/tools/meta_tools.py @@ -250,6 +250,7 @@ def edit_code_artifact( total_lines = len(artifacts[name].splitlines()) if start < 0 or end < 0 or start > end or end > total_lines: + print("[Invalid line range]") return "[Invalid line range]" if start == end: end += 1 From 40569f0d0ea01af218c9f719456dc557b7ad0f7b Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Sat, 21 Sep 2024 21:38:36 +0800 Subject: [PATCH 2/4] fix lint --- vision_agent/agent/vision_agent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 72727f49..962815aa 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -267,7 +267,11 @@ def chat_with_code( orig_chat.append({"role": "observation", "content": artifacts_loaded}) self.streaming_message({"role": "observation", "content": artifacts_loaded}) - user_code_action = parse_execution(last_user_message.get("content"), False) + user_code_action = None + if isinstance(last_user_message.get("content"), str): + user_code_action = parse_execution( + last_user_message.get("content"), False + ) if user_code_action is not None: user_result, user_obs = run_code_action( From e8ac6e634c6dc91d80ed9be6fccce17d71ac7be0 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Sat, 21 Sep 2024 21:44:51 +0800 Subject: [PATCH 3/4] fix lint --- vision_agent/agent/vision_agent.py | 40 ++++++++++++++---------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 962815aa..aea703be 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -218,7 +218,7 @@ def chat_with_code( ) as code_interpreter: orig_chat = copy.deepcopy(chat) int_chat = copy.deepcopy(chat) - last_user_message = chat[-1] + last_user_message_content = chat[-1].get("content") media_list = [] for chat_i in int_chat: if "media" in chat_i: @@ -267,27 +267,23 @@ def chat_with_code( orig_chat.append({"role": "observation", "content": artifacts_loaded}) self.streaming_message({"role": "observation", "content": artifacts_loaded}) - user_code_action = None - if isinstance(last_user_message.get("content"), str): - user_code_action = parse_execution( - last_user_message.get("content"), False - ) - - if user_code_action is not None: - user_result, user_obs = run_code_action( - user_code_action, code_interpreter, str(remote_artifacts_path) - ) - if self.verbosity >= 1: - _LOGGER.info(user_obs) - self.streaming_message( - { - "role": "observation", - "content": user_obs, - "execution": user_result, - "finished": True, - } - ) - finished = True + if isinstance(last_user_message_content, str): + user_code_action = parse_execution(last_user_message_content, False) + if user_code_action is not None: + user_result, user_obs = run_code_action( + user_code_action, code_interpreter, str(remote_artifacts_path) + ) + if self.verbosity >= 1: + _LOGGER.info(user_obs) + self.streaming_message( + { + "role": "observation", + "content": user_obs, + "execution": user_result, + "finished": True, + } + ) + finished = True while not finished and iterations < self.max_iterations: response = run_conversation(self.agent, int_chat) From f994a77399bcfbec1303208b4c6d346532b6dd43 Mon Sep 17 00:00:00 2001 From: wuyiqunLu Date: Sat, 21 Sep 2024 22:21:46 +0800 Subject: [PATCH 4/4] address comment --- vision_agent/agent/vision_agent.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index aea703be..62682524 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -275,6 +275,14 @@ def chat_with_code( ) if self.verbosity >= 1: _LOGGER.info(user_obs) + int_chat.append({"role": "observation", "content": user_obs}) + orig_chat.append( + { + "role": "observation", + "content": user_obs, + "execution": user_result, + } + ) self.streaming_message( { "role": "observation",