From c35161a0cbdafbec5860e72a053773c564358c3b Mon Sep 17 00:00:00 2001 From: Dillon Laird Date: Mon, 14 Oct 2024 12:50:43 -0700 Subject: [PATCH 1/3] fixed issue with uploading images --- examples/chat/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/chat/app.py b/examples/chat/app.py index 2069218b..d77df9b7 100644 --- a/examples/chat/app.py +++ b/examples/chat/app.py @@ -147,7 +147,7 @@ def main(): f.write(uploaded_file.getbuffer()) # make it None so it wont load and overwrite the image - artifacts.artifacts[uploaded_file.name] = None + artifacts.artifacts[uploaded_file.name] = open(WORKSPACE / uploaded_file.name, "rb").read() for file in WORKSPACE.iterdir(): if "__pycache__" not in str(file) and not str(file).startswith("."): From 9f0f155f37f07bfb4a5c1c38331505748e22e978 Mon Sep 17 00:00:00 2001 From: Dillon Laird Date: Mon, 14 Oct 2024 12:50:56 -0700 Subject: [PATCH 2/3] revert back to old format outside of va --- vision_agent/agent/vision_agent.py | 63 +++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py index 38857724..2eb27376 100644 --- a/vision_agent/agent/vision_agent.py +++ b/vision_agent/agent/vision_agent.py @@ -210,6 +210,51 @@ def add_step_descriptions(response: Dict[str, Any]) -> Dict[str, Any]: return response +def new_format_to_old_format(new_format: Dict[str, Any]) -> Dict[str, Any]: + thoughts = new_format["thinking"] if new_format["thinking"] is not None else "" + response = new_format["response"] if new_format["response"] is not None else "" + if new_format["execute_python"] is not None: + response += ( + f"\n\n{new_format['execute_python']}\n" + ) + return { + "thoughts": thoughts, + "response": response, + "let_user_respond": new_format["let_user_respond"], + } + + +def old_format_to_new_format(old_format_str: str) -> str: + try: + old_format = json.loads(old_format_str) + except json.JSONDecodeError: + return old_format_str + + thinking = old_format["thoughts"] if old_format["thoughts"].strip() != "" else None + let_user_respond = old_format["let_user_respond"] + if "" in old_format["response"]: + execute_python = extract_tag(old_format["response"], "execute_python") + response = ( + old_format["response"] + .replace(execute_python, "") + .replace("", "") + .replace("", "") + .strip() + ) + else: + execute_python = None + response = old_format["response"] + + return json.dumps( + { + "thinking": thinking, + "response": response, + "execute_python": execute_python, + "let_user_respond": let_user_respond, + } + ) + + class VisionAgent(Agent): """Vision Agent is an agent that can chat with the user and call tools or other agents to generate code for it. Vision Agent uses python code to execute actions @@ -374,11 +419,11 @@ def chat_with_artifacts( ( { "role": c["role"], - "content": c["content"], + "content": old_format_to_new_format(c["content"]), # type: ignore "media": c["media"], } if "media" in c - else {"role": c["role"], "content": c["content"]} + else {"role": c["role"], "content": old_format_to_new_format(c["content"])} # type: ignore ) for c in int_chat ], @@ -432,13 +477,17 @@ def chat_with_artifacts( int_chat.append( { "role": "assistant", - "content": json.dumps(add_step_descriptions(response)), + "content": json.dumps( + new_format_to_old_format(add_step_descriptions(response)) + ), } ) orig_chat.append( { "role": "assistant", - "content": json.dumps(add_step_descriptions(response)), + "content": json.dumps( + new_format_to_old_format(add_step_descriptions(response)) + ), } ) @@ -471,7 +520,11 @@ def chat_with_artifacts( self.streaming_message( { "role": "assistant", - "content": json.dumps(add_step_descriptions(response)), + "content": json.dumps( + new_format_to_old_format( + add_step_descriptions(response) + ) + ), "finished": finished and code_action is None, } ) From dbabf3eca40521183843eea45799da03298c52ea Mon Sep 17 00:00:00 2001 From: Dillon Laird Date: Mon, 14 Oct 2024 12:55:09 -0700 Subject: [PATCH 3/3] formatting fix --- examples/chat/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/chat/app.py b/examples/chat/app.py index d77df9b7..25188649 100644 --- a/examples/chat/app.py +++ b/examples/chat/app.py @@ -147,7 +147,9 @@ def main(): f.write(uploaded_file.getbuffer()) # make it None so it wont load and overwrite the image - artifacts.artifacts[uploaded_file.name] = open(WORKSPACE / uploaded_file.name, "rb").read() + artifacts.artifacts[uploaded_file.name] = open( + WORKSPACE / uploaded_file.name, "rb" + ).read() for file in WORKSPACE.iterdir(): if "__pycache__" not in str(file) and not str(file).startswith("."):