From d3ba8ee70aa0ffe2c9ff755058b71f817857c9c7 Mon Sep 17 00:00:00 2001 From: Zhichao Date: Thu, 5 Sep 2024 10:53:43 +0800 Subject: [PATCH 1/4] add envs to the new_instance method --- vision_agent/utils/execute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vision_agent/utils/execute.py b/vision_agent/utils/execute.py index 0de10335..d1c0b049 100644 --- a/vision_agent/utils/execute.py +++ b/vision_agent/utils/execute.py @@ -682,12 +682,13 @@ def get_default_instance() -> CodeInterpreter: def new_instance( code_sandbox_runtime: Optional[str] = None, remote_path: Optional[Union[str, Path]] = None, + envs: Optional[Dict[str, str]] = None, ) -> CodeInterpreter: if not code_sandbox_runtime: code_sandbox_runtime = os.getenv("CODE_SANDBOX_RUNTIME", "local") if code_sandbox_runtime == "e2b": instance: CodeInterpreter = E2BCodeInterpreter( - timeout=_SESSION_TIMEOUT, remote_path=remote_path + timeout=_SESSION_TIMEOUT, remote_path=remote_path, envs=envs ) elif code_sandbox_runtime == "local": instance = LocalCodeInterpreter( From fdf80f8b90245d42fdd9d0f5fe930db3a9c4cf9b Mon Sep 17 00:00:00 2001 From: Zhichao Date: Thu, 5 Sep 2024 11:36:13 +0800 Subject: [PATCH 2/4] add envs to chat_with_workflow --- vision_agent/agent/vision_agent_coder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vision_agent/agent/vision_agent_coder.py b/vision_agent/agent/vision_agent_coder.py index dd893d1d..2805cead 100644 --- a/vision_agent/agent/vision_agent_coder.py +++ b/vision_agent/agent/vision_agent_coder.py @@ -675,6 +675,7 @@ def chat_with_workflow( test_multi_plan: bool = True, display_visualization: bool = False, customized_tool_names: Optional[List[str]] = None, + envs: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: """Chat with VisionAgentCoder and return intermediate information regarding the task. @@ -703,7 +704,7 @@ def chat_with_workflow( # NOTE: each chat should have a dedicated code interpreter instance to avoid concurrency issues with CodeInterpreterFactory.new_instance( - code_sandbox_runtime=self.code_sandbox_runtime + code_sandbox_runtime=self.code_sandbox_runtime, envs=envs ) as code_interpreter: chat = copy.deepcopy(chat) media_list = [] @@ -859,7 +860,9 @@ def _create_plans( plans = write_plans( int_chat, T.get_tool_descriptions_by_names( - customized_tool_names, T.FUNCTION_TOOLS, T.UTIL_TOOLS # type: ignore + customized_tool_names, + T.FUNCTION_TOOLS, + T.UTIL_TOOLS, # type: ignore ), format_memory(working_memory), planner, From f0ed54c14bda1079b17d4523df447de75980bd75 Mon Sep 17 00:00:00 2001 From: Zhichao Date: Thu, 5 Sep 2024 11:52:58 +0800 Subject: [PATCH 3/4] fix lint --- vision_agent/agent/vision_agent_coder.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vision_agent/agent/vision_agent_coder.py b/vision_agent/agent/vision_agent_coder.py index 2805cead..dd4e4dec 100644 --- a/vision_agent/agent/vision_agent_coder.py +++ b/vision_agent/agent/vision_agent_coder.py @@ -860,9 +860,7 @@ def _create_plans( plans = write_plans( int_chat, T.get_tool_descriptions_by_names( - customized_tool_names, - T.FUNCTION_TOOLS, - T.UTIL_TOOLS, # type: ignore + customized_tool_names, T.FUNCTION_TOOLS, T.UTIL_TOOLS # type: ignore ), format_memory(working_memory), planner, From 96d877815a6a95c28f6cdd8cd243416257a55c66 Mon Sep 17 00:00:00 2001 From: Zhichao Date: Thu, 5 Sep 2024 14:04:25 +0800 Subject: [PATCH 4/4] add docstring for envs --- vision_agent/agent/vision_agent_coder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vision_agent/agent/vision_agent_coder.py b/vision_agent/agent/vision_agent_coder.py index dd4e4dec..b0b1c812 100644 --- a/vision_agent/agent/vision_agent_coder.py +++ b/vision_agent/agent/vision_agent_coder.py @@ -693,6 +693,8 @@ def chat_with_workflow( show the image(s) created by visualization code (if there is any). customized_tool_names (List[str]): A list of customized tools for agent to pick and use. If not provided, default to full tool set from vision_agent.tools. + envs (Dict[str, str]): A dict of envs that would be passed to the CodeInterpreter. + Mainly for passing local envs to E2B. Local interpreter can pick up envs naturally. Returns: Dict[str, Any]: A dictionary containing the code, test, test result, plan,