From 2eed7e83aceb2b75c87b2aebfae4b58db392d835 Mon Sep 17 00:00:00 2001 From: Dillon Laird Date: Mon, 26 Aug 2024 08:42:52 -0700 Subject: [PATCH] use llama3.1 for ollamavisionagentcoder --- vision_agent/agent/vision_agent_coder.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/vision_agent/agent/vision_agent_coder.py b/vision_agent/agent/vision_agent_coder.py index 48bf5b70..3ddddd75 100644 --- a/vision_agent/agent/vision_agent_coder.py +++ b/vision_agent/agent/vision_agent_coder.py @@ -845,9 +845,12 @@ class OllamaVisionAgentCoder(VisionAgentCoder): """VisionAgentCoder that uses Ollama models for planning, coding, testing. Pre-requisites: - 1. Run ollama pull llava for the LMM (or any other LMM model that can consume images) + 1. Run ollama pull llama3.1 for the LLM 2. Run ollama pull mxbai-embed-large for the embedding similarity model + Technically you should use a VLM such as llava but llava is not able to handle the + context length and crashes. + Example ------- >>> image vision_agent as va @@ -867,14 +870,22 @@ def __init__( ) -> None: super().__init__( planner=( - OllamaLMM(temperature=0.0, json_mode=True) + OllamaLMM(model_name="llama3.1", temperature=0.0, json_mode=True) if planner is None else planner ), - coder=OllamaLMM(temperature=0.0) if coder is None else coder, - tester=OllamaLMM(temperature=0.0) if tester is None else tester, + coder=( + OllamaLMM(model_name="llama3.1", temperature=0.0) + if coder is None + else coder + ), + tester=( + OllamaLMM(model_name="llama3.1", temperature=0.0) + if tester is None + else tester + ), debugger=( - OllamaLMM(temperature=0.0, json_mode=True) + OllamaLMM(model_name="llama3.1", temperature=0.0, json_mode=True) if debugger is None else debugger ),