Merge branch 'main' of github.com:landing-ai/vision-agent into feat/f…

…lorence-fine-tuning

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "vision-agent"
-version = "0.2.96"
+version = "0.2.98"
 description = "Toolset for Vision Agent"
 authors = ["Landing AI <[email protected]>"]
 readme = "README.md"
@@ -36,7 +36,7 @@ rich = "^13.7.1"
 langsmith = "^0.1.58"
 ipykernel = "^6.29.4"
 e2b = "^0.17.1"
-e2b-code-interpreter = "0.0.11a17"
+e2b-code-interpreter = "0.0.11a27"
 tenacity = "^8.3.0"
 pillow-heif = "^0.16.0"
 pytube = "15.0.0"

tests/unit/tools/test_tools.py

@@ -23,9 +23,3 @@ def test_saves_frames_with_output_path(self, tmp_path):
 
  assert output_path == video_output_path
  assert Path(output_path).exists()
-
- # Handles an empty list of frames gracefully
- def test_handles_empty_frames_list(self):
- frames = []
- output_path = save_video(frames)
- assert Path(output_path).exists()

vision_agent/agent/vision_agent_coder.py

@@ -93,7 +93,7 @@ def format_plans(plans: Dict[str, Any]) -> str:
 
 
 def extract_image(
- media: Optional[Sequence[Union[str, Path]]]
+ media: Optional[Sequence[Union[str, Path]]],
 ) -> Optional[Sequence[Union[str, Path]]]:
  if media is None:
  return None
@@ -186,7 +186,8 @@ def pick_plan(
  if tool_output.success
  else "Code execution failed"
  ),
- "payload": tool_output.to_json(),
+ "code": DefaultImports.prepend_imports(code),
+ # "payload": tool_output.to_json(),
  "status": "completed" if tool_output.success else "failed",
  }
  )
@@ -211,6 +212,9 @@ def pick_plan(
  }
  )
  code = extract_code(model(prompt))
+ tool_output = code_interpreter.exec_isolation(
+ DefaultImports.prepend_imports(code)
+ )
  log_progress(
  {
  "type": "log",
@@ -220,13 +224,10 @@ def pick_plan(
  else "Code execution failed"
  ),
  "code": DefaultImports.prepend_imports(code),
- "payload": tool_output.to_json(),
+ # "payload": tool_output.to_json(),
  "status": "completed" if tool_output.success else "failed",
  }
  )
- tool_output = code_interpreter.exec_isolation(
- DefaultImports.prepend_imports(code)
- )
  tool_output_str = ""
  if len(tool_output.logs.stdout) > 0:
  tool_output_str = tool_output.logs.stdout[0]