Add EasyTool (#19)

* add easytool and prompts

* add more tools, reformat doc

* add in imports

* fix llm for tools

* fix doc

* fix formatting

* fix flake8

* fix type errors

* updated tests

* updated tools

* updated easytools

* formatting fix

tests/fixtures.py

@@ -2,6 +2,8 @@
 
 import pytest
 
+from vision_agent.tools import CLIP, GroundingDINO, GroundingSAM
+
 
 @pytest.fixture
 def openai_llm_mock(request):
@@ -27,3 +29,24 @@ def openai_lmm_mock(request):
  choices=[MagicMock(message=MagicMock(content=content))]
  )
  yield mock_instance
+
+
+@pytest.fixture
+def clip_mock(request):
+ with patch.object(CLIP, "__call__", autospec=True) as mock:
+ mock.return_value = "test"
+ yield mock
+
+
+@pytest.fixture
+def grounding_dino_mock(request):
+ with patch.object(GroundingDINO, "__call__", autospec=True) as mock:
+ mock.return_value = "test"
+ yield mock
+
+
+@pytest.fixture
+def grounding_sam_mock(request):
+ with patch.object(GroundingSAM, "__call__", autospec=True) as mock:
+ mock.return_value = "test"
+ yield mock

tests/test_llm.py

@@ -1,9 +1,13 @@
 import pytest
 
 from vision_agent.llm.llm import OpenAILLM
-from vision_agent.tools import CLIP, GroundingDINO, GroundingSAM
 
-from .fixtures import openai_llm_mock # noqa: F401
+from .fixtures import ( # noqa: F401
+ clip_mock,
+ grounding_dino_mock,
+ grounding_sam_mock,
+ openai_llm_mock,
+)
 
 
 @pytest.mark.parametrize(
@@ -57,35 +61,35 @@ def test_call_with_mock(openai_llm_mock): # noqa: F811
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_llm_mock"],
 )
-def test_generate_classifier(openai_llm_mock): # noqa: F811
+def test_generate_classifier(openai_llm_mock, clip_mock): # noqa: F811
  llm = OpenAILLM()
  prompt = "Can you generate a cat classifier?"
  classifier = llm.generate_classifier(prompt)
- assert isinstance(classifier, CLIP)
- assert classifier.prompt == "cat"
+ classifier("image.png")
+ assert clip_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}
 
 
 @pytest.mark.parametrize(
  "openai_llm_mock",
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_llm_mock"],
 )
-def test_generate_detector(openai_llm_mock): # noqa: F811
+def test_generate_detector(openai_llm_mock, grounding_dino_mock): # noqa: F811
  llm = OpenAILLM()
  prompt = "Can you generate a cat detector?"
  detector = llm.generate_detector(prompt)
- assert isinstance(detector, GroundingDINO)
- assert detector.prompt == "cat"
+ detector("image.png")
+ assert grounding_dino_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}
 
 
 @pytest.mark.parametrize(
  "openai_llm_mock",
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_llm_mock"],
 )
-def test_generate_segmentor(openai_llm_mock): # noqa: F811
+def test_generate_segmentor(openai_llm_mock, grounding_sam_mock): # noqa: F811
  llm = OpenAILLM()
  prompt = "Can you generate a cat segmentor?"
  segmentor = llm.generate_segmentor(prompt)
- assert isinstance(segmentor, GroundingSAM)
- assert segmentor.prompt == "cat"
+ segmentor("image.png")
+ assert grounding_sam_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}

tests/test_lmm.py

@@ -4,9 +4,13 @@
 from PIL import Image
 
 from vision_agent.lmm.lmm import OpenAILMM
-from vision_agent.tools import CLIP, GroundingDINO, GroundingSAM
 
-from .fixtures import openai_lmm_mock # noqa: F401
+from .fixtures import ( # noqa: F401
+ clip_mock,
+ grounding_dino_mock,
+ grounding_sam_mock,
+ openai_lmm_mock,
+)
 
 
 def create_temp_image(image_format="jpeg"):
@@ -77,35 +81,35 @@ def test_call_with_mock(openai_lmm_mock): # noqa: F811
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_lmm_mock"],
 )
-def test_generate_classifier(openai_lmm_mock): # noqa: F811
+def test_generate_classifier(openai_lmm_mock, clip_mock): # noqa: F811
  lmm = OpenAILMM()
  prompt = "Can you generate a cat classifier?"
  classifier = lmm.generate_classifier(prompt)
- assert isinstance(classifier, CLIP)
- assert classifier.prompt == "cat"
+ classifier("image.png")
+ assert clip_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}
 
 
 @pytest.mark.parametrize(
  "openai_lmm_mock",
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_lmm_mock"],
 )
-def test_generate_detector(openai_lmm_mock): # noqa: F811
+def test_generate_detector(openai_lmm_mock, grounding_dino_mock): # noqa: F811
  lmm = OpenAILMM()
  prompt = "Can you generate a cat classifier?"
  detector = lmm.generate_detector(prompt)
- assert isinstance(detector, GroundingDINO)
- assert detector.prompt == "cat"
+ detector("image.png")
+ assert grounding_dino_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}
 
 
 @pytest.mark.parametrize(
  "openai_lmm_mock",
  ['{"Parameters": {"prompt": "cat"}}'],
  indirect=["openai_lmm_mock"],
 )
-def test_generate_segmentor(openai_lmm_mock): # noqa: F811
+def test_generate_segmentor(openai_lmm_mock, grounding_sam_mock): # noqa: F811
  lmm = OpenAILMM()
  prompt = "Can you generate a cat classifier?"
  segmentor = lmm.generate_segmentor(prompt)
- assert isinstance(segmentor, GroundingSAM)
- assert segmentor.prompt == "cat"
+ segmentor("image.png")
+ assert grounding_sam_mock.call_args[1] == {"prompt": "cat", "image": "image.png"}

vision_agent/agent/__init__.py

@@ -1,2 +1,3 @@
 from .agent import Agent
 from .reflexion import Reflexion
+from .easytool import EasyTool