minor updates

docs/api/tools.md

@@ -2,4 +2,6 @@
 
 ::: vision_agent.tools.prompts
 
-::: vision_agent.tools.tools
+::: vision_agent.tools.tools
+
+::: vision_agent.tools.video

pyproject.toml

@@ -28,8 +28,6 @@ torch = "2.1.*" # 2.2 causes sentence-transformers to seg fault
 sentence-transformers = "2.*"
 openai = "1.*"
 typing_extensions = "4.*"
-
-[tool.poetry.group.video.dependencies]
 moviepy = "1.*"
 opencv-python-headless = "4.*"
 

tests/tools/test_video.py

@@ -2,6 +2,7 @@
 
 
 def test_extract_frames_from_video():
+ # TODO: consider generating a video on the fly instead
  video_path = "tests/data/video/test.mp4"
  res = extract_frames_from_video(video_path)
  assert len(res) == 1

vision_agent/tools/tools.py

@@ -11,6 +11,7 @@
 from PIL.Image import Image as ImageType
 
 from vision_agent.image_utils import convert_to_b64, get_image_size
+from vision_agent.tools.video import extract_frames_from_video
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -523,12 +524,15 @@ class ExtractFrames(Tool):
  }
 
  def __call__(self, video_uri: str) -> list[tuple[str, float]]:
- try:
- from vision_agent.tools.video import extract_frames_from_video
- except Exception as e:
- raise ImportError(
- "vision_agent is not installed correctly (cause: missing dependencies), please run 'pip install vision-agent[video]' instead."
- ) from e
+ """Extract frames from a video clip with start and end time in seconds.
+
+
+ Parameters:
+ video_uri: the path to the video file or a url points to the video data
+
+ Returns:
+ a list of tuples containing the extracted frame and the timestamp in seconds. E.g. [(path_to_frame1, 0.0), (path_to_frame2, 0.5), ...]. The timestamp is the time in seconds from the start of the video. E.g. 12.125 means 12.125 seconds from the start of the video. The frames are sorted by the timestamp in ascending order.
+ """
  frames = extract_frames_from_video(video_uri)
  result = []
  _LOGGER.info(

vision_agent/tools/video.py

@@ -23,9 +23,9 @@ def extract_frames_from_video(
  ----------
  video_uri: str, the path to the video file or a video file url
  fps: int, the frame rate per second to extract the frames
- motion_detection_threshold: float, the threshold to detect the motion between frames.
- A value between 0-1, the percentage change that is considered a different frame.
- A lower value means more frames will be extracted.
+ motion_detection_threshold: float, The threshold to detect motion between changes/frames.
+ A value between 0-1, which represents the percentage change required for the frames to be considered in motion.
+ For example, a lower value means more frames will be extracted.
 
  Returns
  -------
@@ -181,10 +181,3 @@ def _similar_frame(
  )
  _LOGGER.debug(f"Image diff: {change_percentage}")
  return change_percentage < threshold
-
-
-# res = extract_frames(video)
-if __name__ == "__main__":
- video_path = "/Users/asia/Downloads/frames/baby_cam1.MP4"
- res = extract_frames_from_video(video_path)
- print("done, extracted num frames: ", len(res))