adjust payload

Author: Dayof

vision_agent/tools/tool_utils.py

@@ -1,6 +1,6 @@
+import os
 import inspect
 import logging
-import os
 from base64 import b64encode
 from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple
 
@@ -38,7 +38,7 @@ def send_inference_request(
     v2: bool = False,
     metadata_payload: Optional[Dict[str, Any]] = None,
 ) -> Any:
-    # TODO: runtime_tag and function_name should be metadata_payload and now included
+    # TODO: runtime_tag and function_name should be metadata_payload and not included
     # in the service payload
     if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
         payload["runtime_tag"] = runtime_tag

vision_agent/tools/tools.py

@@ -1,20 +1,20 @@
+import os
 import io
 import json
 import logging
-import os
 import tempfile
 import urllib.request
-from importlib import resources
+from uuid import UUID
 from pathlib import Path
+from importlib import resources
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
-from uuid import UUID
 
 import cv2
-import numpy as np
 import requests
-from PIL import Image, ImageDraw, ImageEnhance, ImageFont
-from pillow_heif import register_heif_opener  # type: ignore
+import numpy as np
 from pytube import YouTube  # type: ignore
+from pillow_heif import register_heif_opener  # type: ignore
+from PIL import Image, ImageDraw, ImageEnhance, ImageFont
 
 from vision_agent.clients.landing_public_api import LandingPublicAPI
 from vision_agent.lmm.lmm import OpenAILMM
@@ -28,7 +28,6 @@
     send_task_inference_request,
 )
 from vision_agent.tools.tools_types import (
-    FineTuning,
     Florence2FtRequest,
     JobStatus,
     ODResponseData,
@@ -194,12 +193,16 @@ def owl_v2_image(
         data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
-            fine_tuning=FineTuning(job_id=UUID(fine_tune_id)),
+            job_id=UUID(fine_tune_id),
+        )
+        data = data_obj.model_dump(by_alias=True, exclude_none=True)
+        detections = send_inference_request(
+            data,
+            "florence2-ft",
+            v2=True,
+            metadata_payload={"function_name": "owl_v2_image"},
         )
-        data = data_obj.model_dump(by_alias=True)
-        detections = send_inference_request(data, "tools", v2=False)
         # get the first frame
         detection = detections[0]
         bboxes_formatted = [
@@ -420,15 +423,17 @@ def florence2_sam2_image(
         req_data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
             postprocessing="sam2",
-            fine_tuning=FineTuning(
-                job_id=UUID(fine_tune_id),
-            ),
+            job_id=UUID(fine_tune_id),
+        )
+        req_data = req_data_obj.model_dump(by_alias=True, exclude_none=True)
+        detections_ft = send_inference_request(
+            req_data,
+            "florence2-ft",
+            v2=True,
+            metadata_payload={"function_name": "florence2_sam2_image"},
         )
-        req_data = req_data_obj.model_dump(by_alias=True)
-        detections_ft = send_inference_request(req_data, "tools", v2=False)
         # get the first frame
         detection = detections_ft[0]
         return_data = []
@@ -1136,6 +1141,9 @@ def florence2_image_caption(image: np.ndarray, detail_caption: bool = True) -> s
     return answer[task]  # type: ignore
 
 
+# TODO: add video
+
+
 def florence2_phrase_grounding(
     prompt: str, image: np.ndarray, fine_tune_id: Optional[str] = None
 ) -> List[Dict[str, Any]]:
@@ -1180,15 +1188,14 @@ def florence2_phrase_grounding(
         data_obj = Florence2FtRequest(
             image=image_b64,
             task=PromptTask.PHRASE_GROUNDING,
-            tool="florencev2_fine_tuning",
             prompt=prompt,
-            fine_tuning=FineTuning(job_id=UUID(fine_tune_id)),
+            job_id=UUID(fine_tune_id),
         )
-        data = data_obj.model_dump(by_alias=True)
+        data = data_obj.model_dump(by_alias=True, exclude_none=True)
         detections = send_inference_request(
             data,
-            "tools",
-            v2=False,
+            "florence2-ft",
+            v2=True,
             metadata_payload={"function_name": "florence2_phrase_grounding"},
         )
         # get the first frame

vision_agent/tools/tools_types.py

@@ -1,6 +1,6 @@
 from enum import Enum
-from typing import List, Optional, Tuple, Union
 from uuid import UUID
+from typing import List, Optional, Tuple, Union
 
 from pydantic import BaseModel, ConfigDict, Field, SerializationInfo, field_serializer
 
@@ -24,25 +24,20 @@ class PromptTask(str, Enum):
     PHRASE_GROUNDING = "<CAPTION_TO_PHRASE_GROUNDING>"
 
 
-class FineTuning(BaseModel):
-    model_config = ConfigDict(populate_by_name=True)
-
-    job_id: UUID = Field(alias="jobId")
-
-    @field_serializer("job_id")
-    def serialize_job_id(self, job_id: UUID, _info: SerializationInfo) -> str:
-        return str(job_id)
-
-
 class Florence2FtRequest(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
-    image: str
+    image: str | None
+    video: bytes | None
     task: PromptTask
-    tool: str
     prompt: Optional[str] = ""
+    chunk_length_frames: Optional[int] = None
     postprocessing: Optional[str] = None
-    fine_tuning: Optional[FineTuning] = Field(None, alias="fineTuning")
+    job_id: Optional[UUID] = Field(None, alias="jobId")
+
+    @field_serializer("job_id")
+    def serialize_job_id(self, job_id: UUID, _info: SerializationInfo) -> str:
+        return str(job_id)
 
 
 class JobStatus(str, Enum):