From 80e8d2ee3187e9eb3ffe02456d07c1c57eb1981b Mon Sep 17 00:00:00 2001 From: Shankar <90070882+shankar-landing-ai@users.noreply.github.com> Date: Mon, 1 Apr 2024 14:50:00 -0700 Subject: [PATCH] get endpoint ready for demo (#33) * get endpoint ready for demo * fixed tools.json * Update vision_agent/tools/tools.py --------- Co-authored-by: Asia <2736300+humpydonkey@users.noreply.github.com> --- vision_agent/tools/tools.json | 154 ++++++++++++++++++++++++++++++++++ vision_agent/tools/tools.py | 19 +++-- 2 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 vision_agent/tools/tools.json diff --git a/vision_agent/tools/tools.json b/vision_agent/tools/tools.json new file mode 100644 index 00000000..8901f9a0 --- /dev/null +++ b/vision_agent/tools/tools.json @@ -0,0 +1,154 @@ +[ + { + "name": "image_question_answering", + "description": "answers a question about an image" + }, + { + "name": "text_question_answering", + "description": "answers a question provided a context in text form" + }, + { + "name": "image_captioning", + "description": "Generate a caption for the image. It can generate a brief description that can be used for image perception and image generation. For example: a) you can use this tool when you want to know what is it in the image\"; and b) when you want to generate a new image similar or resemble to input.png, you can use `image_captioning` to obtain the description about image input.png." + }, + { + "name": "image_to_text", + "description": "Generate a description for the image. It can generate a detailed description that can be used for image perception and image generation. For example: a) you can use this tool when you want to know what is it in the image\"; and b) when you want to generate a new image similar or resemble to input.png, you can use `text_to_image` to obtain the description about image input.png." + }, + { + "name": "image_to_edge", + "description": "get the edge map of the image." + }, + { + "name": "image_to_line", + "description": "get the line map of the image." + }, + { + "name": "image_to_hed", + "description": "get the HED map of the image." + }, + { + "name": "image_to_scribble", + "description": "get the scribble of the image." + }, + { + "name": "image_to_pose", + "description": "Get the pose of the image. It is usually used in image generation conditioned on pose map from input image." + }, + { + "name": "image_to_depth", + "description": "get the depth map of the image." + }, + { + "name": "image_to_normal", + "description": "get the normal map of the image." + }, + { + "name": "object_detection", + "description": "detect all the objects in the image." + }, + { + "name": "image_classification", + "description": "classify the objects in the image." + }, + { + "name": "closed_set_image_classification", + "description": "Given a set of classes as a text prompt, classify the objects in the image based on the given classes." + }, + { + "name": "panoptic_segmentation", + "description": "segment the common objects in the given image." + }, + { + "name": "visual_grounding", + "description": "Visual Grounding (VG) aims to locate the most relevant object or region in an image, based on a natural language query. The query can be a phrase, a sentence or even a multi-round dialogue." + }, + { + "name": "visual_grounding_segment", + "description": "Visual Grounding (VG) aims to locate the most relevant object or region in an image, based on a natural language query. The query can be a phrase, a sentence or even a multi-round dialogue." + }, + { + "name": "optical_character_recognition", + "description": "Optical Character Recognition (OCR) is the process that converts an image of text into a machine-readable text format." + }, + { + "name": "select_category", + "description": "select the target classes in category list with the given condition." + }, + { + "name": "select_bbox", + "description": "select the bounding boxes with the given condition." + }, + { + "name": "select_mask", + "description": "select the masks with the given condition." + }, + { + "name": "count_categories", + "description": "count target categories in the given list." + }, + { + "name": "count_objects", + "description": "count target objects in the given list. It is useful when you want to count the number of objects in the image" + }, + { + "name": "count_masks", + "description": "count target mask in the given list." + }, + { + "name": "video_captioning", + "description": "Generate a caption or description for video. It can generate a detailed description that can be used for video perception and video generation. For example: a) you can use this tool when you want to know what happened in the video\"; and b) when you want to generate tags for input video, you can use translate description obtained from `image_captioning` into tags." + }, + { + "name": "video_classification", + "description": "Classify the video and detect the actions in the video." + }, + { + "name": "frame_captioning", + "description": "Generate a caption or description for video every n seconds. It can generate a detailed description that can be used for video perception and video generation. For example: a) you can use this tool when you want to know what happened in the video\"; and b) when you want to generate tags for input video, you can use translate description obtained from `image_captioning` into tags." + }, + { + "name": "frame_classification", + "description": "Classify the video and detect the actions in the every n seconds." + }, + { + "name": "text_to_text_generation", + "description": "Text to text generation. It can be used for sentence acceptability judgment, Sentiment analysis, Paraphrasing/sentence similarity, Natural language inference, Sentence completion, Word sense disambiguation, Question answering." + }, + { + "name": "openai_chat_model", + "description": "Answer the question by Large Language Model. It is useful for tasks such as generating content, answering questions, engaging in conversations and providing explanations. However, it still has some limitations. For example, it can not directly access the up-to-date information like time, weather, etc." + }, + { + "name": "summarization", + "description": "Summarize sentences, long narratives, articles, papers, textbooks." + }, + { + "name": "text_to_tags", + "description": "Predict the tags of text, article and papers by using the their textual content as input" + }, + { + "name": "sentiment_analysis", + "description": "Sentiment analysis is the process of analyzing digital text to determine if the emotional tone of the message is positive, negative, or neutral." + } +] + +input_keys: [ + { "image": "image file"}, + { "prompt": "text"}, + { "video": "video file"}, + { "context": "context for text qa"}, + { "tool": "tool name mentioned above"}, +] + +output_keys: [ + { "labels": "list of labels for image tasks"}, + { "scores": "list of scores for image and text tasks"}, + { "bboxes": "list of bounding boxes for detection tasks"}, + { "masks": "list of masks for segmentation tasks"}, + { "text": "list of text for text tasks"}, + { "frames": "list of frame numbers for video tasks"}, +] + + + diff --git a/vision_agent/tools/tools.py b/vision_agent/tools/tools.py index f13c14dd..a9ded1be 100644 --- a/vision_agent/tools/tools.py +++ b/vision_agent/tools/tools.py @@ -1,5 +1,6 @@ import logging import tempfile +import os from abc import ABC from collections import Counter as CounterClass from pathlib import Path @@ -139,7 +140,7 @@ class GroundingDINO(Tool): 'scores': [0.98, 0.02]}] """ - _ENDPOINT = "https://chnicr4kes5ku77niv2zoytggq0qyqlp.lambda-url.us-east-2.on.aws" + _ENDPOINT = "https://soi4ewr6fjqqdf5vuss6rrilee0kumxq.lambda-url.us-east-2.on.aws" name = "grounding_dino_" description = "'grounding_dino_' is a tool that can detect arbitrary objects with inputs such as category names or referring expressions." @@ -182,11 +183,15 @@ def __call__(self, prompt: str, image: Union[str, Path, ImageType]) -> Dict: image_b64 = convert_to_b64(image) data = { "prompt": prompt, - "images": [image_b64], + "images": image_b64, + "tool": "visual_grounding", } res = requests.post( self._ENDPOINT, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}", + }, json=data, ) resp_json: Dict[str, Any] = res.json() @@ -230,7 +235,7 @@ class GroundingSAM(Tool): [1, 1, 1, ..., 1, 1, 1]], dtype=uint8)]}] """ - _ENDPOINT = "https://cou5lfmus33jbddl6hoqdfbw7e0qidrw.lambda-url.us-east-2.on.aws" + _ENDPOINT = "https://model-owp50nlq.api.baseten.co/production/predict" name = "grounding_sam_" description = "'grounding_sam_' is a tool that can detect and segment arbitrary objects with inputs such as category names or referring expressions." @@ -274,10 +279,14 @@ def __call__(self, prompt: List[str], image: Union[str, ImageType]) -> Dict: data = { "classes": prompt, "image": image_b64, + "tool": "visual_grounding_segment", } res = requests.post( self._ENDPOINT, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "Authorization": f"Api-Key {os.environ['BASETEN_API_KEY']}", + }, json=data, ) resp_json: Dict[str, Any] = res.json()