updated reflection prompt, add tool usage doc

vision_agent/agent/vision_agent.py

@@ -37,10 +37,10 @@
 
 def parse_json(s: str) -> Any:
  s = (
- s.replace(": true", ": True")
- .replace(": false", ": False")
- .replace(":true", ": True")
- .replace(":false", ": False")
+ s.replace(": True", ": true")
+ .replace(": False", ": false")
+ .replace(":True", ": true")
+ .replace(":False", ": false")
  .replace("```", "")
  .strip()
  )
@@ -62,6 +62,19 @@ def format_tools(tools: Dict[int, Any]) -> str:
  return tool_str
 
 
+def format_tool_usage(tools: Dict[int, Any], tool_result: List[Dict]) -> str:
+ usage = []
+ name_to_usage = {v["name"]: v["usage"] for v in tools.values()}
+ for tool_res in tool_result:
+ if "tool_name" in tool_res:
+ usage.append((tool_res["tool_name"], name_to_usage[tool_res["tool_name"]]))
+
+ usage_str = ""
+ for tool_name, tool_usage in usage:
+ usage_str += f"{tool_name} - {tool_usage}\n"
+ return usage_str
+
+
 def topological_sort(tasks: List[Dict]) -> List[Dict]:
  in_degree = {task["id"]: 0 for task in tasks}
  for task in tasks:
@@ -256,6 +269,7 @@ def self_reflect(
  prompt = VISION_AGENT_REFLECTION.format(
  question=question,
  tools=format_tools({k: v["description"] for k, v in tools.items()}),
+ tool_usage=format_tool_usage(tools, tool_result),
  tool_results=str(tool_result),
  final_answer=final_answer,
  )
@@ -269,6 +283,7 @@ def self_reflect(
 
 
 def parse_reflect(reflect: str) -> Any:
+ reflect = reflect.strip()
  try:
  return parse_json(reflect)
  except Exception:

vision_agent/agent/vision_agent_prompts.py

@@ -1,13 +1,14 @@
-VISION_AGENT_REFLECTION = """You are an advanced reasoning agent that can improve based on self-refection. You will be given a previous reasoning trial in which you were given the user's question, the available tools that the agent has, the decomposed tasks and tools that the agent used to answer the question and the final answer the agent provided. You may also receive an image with the visualized bounding boxes or masks with their associated labels and scores from the tools used. 
+VISION_AGENT_REFLECTION = """You are an advanced reasoning agent that can improve based on self-refection. You will be given a previous reasoning trial in which you were given the user's question, the available tools that the agent has, the decomposed tasks and tools that the agent used to answer the question and the final answer the agent provided. You may also receive an image with the visualized bounding boxes or masks with their associated labels and scores from the tools used.
 
 Please note that:
 1. You must ONLY output parsible JSON format. If the agents output was correct set "Finish" to true, else set "Finish" to false. An example output looks like:
 {{"Finish": true, "Reflection": "The agent's answer was correct."}}
-2. If the agent's answer was incorrect, you must diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, concrete plan that aims to mitigate the same failure with the tools available. An example output looks like:
- {{"Finish": false, "Reflection": "The agent's answer was incorrect. The agent should use the following tools with the following parameters:
- Step 1: Use 'grounding_dion_' with a 'prompt' of 'baby. bed' and a 'box_threshold' of 0.7 to reduce the false positives.
+2. You must utilize the image with the visualized bounding boxes or masks and determine if the tools were used correctly or, using your own judgement, utilized incorrectly.
+3. If the agent's answer was incorrect, you must diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, concrete plan that aims to mitigate the same failure with the tools available. An example output looks like:
+ {{"Finish": false, "Reflection": "I can see from teh visualized bounding boxes that the agent's answer was incorrect because the grounding_dino_ tool produced false positive predictions. The agent should use the following tools with the following parameters:
+ Step 1: Use 'grounding_dino_' with a 'prompt' of 'baby. bed' and a 'box_threshold' of 0.7 to reduce the false positives.
  Step 2: Use 'box_iou_' with the baby bounding box and the bed bounding box to determine if the baby is on the bed or not."}}
-3. If the task cannot be completed with the existing tools, set "Finish" to true.
+4. If the task cannot be completed with the existing tools or by adjusting the parameters, set "Finish" to true.
 
 User's question: {question}
 
@@ -17,6 +18,9 @@
 Tasks and tools used:
 {tool_results}
 
+Tool's used API documentation:
+{tool_usage}
+
 Final answer:
 {final_answer}