Fix Extract Files (#274)

* move extract files to exec code

* made fine tune test case looser

* added if guard test case

tests/integ/test_tools.py

@@ -133,7 +133,7 @@ def test_florence2_phrase_grounding_fine_tune_id():
  fine_tune_id=FINE_TUNE_ID,
  )
  # this calls a fine-tuned florence2 model which is going to be worse at this task
- assert 14 <= len(result) <= 26
+ assert 13 <= len(result) <= 26
  assert [res["label"] for res in result] == ["coin"] * len(result)
  assert all([all([0 <= x <= 1 for x in obj["bbox"]]) for obj in result])
 

tests/unit/test_vac.py

@@ -141,3 +141,16 @@ def check_helmets(image_path):
 # The function can be called with the image path"""
  code_out = strip_function_calls(code, exclusions=["register_heif_opener"])
  assert code_out == expected_code
+
+
+def test_strip_function_call_if_name_equal_main():
+ code = """import os
+def f():
+ print("Hello!")
+if __name__ == "__main__":
+ f()"""
+ expected_code = """import os
+def f():
+ print("Hello!")"""
+ code_out = strip_function_calls(code)
+ assert code_out == expected_code

vision_agent/agent/vision_agent.py

@@ -149,11 +149,11 @@ def execute_code_action(
  result = code_interpreter.exec_isolation(
  BoilerplateCode.add_boilerplate(code, remote_path=artifact_remote_path)
  )
- extract_and_save_files_to_artifacts(artifacts, code)
 
  obs = str(result.logs)
  if result.error:
  obs += f"\n{result.error}"
+ extract_and_save_files_to_artifacts(artifacts, code, obs)
  return result, obs
 
 
@@ -182,6 +182,7 @@ def execute_user_code_action(
  )
  if user_result.error:
  user_obs += f"\n{user_result.error}"
+ extract_and_save_files_to_artifacts(artifacts, user_code_action, user_obs)
  return user_result, user_obs
 
 

vision_agent/tools/meta_tools.py

@@ -451,11 +451,6 @@ def detect_dogs(image_path: str):
  custom_tool_names=custom_tool_names,
  )
 
- # capture and save any files that were saved in the code to the artifacts
- extract_and_save_files_to_artifacts(
- artifacts, response["code"] + "\n" + response["test"]
- )
-
  redisplay_results(response["test_result"])
  code = response["code"]
  artifacts[name] = code
@@ -536,10 +531,6 @@ def detect_dogs(image_path: str):
  test_multi_plan=False,
  custom_tool_names=custom_tool_names,
  )
- # capture and save any files that were saved in the code to the artifacts
- extract_and_save_files_to_artifacts(
- artifacts, response["code"] + "\n" + response["test"]
- )
 
  redisplay_results(response["test_result"])
  code = response["code"]
@@ -766,7 +757,9 @@ def use_object_detection_fine_tuning(
  return diff
 
 
-def extract_and_save_files_to_artifacts(artifacts: Artifacts, code: str) -> None:
+def extract_and_save_files_to_artifacts(
+ artifacts: Artifacts, code: str, obs: str
+) -> None:
  """Extracts and saves files used in the code to the artifacts object.
 
  Parameters:
@@ -776,12 +769,16 @@ def extract_and_save_files_to_artifacts(artifacts: Artifacts, code: str) -> None
  try:
  response = extract_json(
  AnthropicLMM()( # type: ignore
- f"""You are a helpful AI assistant. Your job is to look at a snippet of code and return the file paths that are being saved in the file. Below is the code snippet:
+ f"""You are a helpful AI assistant. Your job is to look at a snippet of code and the output of running that code and return the file paths that are being saved in the file. Below is the code snippet:
 
 ```python
 {code}
 ```
 
+```output
+{obs}
+```
+
 Return the file paths in the following JSON format:
 {{"file_paths": ["/path/to/image1.jpg", "/other/path/to/data.json"]}}"""
  )