move extract files to exec code

vision_agent/agent/vision_agent.py

@@ -149,11 +149,11 @@ def execute_code_action(
  result = code_interpreter.exec_isolation(
  BoilerplateCode.add_boilerplate(code, remote_path=artifact_remote_path)
  )
- extract_and_save_files_to_artifacts(artifacts, code)
 
  obs = str(result.logs)
  if result.error:
  obs += f"\n{result.error}"
+ extract_and_save_files_to_artifacts(artifacts, code, obs)
  return result, obs
 
 
@@ -182,6 +182,7 @@ def execute_user_code_action(
  )
  if user_result.error:
  user_obs += f"\n{user_result.error}"
+ extract_and_save_files_to_artifacts(artifacts, user_code_action, user_obs)
  return user_result, user_obs
 
 

vision_agent/tools/meta_tools.py

@@ -451,11 +451,6 @@ def detect_dogs(image_path: str):
  custom_tool_names=custom_tool_names,
  )
 
- # capture and save any files that were saved in the code to the artifacts
- extract_and_save_files_to_artifacts(
- artifacts, response["code"] + "\n" + response["test"]
- )
-
  redisplay_results(response["test_result"])
  code = response["code"]
  artifacts[name] = code
@@ -536,10 +531,6 @@ def detect_dogs(image_path: str):
  test_multi_plan=False,
  custom_tool_names=custom_tool_names,
  )
- # capture and save any files that were saved in the code to the artifacts
- extract_and_save_files_to_artifacts(
- artifacts, response["code"] + "\n" + response["test"]
- )
 
  redisplay_results(response["test_result"])
  code = response["code"]
@@ -766,7 +757,9 @@ def use_object_detection_fine_tuning(
  return diff
 
 
-def extract_and_save_files_to_artifacts(artifacts: Artifacts, code: str) -> None:
+def extract_and_save_files_to_artifacts(
+ artifacts: Artifacts, code: str, obs: str
+) -> None:
  """Extracts and saves files used in the code to the artifacts object.
 
  Parameters:
@@ -776,12 +769,16 @@ def extract_and_save_files_to_artifacts(artifacts: Artifacts, code: str) -> None
  try:
  response = extract_json(
  AnthropicLMM()( # type: ignore
- f"""You are a helpful AI assistant. Your job is to look at a snippet of code and return the file paths that are being saved in the file. Below is the code snippet:
+ f"""You are a helpful AI assistant. Your job is to look at a snippet of code and the output of running that code and return the file paths that are being saved in the file. Below is the code snippet:
 
 ```python
 {code}
 ```
 
+```output
+{obs}
+```
+
 Return the file paths in the following JSON format:
 {{"file_paths": ["/path/to/image1.jpg", "/other/path/to/data.json"]}}"""
  )