From 80def1252e5ad462d9b6bfecf745860f5dc636d8 Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 14:24:02 -0700 Subject: [PATCH 1/7] Update prompts.py --- vision_agent/tools/prompts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vision_agent/tools/prompts.py b/vision_agent/tools/prompts.py index 0488c3f2..2de05bd0 100644 --- a/vision_agent/tools/prompts.py +++ b/vision_agent/tools/prompts.py @@ -5,15 +5,15 @@ "This is an API tool documentation. Given a user's question, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n" "This is the API tool documentation: {api_doc}\n" "Please note that: \n" - "1. The Example in the API tool documentation can help you better understand the use of the API.\n" - '2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}\n' + "1. The example in the API tool documentation can help you better understand the use of the API.\n" + '2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no parameters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}\n' "3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n" '4. If you need to use this API multiple times, please set "Parameters" to a list.\n' - "5. You must ONLY output in a parsible JSON format. Two examples output looks like:\n" + "5. You must ONLY output in a parsible JSON format. Two example outputs look like:\n" "'''\n" 'Example 1: {{"Parameters":{{"keyword": "Artificial Intelligence", "language": "English"}}}}\n' 'Example 2: {{"Parameters":[{{"keyword": "Artificial Intelligence", "language": "English"}}, {{"keyword": "Machine Learning", "language": "English"}}]}}\n' "'''\n" - "This is user's question: {question}\n" + "This is the user's question: {question}\n" "Output:\n" ) From 9a344033de5ff085c323bdd16891a4b069e14dc3 Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 14:36:10 -0700 Subject: [PATCH 2/7] Update vision_agent_prompts.py --- vision_agent/agent/vision_agent_prompts.py | 42 +++++++++++----------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/vision_agent/agent/vision_agent_prompts.py b/vision_agent/agent/vision_agent_prompts.py index d5d2b1ef..8f131bad 100644 --- a/vision_agent/agent/vision_agent_prompts.py +++ b/vision_agent/agent/vision_agent_prompts.py @@ -1,4 +1,4 @@ -VISION_AGENT_REFLECTION = """You are an advanced reasoning agent that can improve based on self refection. You will be given a previous reasoning trial in which you were given the user's question, the available tools that the agent has, the decomposed tasks and tools that the agent used to answer the question and the final answer the agent provided. You must determine if the agent's answer was correct or incorrect. If the agent's answer was correct, respond with Finish. If the agent's answer was incorrect, you must diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, high level plan that aims to mitigate the same failure with the tools available. Use complete sentences. +VISION_AGENT_REFLECTION = """You are an advanced reasoning agent that can improve based on self-refection. You will be given a previous reasoning trial in which you were given the user's question, the available tools that the agent has, the decomposed tasks and tools that the agent used to answer the question and the final answer the agent provided. You must determine if the agent's answer was correct or incorrect. If the agent's answer was correct, respond with Finish. If the agent's answer was incorrect, you must diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, high level plan that aims to mitigate the same failure with the tools available. Use complete sentences. User's question: {question} @@ -13,14 +13,14 @@ Reflection: """ -TASK_DECOMPOSE = """You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step. +TASK_DECOMPOSE = """You need to decompose a user's complex question into some simple subtasks and let the model execute it step by step. This is the user's question: {question} -This is tool list: +This is the tool list: {tools} Please note that: -1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using one single tool in the tool list. -2. If one subtask need the results from other subtask, you can should write clearly. For example: +1. You should only decompose this user's complex question into some simple subtasks which can be executed easily by using one single tool in the tool list. +2. If one subtask needs the results from other subtask, you should write clearly. For example: {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}} 3. You must ONLY output in a parsible JSON format. An example output looks like: @@ -28,18 +28,18 @@ Output: """ -TASK_DECOMPOSE_DEPENDS = """You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step. +TASK_DECOMPOSE_DEPENDS = """You need to decompose a user's complex question into some simple subtasks and let the model execute it step by step. This is the user's question: {question} -This is tool list: +This is the tool list: {tools} This is a reflection from a previous failed attempt: {reflections} Please note that: -1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using one single tool in the tool list. -2. If one subtask need the results from other subtask, you can should write clearly. For example: +1. You should only decompose this user's complex question into some simple subtasks which can be executed easily by using one single tool in the tool list. +2. If one subtask needs the results from another subtask, you should write clearly. For example: {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}} 3. You must ONLY output in a parsible JSON format. An example output looks like: @@ -53,7 +53,7 @@ {tools} Please note that: -1. You should only chooce one tool the Tool List to solve this question. +1. You should only choose one tool from the tool list to solve this question. 2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like: Example 1: {{"ID": 1}} @@ -70,7 +70,7 @@ {reflections} Please note that: -1. You should only chooce one tool the Tool List to solve this question. +1. You should only choose one tool from the tool list to solve this question. 2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like: Example 1: {{"ID": 1}} @@ -78,14 +78,14 @@ Output: """ -CHOOSE_PARAMETER_DEPENDS = """Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question. +CHOOSE_PARAMETER_DEPENDS = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question. Please note that: -1. The Example in the API tool documentation can help you better understand the use of the API. -2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}} +1. The examples in the API tool documentation can help you better understand the use of the API. +2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}} 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs. 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference. -5. If you need to use this API multiple times,, please set "Parameters" to a list. -6. You must ONLY output in a parsible JSON format. Two examples output looks like: +5. If you need to use this API multiple times, please set "Parameters" to a list. +6. You must ONLY output in a parsible JSON format. Two example outputs look like: Example 1: {{"Parameters":{{"input": [1,2,3]}}}} Example 2: {{"Parameters":[{{"input": [1,2,3]}}, {{"input": [2,3,4]}}]}} @@ -93,16 +93,16 @@ This is a reflection from a previous failed attempt: {reflections} -There are logs of previous questions and answers: +These are logs of previous questions and answers: {previous_log} This is the current user's question: {question} -This is API tool documentation: {tool_usage} +This is the API tool documentation: {tool_usage} Output: """ ANSWER_GENERATE_DEPENDS = """You should answer the question based on the response output by the API tool. Please note that: -1. Try to organize the response into a natural language answer. +1. You should try to organize the response into a natural language answer. 2. We will not show the API response to the user, thus you need to make full use of the response and give the information in the response that can satisfy the user's question in as much detail as possible. 3. If the API tool does not provide useful information in the response, please answer with your knowledge. 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers. @@ -110,7 +110,7 @@ This is a reflection from a previous failed attempt: {reflections} -There are logs of previous questions and answers: +These are logs of previous questions and answers: {previous_log} This is the user's question: {question} @@ -121,7 +121,7 @@ We will not show the API response to the user, thus you need to make full use of the response and give the information in the response that can satisfy the user's question in as much detail as possible. Output: """ -ANSWER_SUMMARIZE_DEPENDS = """We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. You need to organize these answers to each subtask and form a self-consistent final answer to the user's question +ANSWER_SUMMARIZE_DEPENDS = """We break down a user's complex problems into simple subtasks and provide answers to each simple subtask. You need to organize these answers to each subtask and form a self-consistent final answer to the user's question This is the user's question: {question} These are subtasks and their answers: From c1883b54fcfda31d420851cdd17722c1ab666a29 Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 14:53:39 -0700 Subject: [PATCH 3/7] Update reflexion_prompts.py --- vision_agent/agent/reflexion_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision_agent/agent/reflexion_prompts.py b/vision_agent/agent/reflexion_prompts.py index eb4c3d4d..b91af78e 100644 --- a/vision_agent/agent/reflexion_prompts.py +++ b/vision_agent/agent/reflexion_prompts.py @@ -9,7 +9,7 @@ Question: {question}{scratchpad}""" -COT_REFLECT_INSTRUCTION = """You are an advanced reasoning agent that can improve based on self refection. You will be given a previous reasoning trial in which you were given access to relevant context and a question to answer. You were unsuccessful in answering the question either because you guessed the wrong answer with Finish[] or there is a phrasing discrepancy with your provided answer and the answer key. In a few sentences, Diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, high level plan that aims to mitigate the same failure. Use complete sentences. +COT_REFLECT_INSTRUCTION = """You are an advanced reasoning agent that can improve based on self-refection. You will be given a previous reasoning trial in which you were given access to relevant context and a question to answer. You were unsuccessful in answering the question either because you guessed the wrong answer with Finish[] or there is a phrasing discrepancy with your provided answer and the answer key. In a few sentences, diagnose a possible reason for failure or phrasing discrepancy and devise a new, concise, high level plan that aims to mitigate the same failure. Use complete sentences. Here are some examples: {examples} (END OF EXAMPLES) From b2bf29482cb1210361e321dbfe98364166bbbcbe Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 14:57:10 -0700 Subject: [PATCH 4/7] Update vision_agent_prompts.py --- vision_agent/agent/vision_agent_prompts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vision_agent/agent/vision_agent_prompts.py b/vision_agent/agent/vision_agent_prompts.py index 8f131bad..980f63c4 100644 --- a/vision_agent/agent/vision_agent_prompts.py +++ b/vision_agent/agent/vision_agent_prompts.py @@ -53,7 +53,7 @@ {tools} Please note that: -1. You should only choose one tool from the tool list to solve this question. +1. You should only choose one tool from the Tool List to solve this question. 2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like: Example 1: {{"ID": 1}} @@ -70,7 +70,7 @@ {reflections} Please note that: -1. You should only choose one tool from the tool list to solve this question. +1. You should only choose one tool from the Tool List to solve this question. 2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like: Example 1: {{"ID": 1}} @@ -80,7 +80,7 @@ CHOOSE_PARAMETER_DEPENDS = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question. Please note that: -1. The examples in the API tool documentation can help you better understand the use of the API. +1. The Example in the API tool documentation can help you better understand the use of the API. 2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}} 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs. 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference. From d1e0c9adb89f3a929ad1b99b04e12d106544b987 Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 15:02:35 -0700 Subject: [PATCH 5/7] Update easytool_prompts.py --- vision_agent/agent/easytool_prompts.py | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/vision_agent/agent/easytool_prompts.py b/vision_agent/agent/easytool_prompts.py index 445e42b3..6e20dc17 100644 --- a/vision_agent/agent/easytool_prompts.py +++ b/vision_agent/agent/easytool_prompts.py @@ -1,11 +1,11 @@ -TASK_DECOMPOSE = """You need to decompose a complex user's question into some simple subtasks and let the model execute it step by step. +TASK_DECOMPOSE = """You need to decompose a user's complex question into some simple subtasks and let the model execute it step by step. This is the user's question: {question} -This is tool list: +This is the tool list: {tools} Please note that: 1. You should only decompose this complex user's question into some simple subtasks which can be executed easily by using one single tool in the tool list. -2. If one subtask need the results from other subtask, you can should write clearly. For example: +2. If one subtask needs the results from another subtask, you should write clearly. For example: {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}} 3. You must ONLY output in a parsible JSON format. An example output looks like: @@ -13,7 +13,7 @@ Output: """ -TASK_TOPOLOGY = """Given a complex user's question, I have decompose this question into some simple subtasks. I think there exists a logical connections and order amontg the tasks. Thus you need to help me output this logical connections and order. +TASK_TOPOLOGY = """Given a user's complex question, I have decomposed this question into some simple subtasks. I think there exist logical connections and order among the tasks. Thus, you need to help me output these logical connections and order. You must ONLY output in a parsible JSON format with the following format: {{"Tasks": [{{"task": task, "id", task_id, "dep": [dependency_task_id1, dependency_task_id2, ...]}}]}} @@ -21,7 +21,7 @@ The "dep" field denotes the id of the previous task which generates a new resource upon which the current task depends. If there are no dependencies, set "dep" to -1. -This is user's question: {question} +This is the user's question: {question} These are subtasks of this question: @@ -34,7 +34,7 @@ {tools} Please note that: -1. You should only chooce one tool the Tool List to solve this question. +1. You should only choose one tool from the Tool List to solve this question. 2. You must ONLY output the ID of the tool you chose in a parsible JSON format. Two example outputs look like: Example 1: {{"ID": 1}} @@ -42,22 +42,22 @@ Output: """ -CHOOSE_PARAMETER = """Given a user's question and a API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question. +CHOOSE_PARAMETER = """Given a user's question and an API tool documentation, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question. Please note that: 1. The Example in the API tool documentation can help you better understand the use of the API. -2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}} +2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no paremters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}} 3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs. 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers for your reference. -5. If you need to use this API multiple times,, please set "Parameters" to a list. -6. You must ONLY output in a parsible JSON format. Two examples output looks like: +5. If you need to use this API multiple times, please set "Parameters" to a list. +6. You must ONLY output in a parsible JSON format. Two example outputs looks like: Example 1: {{"Parameters":{{"input": [1,2,3]}}}} Example 2: {{"Parameters":[{{"input": [1,2,3]}}, {{"input": [2,3,4]}}]}} -There are logs of previous questions and answers: +These are logs of previous questions and answers: {previous_log} This is the current user's question: {question} -This is API tool documentation: {tool_usage} +This is the API tool documentation: {tool_usage} Output: """ @@ -67,7 +67,7 @@ 2. We will not show the API response to the user, thus you need to make full use of the response and give the information in the response that can satisfy the user's question in as much detail as possible. 3. If the API tool does not provide useful information in the response, please answer with your knowledge. 4. The question may have dependencies on answers of other questions, so we will provide logs of previous questions and answers. -There are logs of previous questions and answers: +These are logs of previous questions and answers: {previous_log} This is the user's question: {question} This is the response output by the API tool: @@ -75,7 +75,7 @@ We will not show the API response to the user, thus you need to make full use of the response and give the information in the response that can satisfy the user's question in as much detail as possible. Output: """ -ANSWER_SUMMARIZE = """We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. You need to organize these answers to each subtask and form a self-consistent final answer to the user's question +ANSWER_SUMMARIZE = """We break down a complex user's problems into simple subtasks and provide answers to each simple subtask. You need to organize these answers to each subtask and form a self-consistent final answer to the user's question. This is the user's question: {question} These are subtasks and their answers: {answers} Final answer: """ From a92fdbb53e03e9b8e1aa834c671b43ea991ff63a Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 15:04:45 -0700 Subject: [PATCH 6/7] Update prompts.py --- vision_agent/tools/prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision_agent/tools/prompts.py b/vision_agent/tools/prompts.py index 2de05bd0..b678ad45 100644 --- a/vision_agent/tools/prompts.py +++ b/vision_agent/tools/prompts.py @@ -5,7 +5,7 @@ "This is an API tool documentation. Given a user's question, you need to output parameters according to the API tool documentation to successfully call the API to solve the user's question.\n" "This is the API tool documentation: {api_doc}\n" "Please note that: \n" - "1. The example in the API tool documentation can help you better understand the use of the API.\n" + "1. The Example in the API tool documentation can help you better understand the use of the API.\n" '2. Ensure the parameters you output are correct. The output must contain the required parameters, and can contain the optional parameters based on the question. If there are no parameters in the required parameters and optional parameters, just leave it as {{"Parameters":{{}}}}\n' "3. If the user's question mentions other APIs, you should ONLY consider the API tool documentation I give and do not consider other APIs.\n" '4. If you need to use this API multiple times, please set "Parameters" to a list.\n' From db396c4e948c3081cd761910eb999f52110250d7 Mon Sep 17 00:00:00 2001 From: Cameron Maloney Date: Tue, 9 Apr 2024 15:11:13 -0700 Subject: [PATCH 7/7] Update vision_agent_prompts.py --- vision_agent/agent/vision_agent_prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vision_agent/agent/vision_agent_prompts.py b/vision_agent/agent/vision_agent_prompts.py index 980f63c4..f451a66e 100644 --- a/vision_agent/agent/vision_agent_prompts.py +++ b/vision_agent/agent/vision_agent_prompts.py @@ -20,7 +20,7 @@ Please note that: 1. You should only decompose this user's complex question into some simple subtasks which can be executed easily by using one single tool in the tool list. -2. If one subtask needs the results from other subtask, you should write clearly. For example: +2. If one subtask needs the results from another subtask, you should write clearly. For example: {{"Tasks": ["Convert 23 km/h to X km/min by 'divide_'", "Multiply X km/min by 45 min to get Y by 'multiply_'"]}} 3. You must ONLY output in a parsible JSON format. An example output looks like: