diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/defining_a_tool_in_code.py b/examples/code/home/build-tools/what-is-sent-to-language-models/defining_a_tool_in_code.py new file mode 100644 index 00000000..d900f18a --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/defining_a_tool_in_code.py @@ -0,0 +1,32 @@ +from enum import Enum +from typing import Annotated, Optional + +from arcade.sdk.auth import Slack +from arcade.sdk import ToolContext, tool + + +class ConversationType(str, Enum): + # Enum values are sent to the model + PUBLIC_CHANNEL = "public_channel" + PRIVATE_CHANNEL = "private_channel" + MULTI_PERSON_DIRECT_MESSAGE = "multi_person_direct_message" + DIRECT_MESSAGE = "direct_message" + + +@tool( + requires_auth=Slack( + scopes=["channels:read", "groups:read", "im:read", "mpim:read"], + ) # Not sent to the model +) +async def list_conversations_metadata( # Tool name and toolkit name is sent to the model + context: ToolContext, # ToolContext is never sent to the model + conversation_types: Annotated[Optional[list[ConversationType]], "The type(s) of conversations to list. Defaults to all types."] = None, # Sent to the model + limit: Annotated[Optional[int], "The maximum number of conversations to list."] = None, # Sent to the model + next_cursor: Annotated[Optional[str], "The cursor to use for pagination." = None, # Sent to the model +) -> Annotated[dict,"The conversations metadata list and a pagination 'next_cursor', if there are more conversations to retrieve."]: # Not sent to the model + """ + List metadata for Slack conversations (channels and/or direct messages) that the user + is a member of. + """ # Docstring is sent to the model + # The body of the function is not sent to the model + ... diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.js b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.js new file mode 100644 index 00000000..b63bf122 --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.js @@ -0,0 +1,10 @@ +import Arcade from "@arcadeai/arcadejs"; + +const client = new Arcade(); + +// Get what is sent to the language model for a specific tool +const listConversationsMetadata = await client.tools.formatted.get("Slack.ListConversationsMetadata", + { format: "openai" } +); + +console.log(listConversationsMetadata); diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.py b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.py new file mode 100644 index 00000000..c31474cc --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.py @@ -0,0 +1,8 @@ +from arcadepy import Arcade + +client = Arcade() +list_conversations_metadata = client.tools.formatted.get( + name="Slack.ListConversationsMetadata", format="openai" +) + +print(list_conversations_metadata) diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.sh b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.sh new file mode 100644 index 00000000..c3ab097b --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.sh @@ -0,0 +1,3 @@ +curl -X 'GET' 'https://api.arcade.dev/v1/formatted_tools/Slack.ListConversationsMetadata?format=openai' \ + -H 'Accept: application/json' \ + -H 'Authorization: Bearer {{arcade_api_key}}' diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai_output.json b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai_output.json new file mode 100644 index 00000000..be26e247 --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai_output.json @@ -0,0 +1,34 @@ +{ + "type": "function", + "function": { + "name": "Slack_ListConversationsMetadata", + "description": "List metadata for Slack conversations (channels and/or direct messages) that the user\nis a member of.", + "parameters": { + "type": "object", + "properties": { + "conversation_types": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "public_channel", + "private_channel", + "multi_person_direct_message", + "direct_message" + ] + }, + "description": "The type(s) of conversations to list. Defaults to all types." + }, + "limit": { + "type": "integer", + "description": "The maximum number of conversations to list." + }, + "next_cursor": { + "type": "string", + "description": "The cursor to use for pagination." + } + }, + "required": [] + } + } +} diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.py b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.py new file mode 100644 index 00000000..cd6e3611 --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.py @@ -0,0 +1,8 @@ +from arcade_slack.tools.chat import list_conversations_metadata + +from arcade.sdk import ToolCatalog + +tool_def = ToolCatalog.create_tool_definition(list_conversations_metadata, "Slack") +tool_def_json = tool_def.model_dump_json() + +print(tool_def_json) diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.sh b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.sh new file mode 100644 index 00000000..5879d3fd --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.sh @@ -0,0 +1,3 @@ +curl -X 'GET' 'https://api.arcade.dev/v1/tools/Slack.ListConversationsMetadata' \ + -H 'Accept: application/json' \ + -H 'Authorization: Bearer {{arcade_api_key}}' diff --git a/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition_output.json b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition_output.json new file mode 100644 index 00000000..65a277e1 --- /dev/null +++ b/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition_output.json @@ -0,0 +1,63 @@ +{ + "name": "ListConversationsMetadata", + "fully_qualified_name": "Slack.ListConversationsMetadata", + "description": "List metadata for Slack conversations (channels and/or direct messages) that the user\nis a member of.", + "toolkit": { "name": "Slack", "description": null, "version": null }, + "input": { + "parameters": [ + { + "name": "conversation_types", + "required": false, + "description": "The type(s) of conversations to list. Defaults to all types.", + "value_schema": { + "val_type": "array", + "inner_val_type": "string", + "enum": [ + "public_channel", + "private_channel", + "multi_person_direct_message", + "direct_message" + ] + }, + "inferrable": true + }, + { + "name": "limit", + "required": false, + "description": "The maximum number of conversations to list.", + "value_schema": { + "val_type": "integer", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "next_cursor", + "required": false, + "description": "The cursor to use for pagination.", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "description": "The conversations metadata list and a pagination 'next_cursor', if there are more conversations to retrieve.", + "available_modes": ["value", "error"], + "value_schema": { "val_type": "json", "inner_val_type": null, "enum": null } + }, + "requirements": { + "authorization": { + "provider_id": "slack", + "provider_type": "oauth2", + "id": null, + "oauth2": { + "scopes": ["channels:read", "groups:read", "im:read", "mpim:read"] + } + } + } +} diff --git a/pages/home/build-tools/what-is-sent-to-language-models.mdx b/pages/home/build-tools/what-is-sent-to-language-models.mdx new file mode 100644 index 00000000..7cdd30ff --- /dev/null +++ b/pages/home/build-tools/what-is-sent-to-language-models.mdx @@ -0,0 +1,94 @@ +--- +title: "What is sent to language models?" +description: "Documentation for what is sent to the language model when defining a tool" +--- + +import { Tabs } from "nextra/components"; + +# Understanding what parts of a tool are sent to language models + +Which parts of your tool definitions are sent to language models? By understanding this, you can ensure that your tools effectively communicate their purpose and usage to the models. + +The Arcade worker converts tools (functions with the `@tool` decorator) into `ToolDefinition` objects. The `ToolDefinition` object can be thought of as the intermediate representation of a tool before it is formatted for a specific model provider and sent to the model. + +## Defining a tool in code + +First, let's take a look at a tool written in Python before it is converted into a `ToolDefinition` object. + +```python file=/examples/code/home/build-tools/what-is-sent-to-language-models/defining_a_tool_in_code.py + +``` + +## A tool as a `ToolDefinition` object + +As mentioned above, the Arcade worker converts tools into `ToolDefinition` objects. Let's take a look at the `list_conversations_metadata` tool as a `ToolDefinition` object (the intermediate representation of the tool before it is formatted for a specific model). + + + +```python file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.py + +``` + + + +```bash file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition.sh + +``` + + + +
+ + + +```json file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_tool_definition_output.json + +``` + + + +## Formatting tools for model compatibility + +Before tools are sent to models, the `ToolDefinition` object is formatted such that it is compatible with the specific model provider's function calling format. For example, let's take a look at the `list_conversations_metadata` tool formatted for OpenAI. + + + + + +```python file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.py +``` + + + + + +```js file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.js +``` + + + + + +```bash file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai.sh +``` + + + + + + + +```json file=/examples/code/home/build-tools/what-is-sent-to-language-models/get_formatted_tool_oai_output.json +``` + + + +## Additional Information + +- _**Defaults:**_ The default value for parameters listed as optional is not sent to the language model. +- _**Enum Values:**_ If a parameter is an enum, each possible value for the enum is sent to the language model. This provides clarity on the valid options for that parameter. +- _**Annotations:**_ The annotation for each parameter is sent to the language model, thus each parameter must be annotated. +- _**Tool Definition Schema:**_ Check out the [Tool Definition JSON schema](https://github.com/ArcadeAI/arcade-ai/blob/main/schemas/preview/tool_definition.schema.jsonc) for more details on how to define your tools. + + +In this example, we've learned that tools are converted into `ToolDefinition` objects and then formatted for specific models. Understanding that a tool is converted into a `ToolDefinition` object and then formatted for a specific model is important because it informs you, the tool developer, what parts of the tool are sent to the language model.