diff --git a/lib/chat_models/chat_vertex_ai.ex b/lib/chat_models/chat_vertex_ai.ex index 3adc9943..393b9f25 100644 --- a/lib/chat_models/chat_vertex_ai.ex +++ b/lib/chat_models/chat_vertex_ai.ex @@ -216,7 +216,7 @@ defmodule LangChain.ChatModels.ChatVertexAI do %{ "fileData" => %{ "mimeType" => Keyword.fetch!(part.options, :media), - "data" => part.content + "fileUri" => part.content } } end diff --git a/test/chat_models/chat_google_ai_test.exs b/test/chat_models/chat_google_ai_test.exs index f1453797..e24c800d 100644 --- a/test/chat_models/chat_google_ai_test.exs +++ b/test/chat_models/chat_google_ai_test.exs @@ -155,7 +155,7 @@ defmodule ChatModels.ChatGoogleAITest do } = tool_result end - test "generate a map containing a text and an image part (bug #209)", %{google_ai: google_ai} do + test "generate a map containing text and inline image parts", %{google_ai: google_ai} do messages = [ %LangChain.Message{ content: diff --git a/test/chat_models/chat_vertex_ai_test.exs b/test/chat_models/chat_vertex_ai_test.exs index b2f7035c..53c4e2c4 100644 --- a/test/chat_models/chat_vertex_ai_test.exs +++ b/test/chat_models/chat_vertex_ai_test.exs @@ -61,6 +61,60 @@ defmodule ChatModels.ChatVertexAITest do assert %{"temperature" => 1.0, "topK" => 1.0, "topP" => 1.0} = config end + test "generate a map containing a text, inline image, and image url parts", %{ + vertex_ai: google_ai + } do + messages = [ + %LangChain.Message{ + content: + "You are an expert at providing an image description for assistive technology and SEO benefits.", + role: :system + }, + %LangChain.Message{ + content: [ + %LangChain.Message.ContentPart{ + type: :text, + content: "This is the text." + }, + %LangChain.Message.ContentPart{ + type: :image, + content: "/9j/4AAQSkz", + options: [media: "image/jpeg"] + }, + %LangChain.Message.ContentPart{ + type: :image_url, + content: "http://localhost:1234/image.jpg", + options: [media: "image/jpeg"] + } + ], + role: :user + } + ] + + data = ChatVertexAI.for_api(google_ai, messages, []) + assert %{"contents" => [msg1]} = data + + assert %{ + "parts" => [ + %{ + "text" => "This is the text." + }, + %{ + "inlineData" => %{ + "mimeType" => "image/jpeg", + "data" => "/9j/4AAQSkz" + } + }, + %{ + "fileData" => %{ + "fileUri" => "http://localhost:1234/image.jpg", + "mimeType" => "image/jpeg" + } + } + ] + } = msg1 + end + test "generates a map containing user and assistant messages", %{vertex_ai: vertex_ai} do user_message = "Hello Assistant!" assistant_message = "Hello User!"