.Net: Support for configuring dimensions in Google AI embeddings generation (#10489)

### Motivation and Context

This change addresses a limitation in the current implementation of the
Google AI embeddings generation service in Semantic Kernel. Currently,
users cannot configure the output dimensionality of the embeddings, even
though the underlying Google AI API supports specifying the number of
dimensions via the `output_dimensionality` parameter.

**Why is this change required?**  
Allowing configuration of the dimensions provides greater flexibility
for users to tailor the embeddings to their specific use cases—whether
for optimizing memory usage, improving performance, or ensuring
compatibility with downstream systems that expect a particular embedding
size.

**What problem does it solve?**  
It solves the issue of inflexibility by exposing the `dimensions`
parameter in the service constructors, builder methods, and API request
payloads. This ensures that developers can leverage the full
capabilities of the Google API without being limited to the default
embedding size.

**What scenario does it contribute to?**  
This feature is particularly useful in scenarios where:
- Users need to optimize storage or computational resources.
- Downstream tasks or integrations require embeddings of a specific
dimensionality.
- Fine-tuning the model output is essential for performance or
compatibility reasons.

Relevant issue link:
#10488

### Description

This PR introduces support for specifying the output dimensionality in
the Google AI embeddings generation workflow. The main changes include:

- **Service Constructor Update:**  
The `GoogleAITextEmbeddingGenerationService` constructor now accepts an
optional `dimensions` parameter, which is then forwarded to the
lower-level client implementations.

- **Builder and Extension Methods:**  
Extension methods such as `AddGoogleAIEmbeddingGeneration` have been
updated to accept a `dimensions` parameter. This allows developers to
configure the embedding dimensions using the builder pattern.

- **Request Payload Enhancement:**  
The `GoogleAIEmbeddingRequest` class now includes a new optional
property `Dimensions` (serialized as `output_dimensionality`). When
provided, this value is included in the JSON payload sent to the Google
AI API.

- **Metadata and Attributes Update:**  
The service’s metadata now reflects the provided dimensions, ensuring
consistency in configuration tracking.

- **Unit Testing:**  
  New unit tests have been added to confirm that:
- When a `dimensions` value is provided, it is correctly included in the
JSON request.
  - When not provided, the default behavior remains unchanged.
  
This enhancement maintains backward compatibility since the new
parameter is optional. Existing implementations that do not specify a
dimension will continue to work as before.

### Contribution Checklist

- [x] The code builds clean without any errors or warnings.
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations.
- [x] All unit tests pass, and I have added new tests where possible.
- [x] I didn't break anyone 😄

---------

Co-authored-by: Roger Barreto <[email protected]>

Author: ArieSLV

dotnet/Directory.Packages.props

@@ -24,6 +24,7 @@
     <PackageVersion Include="Dapr.Actors.AspNetCore" Version="1.14.0" />
     <PackageVersion Include="Dapr.AspNetCore" Version="1.14.0" />
     <PackageVersion Include="FastBertTokenizer" Version="1.0.28" />
+    <PackageVersion Include="Google.Apis.Auth" Version="1.69.0" />
     <PackageVersion Include="mcpdotnet" Version="1.0.1.3" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.13" />
     <PackageVersion Include="Microsoft.AspNetCore.OpenApi" Version="8.0.13" />
@@ -142,9 +143,9 @@
     <PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />
     <PackageVersion Include="Testcontainers" Version="4.1.0" />
     <PackageVersion Include="Testcontainers.Milvus" Version="4.1.0" />
-    <PackageVersion Include="Testcontainers.MongoDB" Version="4.1.0"/>
-    <PackageVersion Include="Testcontainers.PostgreSql" Version="4.1.0"/>
-    <PackageVersion Include="Testcontainers.Redis" Version="4.1.0"/>
+    <PackageVersion Include="Testcontainers.MongoDB" Version="4.1.0" />
+    <PackageVersion Include="Testcontainers.PostgreSql" Version="4.1.0" />
+    <PackageVersion Include="Testcontainers.Redis" Version="4.1.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.2" />
     <PackageVersion Include="Qdrant.Client" Version="1.12.0" />
     <!-- Symbols -->

dotnet/samples/Concepts/Concepts.csproj

@@ -16,6 +16,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Net.Compilers.Toolset" />
     <PackageReference Include="Docker.DotNet" />
+    <PackageReference Include="Google.Apis.Auth" />
     <PackageReference Include="Microsoft.Extensions.AI.OpenAI" />
     <PackageReference Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />

dotnet/samples/Concepts/Memory/Google_EmbeddingGeneration.cs

@@ -0,0 +1,120 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Google.Apis.Auth.OAuth2;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Embeddings;
+using xRetry;
+
+namespace Memory;
+
+// The following example shows how to use Semantic Kernel with Google AI and Google's Vertex AI for embedding generation,
+// including the ability to specify custom dimensions.
+public class Google_EmbeddingGeneration(ITestOutputHelper output) : BaseTest(output)
+{
+    /// <summary>
+    /// This test demonstrates how to use the Google Vertex AI embedding generation service with default dimensions.
+    /// </summary>
+    /// <remarks>
+    /// Currently custom dimensions are not supported for Vertex AI.
+    /// </remarks>
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithDefaultDimensionsUsingVertexAI()
+    {
+        string? bearerToken = null;
+
+        Assert.NotNull(TestConfiguration.VertexAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.VertexAI.ClientId);
+        Assert.NotNull(TestConfiguration.VertexAI.ClientSecret);
+        Assert.NotNull(TestConfiguration.VertexAI.Location);
+        Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddVertexAIEmbeddingGeneration(
+                modelId: TestConfiguration.VertexAI.EmbeddingModelId!,
+                bearerTokenProvider: GetBearerToken,
+                location: TestConfiguration.VertexAI.Location,
+                projectId: TestConfiguration.VertexAI.ProjectId);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the default dimensions for the model
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (default) for the provided text");
+
+        // Uses Google.Apis.Auth.OAuth2 to get the bearer token
+        async ValueTask<string> GetBearerToken()
+        {
+            if (!string.IsNullOrEmpty(bearerToken))
+            {
+                return bearerToken;
+            }
+
+            var credential = GoogleWebAuthorizationBroker.AuthorizeAsync(
+                new ClientSecrets
+                {
+                    ClientId = TestConfiguration.VertexAI.ClientId,
+                    ClientSecret = TestConfiguration.VertexAI.ClientSecret
+                },
+                ["https://www.googleapis.com/auth/cloud-platform"],
+                "user",
+                CancellationToken.None);
+
+            var userCredential = await credential.WaitAsync(CancellationToken.None);
+            bearerToken = userCredential.Token.AccessToken;
+
+            return bearerToken;
+        }
+    }
+
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithDefaultDimensionsUsingGoogleAI()
+    {
+        Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddGoogleAIEmbeddingGeneration(
+                modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
+                apiKey: TestConfiguration.GoogleAI.ApiKey);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the default dimensions for the model
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (default) for the provided text");
+    }
+
+    [RetryFact(typeof(HttpOperationException))]
+    public async Task GenerateEmbeddingWithCustomDimensionsUsingGoogleAI()
+    {
+        Assert.NotNull(TestConfiguration.GoogleAI.EmbeddingModelId);
+        Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
+
+        // Specify custom dimensions for the embeddings
+        const int CustomDimensions = 512;
+
+        IKernelBuilder kernelBuilder = Kernel.CreateBuilder();
+        kernelBuilder.AddGoogleAIEmbeddingGeneration(
+                modelId: TestConfiguration.GoogleAI.EmbeddingModelId!,
+                apiKey: TestConfiguration.GoogleAI.ApiKey,
+                dimensions: CustomDimensions);
+        Kernel kernel = kernelBuilder.Build();
+
+        var embeddingGenerator = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        // Generate embeddings with the specified custom dimensions
+        var embeddings = await embeddingGenerator.GenerateEmbeddingsAsync(
+            ["Semantic Kernel is a lightweight, open-source development kit that lets you easily build AI agents and integrate the latest AI models into your codebase."]);
+
+        Console.WriteLine($"Generated '{embeddings.Count}' embedding(s) with '{embeddings[0].Length}' dimensions (custom: '{CustomDimensions}') for the provided text");
+
+        // Verify that we received embeddings with our requested dimensions
+        Assert.Equal(CustomDimensions, embeddings[0].Length);
+    }
+}

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/GoogleAI/GoogleAIClientEmbeddingsGenerationTests.cs

@@ -142,14 +142,61 @@ public async Task ItCreatesPostRequestWithSemanticKernelVersionHeaderAsync()
         Assert.Equal(expectedVersion, header);
     }
 
+    [Fact]
+    public async Task ShouldIncludeDimensionsInAllRequestsAsync()
+    {
+        // Arrange
+        const int Dimensions = 512;
+        var client = this.CreateEmbeddingsClient(dimensions: Dimensions);
+        var dataToEmbed = new List<string>()
+        {
+            "First text to embed",
+            "Second text to embed",
+            "Third text to embed"
+        };
+
+        // Act
+        await client.GenerateEmbeddingsAsync(dataToEmbed);
+
+        // Assert
+        var request = JsonSerializer.Deserialize<GoogleAIEmbeddingRequest>(this._messageHandlerStub.RequestContent);
+        Assert.NotNull(request);
+        Assert.Equal(dataToEmbed.Count, request.Requests.Count);
+        Assert.All(request.Requests, item => Assert.Equal(Dimensions, item.Dimensions));
+    }
+
+    [Fact]
+    public async Task ShouldNotIncludeDimensionsInAllRequestsWhenNotProvidedAsync()
+    {
+        // Arrange
+        var client = this.CreateEmbeddingsClient();
+        var dataToEmbed = new List<string>()
+        {
+            "First text to embed",
+            "Second text to embed",
+            "Third text to embed"
+        };
+
+        // Act
+        await client.GenerateEmbeddingsAsync(dataToEmbed);
+
+        // Assert
+        var request = JsonSerializer.Deserialize<GoogleAIEmbeddingRequest>(this._messageHandlerStub.RequestContent);
+        Assert.NotNull(request);
+        Assert.Equal(dataToEmbed.Count, request.Requests.Count);
+        Assert.All(request.Requests, item => Assert.Null(item.Dimensions));
+    }
+
     private GoogleAIEmbeddingClient CreateEmbeddingsClient(
-        string modelId = "fake-model")
+        string modelId = "fake-model",
+        int? dimensions = null)
     {
         var client = new GoogleAIEmbeddingClient(
             httpClient: this._httpClient,
             modelId: modelId,
             apiVersion: GoogleAIVersion.V1,
-            apiKey: "fake-key");
+            apiKey: "fake-key",
+            dimensions: dimensions);
         return client;
     }
 

dotnet/src/Connectors/Connectors.Google.UnitTests/Core/GoogleAI/GoogleAIEmbeddingRequestTests.cs

@@ -1,41 +1,86 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Text.Json;
 using Microsoft.SemanticKernel.Connectors.Google.Core;
 using Xunit;
 
 namespace SemanticKernel.Connectors.Google.UnitTests.Core.GoogleAI;
 
 public sealed class GoogleAIEmbeddingRequestTests
 {
+    // Arrange
+    private static readonly string[] s_data = ["text1", "text2"];
+    private const string ModelId = "modelId";
+    private const string DimensionalityJsonPropertyName = "\"outputDimensionality\"";
+    private const int Dimensions = 512;
+
     [Fact]
     public void FromDataReturnsValidRequestWithData()
     {
-        // Arrange
-        string[] data = ["text1", "text2"];
-        var modelId = "modelId";
-
         // Act
-        var request = GoogleAIEmbeddingRequest.FromData(data, modelId);
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
 
         // Assert
         Assert.Equal(2, request.Requests.Count);
-        Assert.Equal(data[0], request.Requests[0].Content.Parts![0].Text);
-        Assert.Equal(data[1], request.Requests[1].Content.Parts![0].Text);
+        Assert.Equal(s_data[0], request.Requests[0].Content.Parts![0].Text);
+        Assert.Equal(s_data[1], request.Requests[1].Content.Parts![0].Text);
     }
 
     [Fact]
     public void FromDataReturnsValidRequestWithModelId()
     {
-        // Arrange
-        string[] data = ["text1", "text2"];
-        var modelId = "modelId";
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
+
+        // Assert
+        Assert.Equal(2, request.Requests.Count);
+        Assert.Equal($"models/{ModelId}", request.Requests[0].Model);
+        Assert.Equal($"models/{ModelId}", request.Requests[1].Model);
+    }
 
+    [Fact]
+    public void FromDataSetsDimensionsToNullWhenNotProvided()
+    {
         // Act
-        var request = GoogleAIEmbeddingRequest.FromData(data, modelId);
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
 
         // Assert
         Assert.Equal(2, request.Requests.Count);
-        Assert.Equal($"models/{modelId}", request.Requests[0].Model);
-        Assert.Equal($"models/{modelId}", request.Requests[1].Model);
+        Assert.Null(request.Requests[0].Dimensions);
+        Assert.Null(request.Requests[1].Dimensions);
+    }
+
+    [Fact]
+    public void FromDataJsonDoesNotIncludeDimensionsWhenNull()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId);
+        string json = JsonSerializer.Serialize(request);
+
+        // Assert
+        Assert.DoesNotContain(DimensionalityJsonPropertyName, json);
+    }
+
+    [Fact]
+    public void FromDataSetsDimensionsWhenProvided()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId, Dimensions);
+
+        // Assert
+        Assert.Equal(2, request.Requests.Count);
+        Assert.Equal(Dimensions, request.Requests[0].Dimensions);
+        Assert.Equal(Dimensions, request.Requests[1].Dimensions);
+    }
+
+    [Fact]
+    public void FromDataJsonIncludesDimensionsWhenProvided()
+    {
+        // Act
+        var request = GoogleAIEmbeddingRequest.FromData(s_data, ModelId, Dimensions);
+        string json = JsonSerializer.Serialize(request);
+
+        // Assert
+        Assert.Contains($"{DimensionalityJsonPropertyName}:{Dimensions}", json);
     }
 }