-
Notifications
You must be signed in to change notification settings - Fork 3.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Adding Kusto as an external memory (#2257)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description Adding Kusto as an external memory. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Dmytro Struk <[email protected]> Co-authored-by: Shawn Callegari <[email protected]>
- Loading branch information
3 people
authored
Aug 10, 2023
1 parent
bb163b6
commit 4721625
Showing
14 changed files
with
1,194 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.Memory.Kusto; | ||
| using Microsoft.SemanticKernel.Memory; | ||
| using RepoUtils; | ||
|
|
||
| // ReSharper disable once InconsistentNaming | ||
| public static class Example53_Kusto | ||
| { | ||
| private const string MemoryCollectionName = "kusto_test"; | ||
|
|
||
| public static async Task RunAsync() | ||
| { | ||
| var connectionString = new Kusto.Data.KustoConnectionStringBuilder(TestConfiguration.Kusto.ConnectionString).WithAadUserPromptAuthentication(); | ||
| using KustoMemoryStore memoryStore = new(connectionString, "MyDatabase"); | ||
|
|
||
| IKernel kernel = Kernel.Builder | ||
| .WithLogger(ConsoleLogger.Logger) | ||
| .WithOpenAITextCompletionService( | ||
| modelId: TestConfiguration.OpenAI.ModelId, | ||
| apiKey: TestConfiguration.OpenAI.ApiKey) | ||
| .WithOpenAITextEmbeddingGenerationService( | ||
| modelId: TestConfiguration.OpenAI.EmbeddingModelId, | ||
| apiKey: TestConfiguration.OpenAI.ApiKey) | ||
| .WithMemoryStorage(memoryStore) | ||
| .Build(); | ||
|
|
||
| Console.WriteLine("== Printing Collections in DB =="); | ||
| var collections = memoryStore.GetCollectionsAsync(); | ||
| await foreach (var collection in collections) | ||
| { | ||
| Console.WriteLine(collection); | ||
| } | ||
|
|
||
| Console.WriteLine("== Adding Memories =="); | ||
|
|
||
| var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair"); | ||
| var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby"); | ||
| var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat"); | ||
|
|
||
| Console.WriteLine("== Printing Collections in DB =="); | ||
| collections = memoryStore.GetCollectionsAsync(); | ||
| await foreach (var collection in collections) | ||
| { | ||
| Console.WriteLine(collection); | ||
| } | ||
|
|
||
| Console.WriteLine("== Retrieving Memories Through the Kernel =="); | ||
| MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1"); | ||
| Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found"); | ||
|
|
||
| Console.WriteLine("== Retrieving Memories Directly From the Store =="); | ||
| var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1); | ||
| var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2); | ||
| var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3); | ||
| Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found"); | ||
| Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found"); | ||
| Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found"); | ||
|
|
||
| Console.WriteLine("== Similarity Searching Memories: My favorite color is orange =="); | ||
| var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8); | ||
|
|
||
| await foreach (var item in searchResults) | ||
| { | ||
| Console.WriteLine(item.Metadata.Text + " : " + item.Relevance); | ||
| } | ||
|
|
||
| Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName); | ||
| await memoryStore.DeleteCollectionAsync(MemoryCollectionName); | ||
|
|
||
| Console.WriteLine("== Printing Collections in DB =="); | ||
| await foreach (var collection in collections) | ||
| { | ||
| Console.WriteLine(collection); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
dotnet/src/Connectors/Connectors.Memory.Kusto/Connectors.Memory.Kusto.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <!-- THIS PROPERTY GROUP MUST COME FIRST --> | ||
| <AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Kusto</AssemblyName> | ||
| <RootNamespace>Microsoft.SemanticKernel.Connectors.Memory.Kusto</RootNamespace> | ||
| <TargetFramework>netstandard2.0</TargetFramework> | ||
|
|
||
| <!--NU5104: A stable release of a package should not have a prerelease dependency.--> | ||
| <NoWarn>NU5104</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
| <!-- IMPORT NUGET PACKAGE SHARED PROPERTIES --> | ||
| <Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" /> | ||
| <Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/src/InternalUtilities.props" /> | ||
|
|
||
| <PropertyGroup> | ||
| <!-- NuGet Package Settings --> | ||
| <PackageId>Microsoft.SemanticKernel.Connectors.Memory.Kusto</PackageId> | ||
| <Title>Semantic Kernel - Azure Data Explorer (Kusto) Semantic Memory</Title> | ||
| <Description>Azure Data Explorer (Kusto) Semantic Memory connector for Semantic Kernel</Description> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Azure.Kusto.Data" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
95 changes: 95 additions & 0 deletions
95
dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryRecord.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using Kusto.Cloud.Platform.Utils; | ||
| using Microsoft.SemanticKernel.AI.Embeddings; | ||
| using Microsoft.SemanticKernel.Memory; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.Memory.Kusto; | ||
|
|
||
| /// <summary> | ||
| /// Kusto memory record entity. | ||
| /// </summary> | ||
| public sealed class KustoMemoryRecord | ||
| { | ||
| /// <summary> | ||
| /// Entity key. | ||
| /// </summary> | ||
| public string Key { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Metadata associated with memory entity. | ||
| /// </summary> | ||
| public MemoryRecordMetadata Metadata { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Source content embedding. | ||
| /// </summary> | ||
| public Embedding<float> Embedding { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Optional timestamp. | ||
| /// </summary> | ||
| public DateTimeOffset? Timestamp { get; set; } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class. | ||
| /// </summary> | ||
| /// <param name="record">Instance of <see cref="MemoryRecord"/>.</param> | ||
| public KustoMemoryRecord(MemoryRecord record) : this(record.Key, record.Metadata, record.Embedding, record.Timestamp) { } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class. | ||
| /// </summary> | ||
| /// <param name="key">Entity key.</param> | ||
| /// <param name="metadata">Metadata associated with memory entity.</param> | ||
| /// <param name="embedding">Source content embedding.</param> | ||
| /// <param name="timestamp">Optional timestamp.</param> | ||
| public KustoMemoryRecord(string key, MemoryRecordMetadata metadata, Embedding<float> embedding, DateTimeOffset? timestamp = null) | ||
| { | ||
| this.Key = key; | ||
| this.Metadata = metadata; | ||
| this.Embedding = embedding; | ||
| this.Timestamp = timestamp; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class. | ||
| /// </summary> | ||
| /// <param name="key">Entity key.</param> | ||
| /// <param name="metadata">Serialized metadata associated with memory entity.</param> | ||
| /// <param name="embedding">Source content embedding.</param> | ||
| /// <param name="timestamp">Optional timestamp.</param> | ||
| public KustoMemoryRecord(string key, string metadata, string? embedding, string? timestamp = null) | ||
| { | ||
| this.Key = key; | ||
| this.Metadata = KustoSerializer.DeserializeMetadata(metadata); | ||
| this.Embedding = KustoSerializer.DeserializeEmbedding(embedding); | ||
| this.Timestamp = KustoSerializer.DeserializeDateTimeOffset(timestamp); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Returns instance of mapped <see cref="MemoryRecord"/>. | ||
| /// </summary> | ||
| public MemoryRecord ToMemoryRecord() | ||
| { | ||
| return new MemoryRecord(this.Metadata, this.Embedding, this.Key, this.Timestamp); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Writes properties of <see cref="KustoMemoryRecord"/> instance to stream using <see cref="CsvWriter"/>. | ||
| /// </summary> | ||
| /// <param name="streamWriter">Instance of <see cref="CsvWriter"/> to write properties to stream.</param> | ||
| public void WriteToCsvStream(CsvWriter streamWriter) | ||
| { | ||
| var jsonifiedMetadata = KustoSerializer.SerializeMetadata(this.Metadata); | ||
| var jsonifiedEmbedding = KustoSerializer.SerializeEmbedding(this.Embedding); | ||
| var isoFormattedDate = KustoSerializer.SerializeDateTimeOffset(this.Timestamp); | ||
|
|
||
| streamWriter.WriteField(this.Key); | ||
| streamWriter.WriteField(jsonifiedMetadata); | ||
| streamWriter.WriteField(jsonifiedEmbedding); | ||
| streamWriter.WriteField(isoFormattedDate); | ||
| streamWriter.CompleteRecord(); | ||
| } | ||
| } |
Oops, something went wrong.