From 01fdb69511ec1f6aea2110716c851e758b65c87c Mon Sep 17 00:00:00 2001 From: Gina Triolo <51341242+gitri-ms@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:33:32 -0800 Subject: [PATCH] .Net: Function Calling Stepwise Planner (#3434) ### Motivation and Context Stepwise planner built on OpenAI function calling ### Description ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --------- Co-authored-by: Ben Thomas Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com> --- dotnet/SK-dotnet.sln | 9 + ...xample66_FunctionCallingStepwisePlanner.cs | 62 ++++ .../KernelSyntaxExamples.csproj | 1 + .../Plugins/EmailPlugin.cs | 8 +- .../AzureSdk/OpenAIFunction.cs | 2 +- .../OpenAIChatCompletionTests.cs | 10 +- .../FunctionViewExtensionsTests.cs | 2 +- .../OpenAIFunctionResponseTests.cs | 2 +- .../FunctionCalling/OpenAIFunctionTests.cs | 6 +- .../IntegrationTests/IntegrationTests.csproj | 1 + .../FunctionCallingStepwisePlannerTests.cs | 126 +++++++ .../Planners.Core/Utils/EmbeddedResource.cs | 5 +- .../Planners.OpenAI/Planners.OpenAI.csproj | 36 ++ .../FunctionCallingStepwisePlanner.cs | 335 ++++++++++++++++++ .../FunctionCallingStepwisePlannerConfig.cs | 43 +++ .../FunctionCallingStepwisePlannerResult.cs | 29 ++ .../Stepwise/InitialPlanPrompt.txt | 12 + .../Planners.OpenAI/Stepwise/StepPrompt.txt | 6 + .../Planners.OpenAI/Utils/EmbeddedResource.cs | 25 ++ .../Extensions/FunctionViewExtensions.cs | 2 +- 20 files changed, 706 insertions(+), 16 deletions(-) create mode 100644 dotnet/samples/KernelSyntaxExamples/Example66_FunctionCallingStepwisePlanner.cs create mode 100644 dotnet/src/IntegrationTests/Planners/StepwisePlanner/FunctionCallingStepwisePlannerTests.cs create mode 100644 dotnet/src/Planners/Planners.OpenAI/Planners.OpenAI.csproj create mode 100644 dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlanner.cs create mode 100644 dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerConfig.cs create mode 100644 dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerResult.cs create mode 100644 dotnet/src/Planners/Planners.OpenAI/Stepwise/InitialPlanPrompt.txt create mode 100644 dotnet/src/Planners/Planners.OpenAI/Stepwise/StepPrompt.txt create mode 100644 dotnet/src/Planners/Planners.OpenAI/Utils/EmbeddedResource.cs diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index 28304160e5b6..d9ff45c71151 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -167,6 +167,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Experimental.Orchestration. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Experimental.Orchestration.Flow.IntegrationTests", "src\Experimental\Orchestration.Flow.IntegrationTests\Experimental.Orchestration.Flow.IntegrationTests.csproj", "{C17DD7E1-E588-46AF-A87E-DF8829A021DC}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planners.OpenAI", "src\Planners\Planners.OpenAI\Planners.OpenAI.csproj", "{348BBF45-23B4-4599-83A6-8AE1795227FB}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Experimental.Assistants", "src\Experimental\Assistants\Experimental.Assistants.csproj", "{35EDBB17-925D-4776-93E0-3E6A6CA5D8FA}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Experimental.Assistants.UnitTests", "src\Experimental\Assistants.UnitTests\Experimental.Assistants.UnitTests.csproj", "{815E113D-6AAC-4AE8-8E14-5381E01E49C7}" @@ -419,6 +421,12 @@ Global {C17DD7E1-E588-46AF-A87E-DF8829A021DC}.Publish|Any CPU.Build.0 = Debug|Any CPU {C17DD7E1-E588-46AF-A87E-DF8829A021DC}.Release|Any CPU.ActiveCfg = Release|Any CPU {C17DD7E1-E588-46AF-A87E-DF8829A021DC}.Release|Any CPU.Build.0 = Release|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Publish|Any CPU.ActiveCfg = Publish|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Publish|Any CPU.Build.0 = Publish|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {348BBF45-23B4-4599-83A6-8AE1795227FB}.Release|Any CPU.Build.0 = Release|Any CPU {35EDBB17-925D-4776-93E0-3E6A6CA5D8FA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {35EDBB17-925D-4776-93E0-3E6A6CA5D8FA}.Debug|Any CPU.Build.0 = Debug|Any CPU {35EDBB17-925D-4776-93E0-3E6A6CA5D8FA}.Publish|Any CPU.ActiveCfg = Publish|Any CPU @@ -503,6 +511,7 @@ Global {C8BFFC74-3050-4F63-9E9D-C4F600427DE9} = {A2357CF8-3BB9-45A1-93F1-B366C9B63658} {CB6E74CD-3A25-459F-A578-DEF25A414335} = {A2357CF8-3BB9-45A1-93F1-B366C9B63658} {C17DD7E1-E588-46AF-A87E-DF8829A021DC} = {A2357CF8-3BB9-45A1-93F1-B366C9B63658} + {348BBF45-23B4-4599-83A6-8AE1795227FB} = {A21FAC7C-0C09-4EAD-843B-926ACEF73C80} {35EDBB17-925D-4776-93E0-3E6A6CA5D8FA} = {A2357CF8-3BB9-45A1-93F1-B366C9B63658} {815E113D-6AAC-4AE8-8E14-5381E01E49C7} = {A2357CF8-3BB9-45A1-93F1-B366C9B63658} {6009CC87-32F1-4282-88BB-8E5A7BA12925} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C} diff --git a/dotnet/samples/KernelSyntaxExamples/Example66_FunctionCallingStepwisePlanner.cs b/dotnet/samples/KernelSyntaxExamples/Example66_FunctionCallingStepwisePlanner.cs new file mode 100644 index 000000000000..3c44de0d4272 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example66_FunctionCallingStepwisePlanner.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Core; +using Plugins; +using RepoUtils; + +// ReSharper disable once InconsistentNaming +public static class Example66_FunctionCallingStepwisePlanner +{ + public static async Task RunAsync() + { + string[] questions = new string[] + { + "What is the current hour number, plus 5?", + "What is 387 minus 22? Email the solution to John and Mary.", + "Write a limerick, translate it to Spanish, and send it to Jane", + }; + + var kernel = InitializeKernel(); + + var config = new FunctionCallingStepwisePlannerConfig + { + MaxIterations = 15, + MaxTokens = 4000, + }; + var planner = new FunctionCallingStepwisePlanner(kernel, config); + + foreach (var question in questions) + { + FunctionCallingStepwisePlannerResult result = await planner.ExecuteAsync(question); + Console.WriteLine($"Q: {question}\nA: {result.FinalAnswer}"); + + // You can uncomment the line below to see the planner's process for completing the request. + // Console.WriteLine($"Chat history:\n{result.ChatHistory?.AsJson()}"); + } + } + + /// + /// Initialize the kernel and load plugins. + /// + /// A kernel instance + private static IKernel InitializeKernel() + { + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureOpenAIChatCompletionService( + TestConfiguration.AzureOpenAI.ChatDeploymentName, + TestConfiguration.AzureOpenAI.Endpoint, + TestConfiguration.AzureOpenAI.ApiKey) + .Build(); + + kernel.ImportFunctions(new EmailPlugin(), "EmailPlugin"); + kernel.ImportFunctions(new MathPlugin(), "MathPlugin"); + kernel.ImportFunctions(new TimePlugin(), "TimePlugin"); + + return kernel; + } +} diff --git a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj index 142075202f34..20434eb2f914 100644 --- a/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj +++ b/dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj @@ -48,6 +48,7 @@ + diff --git a/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs b/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs index c957608a2f85..cdba447bdef1 100644 --- a/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs +++ b/dotnet/samples/KernelSyntaxExamples/Plugins/EmailPlugin.cs @@ -23,6 +23,12 @@ public string GetEmailAddress( // Sensitive data, logging as trace, disabled by default logger?.LogTrace("Returning hard coded email for {0}", input); - return "johndoe1234@example.com"; + return input switch + { + "Jane" => "janedoe4321@example.com", + "Paul" => "paulsmith5678@example.com", + "Mary" => "maryjones8765@example.com", + _ => "johndoe1234@example.com", + }; } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs index 9e177d32b406..71f2906fccd7 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/OpenAIFunction.cs @@ -77,7 +77,7 @@ public class OpenAIFunction /// /// Separator between the plugin name and the function name /// - public const string NameSeparator = "-"; + public const string NameSeparator = "_"; /// /// Name of the function diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletion/OpenAIChatCompletionTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletion/OpenAIChatCompletionTests.cs index ee92079c6af1..a17bd17da330 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletion/OpenAIChatCompletionTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/ChatCompletion/OpenAIChatCompletionTests.cs @@ -84,8 +84,8 @@ public async Task ItCreatesCorrectFunctionsWhenUsingAutoAsync() Assert.NotNull(actualRequestContent); var optionsJson = JsonSerializer.Deserialize(actualRequestContent); Assert.Equal(2, optionsJson.GetProperty("functions").GetArrayLength()); - Assert.Equal("TimePlugin-Date", optionsJson.GetProperty("functions")[0].GetProperty("name").GetString()); - Assert.Equal("TimePlugin-Now", optionsJson.GetProperty("functions")[1].GetProperty("name").GetString()); + Assert.Equal("TimePlugin_Date", optionsJson.GetProperty("functions")[0].GetProperty("name").GetString()); + Assert.Equal("TimePlugin_Now", optionsJson.GetProperty("functions")[1].GetProperty("name").GetString()); } [Fact] @@ -95,7 +95,7 @@ public async Task ItCreatesCorrectFunctionsWhenUsingNowAsync() var chatCompletion = new OpenAIChatCompletion(modelId: "gpt-3.5-turbo", apiKey: "NOKEY", httpClient: this._httpClient); this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Content = new StringContent(ChatCompletionResponse) }; - this._requestSettings.FunctionCall = "TimePlugin-Now"; + this._requestSettings.FunctionCall = "TimePlugin_Now"; // Act await chatCompletion.GetChatCompletionsAsync(new ChatHistory(), this._requestSettings); @@ -105,7 +105,7 @@ public async Task ItCreatesCorrectFunctionsWhenUsingNowAsync() Assert.NotNull(actualRequestContent); var optionsJson = JsonSerializer.Deserialize(actualRequestContent); Assert.Equal(1, optionsJson.GetProperty("functions").GetArrayLength()); - Assert.Equal("TimePlugin-Now", optionsJson.GetProperty("functions")[0].GetProperty("name").GetString()); + Assert.Equal("TimePlugin_Now", optionsJson.GetProperty("functions")[0].GetProperty("name").GetString()); } [Fact] @@ -189,7 +189,7 @@ public void Dispose() ""role"": ""assistant"", ""content"": null, ""function_call"": { - ""name"": ""TimePlugin-Date"", + ""name"": ""TimePlugin_Date"", ""arguments"": ""{}"" } }, diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs index 8db9a4d5c79c..d4f924509f42 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/FunctionViewExtensionsTests.cs @@ -27,7 +27,7 @@ public void ItCanConvertToOpenAIFunctionNoParameters() Assert.Equal(sut.Name, result.FunctionName); Assert.Equal(sut.PluginName, result.PluginName); Assert.Equal(sut.Description, result.Description); - Assert.Equal($"{sut.PluginName}-{sut.Name}", result.FullyQualifiedName); + Assert.Equal($"{sut.PluginName}_{sut.Name}", result.FullyQualifiedName); Assert.NotNull(result.ReturnParameter); Assert.Equivalent(new OpenAIFunctionReturnParameter { Description = "retDesc", Schema = JsonDocument.Parse("\"schema\"") }, result.ReturnParameter); } diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs index ee2b6d35dbd4..d1db4d2a05ea 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionResponseTests.cs @@ -12,7 +12,7 @@ public sealed class OpenAIFunctionResponseTests public void ItCanConvertFromFunctionCallWithPluginName() { // Arrange - var sut = new FunctionCall("foo-bar", "{}"); + var sut = new FunctionCall("foo_bar", "{}"); // Act var result = OpenAIFunctionResponse.FromFunctionCall(sut); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs index b550f6ecbe9e..d6af976a355e 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/FunctionCalling/OpenAIFunctionTests.cs @@ -42,7 +42,7 @@ public void ItCanConvertToFunctionDefinitionWithPluginName() FunctionDefinition result = sut.ToFunctionDefinition(); // Assert - Assert.Equal("myplugin-myfunc", result.Name); + Assert.Equal("myplugin_myfunc", result.Name); Assert.Equal(sut.Description, result.Description); } @@ -90,7 +90,7 @@ public void ItCanConvertToFunctionDefinitionsWithParameterTypesAndReturnParamete var act = JsonSerializer.Serialize(JsonDocument.Parse(functionDefinition.Parameters)); Assert.NotNull(functionDefinition); - Assert.Equal("Tests-TestFunction", functionDefinition.Name); + Assert.Equal("Tests_TestFunction", functionDefinition.Name); Assert.Equal("My test function", functionDefinition.Description); Assert.Equal(JsonSerializer.Serialize(JsonDocument.Parse(expectedParameterSchema)), JsonSerializer.Serialize(JsonDocument.Parse(functionDefinition.Parameters))); } @@ -129,7 +129,7 @@ public void ItCanConvertToFunctionDefinitionsWithParameterTypesAndNoReturnParame FunctionDefinition functionDefinition = sut.ToFunctionDefinition(); Assert.NotNull(functionDefinition); - Assert.Equal("Tests-TestFunction", functionDefinition.Name); + Assert.Equal("Tests_TestFunction", functionDefinition.Name); Assert.Equal("My test function", functionDefinition.Description); Assert.Equal(JsonSerializer.Serialize(JsonDocument.Parse(expectedParameterSchema)), JsonSerializer.Serialize(JsonDocument.Parse(functionDefinition.Parameters))); } diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 7afc4f009d21..0cbcc2020d52 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -41,6 +41,7 @@ + diff --git a/dotnet/src/IntegrationTests/Planners/StepwisePlanner/FunctionCallingStepwisePlannerTests.cs b/dotnet/src/IntegrationTests/Planners/StepwisePlanner/FunctionCallingStepwisePlannerTests.cs new file mode 100644 index 000000000000..7da332e2c6ea --- /dev/null +++ b/dotnet/src/IntegrationTests/Planners/StepwisePlanner/FunctionCallingStepwisePlannerTests.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planners; +using Microsoft.SemanticKernel.Plugins.Core; +using Microsoft.SemanticKernel.Plugins.Web; +using Microsoft.SemanticKernel.Plugins.Web.Bing; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planners.StepwisePlanner; + +public sealed class FunctionCallingStepwisePlannerTests : IDisposable +{ + private readonly string _bingApiKey; + + public FunctionCallingStepwisePlannerTests(ITestOutputHelper output) + { + this._loggerFactory = NullLoggerFactory.Instance; + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + + string? bingApiKeyCandidate = this._configuration["Bing:ApiKey"]; + Assert.NotNull(bingApiKeyCandidate); + this._bingApiKey = bingApiKeyCandidate; + } + + [Theory(Skip = "Requires model deployment that supports function calling.")] + [InlineData("What is the tallest mountain on Earth? How tall is it?", "Everest")] + [InlineData("What is the weather in Seattle?", "Seattle")] + public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpectedAnswer) + { + // Arrange + bool useEmbeddings = false; + IKernel kernel = this.InitializeKernel(useEmbeddings); + var bingConnector = new BingConnector(this._bingApiKey); + var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); + kernel.ImportFunctions(webSearchEnginePlugin, "WebSearch"); + kernel.ImportFunctions(new TimePlugin(), "time"); + + var planner = new FunctionCallingStepwisePlanner( + kernel, + new FunctionCallingStepwisePlannerConfig() { MaxIterations = 10 }); + + // Act + var planResult = await planner.ExecuteAsync(prompt); + + // Assert - should contain the expected answer + Assert.NotNull(planResult); + Assert.NotEqual(string.Empty, planResult.FinalAnswer); + Assert.Contains(partialExpectedAnswer, planResult.FinalAnswer, StringComparison.InvariantCultureIgnoreCase); + Assert.True(planResult.Iterations > 0); + Assert.True(planResult.Iterations <= 10); + } + + private IKernel InitializeKernel(bool useEmbeddings = false) + { + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var builder = new KernelBuilder() + .WithLoggerFactory(this._loggerFactory) + .WithRetryBasic(); + + builder.WithAzureOpenAIChatCompletionService( + deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + + if (useEmbeddings) + { + builder.WithAzureOpenAITextEmbeddingGenerationService( + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + } + + var kernel = builder.Build(); + + return kernel; + } + + private readonly ILoggerFactory _loggerFactory; + private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~FunctionCallingStepwisePlannerTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + if (this._loggerFactory is IDisposable ld) + { + ld.Dispose(); + } + + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs b/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs index c759b9ef9171..67a5a57bd6c7 100644 --- a/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs +++ b/dotnet/src/Planners/Planners.Core/Utils/EmbeddedResource.cs @@ -2,7 +2,6 @@ using System.IO; using System.Reflection; -using Microsoft.SemanticKernel.Diagnostics; #pragma warning disable IDE0130 namespace Microsoft.SemanticKernel.Planners; @@ -15,10 +14,10 @@ internal static class EmbeddedResource internal static string Read(string name) { var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; - if (assembly == null) { throw new SKException($"[{s_namespace}] {name} assembly not found"); } + if (assembly == null) { throw new FileNotFoundException($"[{s_namespace}] {name} assembly not found"); } using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); - if (resource == null) { throw new SKException($"[{s_namespace}] {name} resource not found"); } + if (resource == null) { throw new FileNotFoundException($"[{s_namespace}] {name} resource not found"); } using var reader = new StreamReader(resource); return reader.ReadToEnd(); diff --git a/dotnet/src/Planners/Planners.OpenAI/Planners.OpenAI.csproj b/dotnet/src/Planners/Planners.OpenAI/Planners.OpenAI.csproj new file mode 100644 index 000000000000..4bc92ae8b09c --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Planners.OpenAI.csproj @@ -0,0 +1,36 @@ + + + + + Microsoft.SemanticKernel.Planners.OpenAI + Microsoft.SemanticKernel.Planners + netstandard2.0 + + + + + + + + Semantic Kernel - Planners + Semantic Kernel OpenAI Planners. + + + + + Always + + + Always + + + + + + + + + + + + diff --git a/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlanner.cs b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlanner.cs new file mode 100644 index 000000000000..93e7ece6822a --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlanner.cs @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Json.More; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Functions.OpenAPI.Model; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.TemplateEngine; +using Microsoft.SemanticKernel.TemplateEngine.Basic; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// A planner that uses OpenAI function calling in a stepwise manner to fulfill a user goal or question. +/// +public sealed class FunctionCallingStepwisePlanner +{ + /// + /// Initialize a new instance of the class. + /// + /// The semantic kernel instance. + /// The planner configuration. + public FunctionCallingStepwisePlanner( + IKernel kernel, + FunctionCallingStepwisePlannerConfig? config = null) + { + Verify.NotNull(kernel); + this._kernel = kernel; + this._chatCompletion = kernel.GetService(); + + // Initialize prompt renderer + this._promptTemplateFactory = new BasicPromptTemplateFactory(this._kernel.LoggerFactory); + + // Set up Config with default values and excluded plugins + this.Config = config ?? new(); + this.Config.ExcludedPlugins.Add(RestrictedPluginName); + + this._initialPlanPrompt = this.Config.GetPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Stepwise.InitialPlanPrompt.txt"); + this._stepPrompt = this.Config.GetStepPromptTemplate?.Invoke() ?? EmbeddedResource.Read("Stepwise.StepPrompt.txt"); + + // Create context and logger + this._logger = this._kernel.LoggerFactory.CreateLogger(this.GetType()); + } + + /// + /// Execute a plan + /// + /// The question to answer + /// The to monitor for cancellation requests. The default is . + /// Result containing the model's response message and chat history. + public async Task ExecuteAsync( + string question, + CancellationToken cancellationToken = default) + { + Verify.NotNullOrWhiteSpace(question); + + // Add the final answer function + this._kernel.ImportFunctions(new UserInteraction(), "UserInteraction"); + + // Request completion for initial plan + var chatHistoryForPlan = await this.BuildChatHistoryForInitialPlanAsync(question, cancellationToken).ConfigureAwait(false); + string initialPlan = (await this._chatCompletion.GenerateMessageAsync(chatHistoryForPlan, null /* request settings */, cancellationToken).ConfigureAwait(false)); + + var chatHistoryForSteps = await this.BuildChatHistoryForStepAsync(question, initialPlan, cancellationToken).ConfigureAwait(false); + + for (int i = 0; i < this.Config.MaxIterations; i++) + { + // sleep for a bit to avoid rate limiting + if (i > 0) + { + await Task.Delay(this.Config.MinIterationTimeMs, cancellationToken).ConfigureAwait(false); + } + + // For each step, request another completion to select a function for that step + chatHistoryForSteps.AddUserMessage(StepwiseUserMessage); + var chatResult = await this.GetCompletionWithFunctionsAsync(chatHistoryForSteps, cancellationToken).ConfigureAwait(false); + chatHistoryForSteps.AddAssistantMessage(chatResult); + + // Check for function response + if (!this.TryGetFunctionResponse(chatResult, out OpenAIFunctionResponse? functionResponse, out string? functionResponseError)) + { + // No function response found. Either AI returned a chat message, or something went wrong when parsing the function. + // Log the error (if applicable), then let the planner continue. + if (functionResponseError is not null) + { + chatHistoryForSteps.AddUserMessage(functionResponseError); + } + continue; + } + + // Check for final answer in the function response + if (this.TryFindFinalAnswer(functionResponse, out string finalAnswer, out string? finalAnswerError)) + { + if (finalAnswerError is not null) + { + // We found a final answer, but failed to parse it properly. + // Log the error message in chat history and let the planner try again. + chatHistoryForSteps.AddUserMessage(finalAnswerError); + continue; + } + + // Success! We found a final answer, so return the planner result. + return new FunctionCallingStepwisePlannerResult + { + FinalAnswer = finalAnswer, + ChatHistory = chatHistoryForSteps, + Iterations = i + 1, + }; + } + + // Look up function in kernel + if (this._kernel.Functions.TryGetFunctionAndContext(functionResponse, out ISKFunction? pluginFunction, out ContextVariables? funcContext)) + { + try + { + // Execute function and add to result to chat history + var result = (await this._kernel.RunAsync(funcContext, cancellationToken, pluginFunction).ConfigureAwait(false)).GetValue(); + chatHistoryForSteps.AddFunctionMessage(ParseObjectAsString(result), functionResponse.FullyQualifiedName); + } + catch (SKException) + { + chatHistoryForSteps.AddUserMessage($"Failed to execute function {functionResponse.FullyQualifiedName}. Try something else!"); + } + } + else + { + chatHistoryForSteps.AddUserMessage($"Function {functionResponse.FullyQualifiedName} does not exist in the kernel. Try something else!"); + } + } + + // We've completed the max iterations, but the model hasn't returned a final answer. + return new FunctionCallingStepwisePlannerResult + { + FinalAnswer = string.Empty, + ChatHistory = chatHistoryForSteps, + Iterations = this.Config.MaxIterations, + }; + } + + #region private + + private async Task GetCompletionWithFunctionsAsync( + ChatHistory chatHistory, + CancellationToken cancellationToken) + { + var requestSettings = this.PrepareOpenAIRequestSettingsWithFunctions(); + return (await this._chatCompletion.GetChatCompletionsAsync(chatHistory, requestSettings, cancellationToken).ConfigureAwait(false))[0]; + } + + private async Task GetFunctionsManualAsync(CancellationToken cancellationToken) + { + return await this._kernel.Functions.GetJsonSchemaFunctionsManualAsync(this.Config, null, this._logger, false, cancellationToken).ConfigureAwait(false); + } + + private OpenAIRequestSettings PrepareOpenAIRequestSettingsWithFunctions() + { + var requestSettings = this.Config.ModelSettings ?? new OpenAIRequestSettings(); + requestSettings.FunctionCall = OpenAIRequestSettings.FunctionCallAuto; + requestSettings.Functions = this._kernel.Functions.GetFunctionViews().Select(f => f.ToOpenAIFunction()).ToList(); + return requestSettings; + } + + private async Task BuildChatHistoryForInitialPlanAsync( + string goal, + CancellationToken cancellationToken) + { + var chatHistory = this._chatCompletion.CreateNewChat(); + + var systemContext = this._kernel.CreateNewContext(); + string functionsManual = await this.GetFunctionsManualAsync(cancellationToken).ConfigureAwait(false); + systemContext.Variables.Set(AvailableFunctionsKey, functionsManual); + string systemMessage = await this._promptTemplateFactory.Create(this._initialPlanPrompt, new PromptTemplateConfig()).RenderAsync(systemContext, cancellationToken).ConfigureAwait(false); + + chatHistory.AddSystemMessage(systemMessage); + chatHistory.AddUserMessage(goal); + + return chatHistory; + } + + private async Task BuildChatHistoryForStepAsync( + string goal, + string initialPlan, + CancellationToken cancellationToken) + { + var chatHistory = this._chatCompletion.CreateNewChat(); + + // Add system message with context about the initial goal/plan + var systemContext = this._kernel.CreateNewContext(); + systemContext.Variables.Set(GoalKey, goal); + systemContext.Variables.Set(InitialPlanKey, initialPlan); + var systemMessage = await this._promptTemplateFactory.Create(this._stepPrompt, new PromptTemplateConfig()).RenderAsync(systemContext, cancellationToken).ConfigureAwait(false); + + chatHistory.AddSystemMessage(systemMessage); + + return chatHistory; + } + + private bool TryGetFunctionResponse(IChatResult chatResult, [NotNullWhen(true)] out OpenAIFunctionResponse? functionResponse, out string? errorMessage) + { + functionResponse = null; + errorMessage = null; + try + { + functionResponse = chatResult.GetOpenAIFunctionResponse(); + } + catch (JsonException) + { + errorMessage = "That function call is invalid. Try something else!"; + } + + return functionResponse is not null; + } + + private bool TryFindFinalAnswer(OpenAIFunctionResponse functionResponse, out string finalAnswer, out string? errorMessage) + { + finalAnswer = string.Empty; + errorMessage = null; + + if (functionResponse.PluginName == "UserInteraction" && functionResponse.FunctionName == "SendFinalAnswer") + { + if (functionResponse.Parameters.Count > 0 && functionResponse.Parameters.TryGetValue("answer", out object valueObj)) + { + finalAnswer = ParseObjectAsString(valueObj); + } + else + { + errorMessage = "Returned answer in incorrect format. Try again!"; + } + return true; + } + return false; + } + + private static string ParseObjectAsString(object? valueObj) + { + string resultStr = string.Empty; + + if (valueObj is RestApiOperationResponse apiResponse) + { + resultStr = apiResponse.Content as string ?? string.Empty; + } + else if (valueObj is string valueStr) + { + resultStr = valueStr; + } + else if (valueObj is JsonElement valueElement) + { + if (valueElement.ValueKind == JsonValueKind.String) + { + resultStr = valueElement.GetString() ?? ""; + } + else + { + resultStr = valueElement.ToJsonString(); + } + } + else + { + resultStr = JsonSerializer.Serialize(valueObj); + } + + return resultStr; + } + + /// + /// The configuration for the StepwisePlanner + /// + private FunctionCallingStepwisePlannerConfig Config { get; } + + // Context used to access the list of functions in the kernel + private readonly IKernel _kernel; + private readonly IChatCompletion _chatCompletion; + private readonly ILogger? _logger; + + /// + /// The prompt (system message) used to generate the initial set of steps to perform. + /// + private readonly string _initialPlanPrompt; + + /// + /// The prompt (system message) for performing the steps. + /// + private readonly string _stepPrompt; + + /// + /// The prompt renderer to use for the system step + /// + private readonly BasicPromptTemplateFactory _promptTemplateFactory; + + /// + /// The name to use when creating semantic functions that are restricted from plan creation + /// + private const string RestrictedPluginName = "OpenAIFunctionsStepwisePlanner_Excluded"; + + /// + /// The user message to add to the chat history for each step of the plan. + /// + private const string StepwiseUserMessage = "Perform the next step of the plan if there is more work to do. When you have reached a final answer, use the UserInteraction_SendFinalAnswer function to communicate this back to the user."; + + // Context variable keys + private const string AvailableFunctionsKey = "available_functions"; + private const string InitialPlanKey = "initial_plan"; + private const string GoalKey = "goal"; + + #endregion private + + /// + /// Plugin used by the to interact with the caller. + /// + public sealed class UserInteraction + { + /// + /// This function is used by the to indicate when the final answer has been found. + /// + /// The final answer for the plan. + [SKFunction] + [Description("This function is used to send the final answer of a plan to the user.")] + public string SendFinalAnswer([Description("The final answer")] string answer) + { + return "Thanks"; + } + } +} diff --git a/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerConfig.cs b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerConfig.cs new file mode 100644 index 000000000000..5f62e0eaac29 --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerConfig.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Configuration for Stepwise planner instances. +/// +public sealed class FunctionCallingStepwisePlannerConfig : PlannerConfigBase +{ + /// + /// Initializes a new instance of the + /// + public FunctionCallingStepwisePlannerConfig() + { + this.MaxTokens = 4000; + } + + /// + /// Delegate to get the prompt template string for the step execution phase. + /// + public Func? GetStepPromptTemplate { get; set; } + + /// + /// The maximum number of iterations to allow in a plan. + /// + public int MaxIterations { get; set; } = 15; + + /// + /// The minimum time to wait between iterations in milliseconds. + /// + public int MinIterationTimeMs { get; set; } + + /// + /// The configuration to use for the prompt template. + /// + public OpenAIRequestSettings? ModelSettings { get; set; } +} diff --git a/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerResult.cs b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerResult.cs new file mode 100644 index 000000000000..53cd157cea30 --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Stepwise/FunctionCallingStepwisePlannerResult.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.AI.ChatCompletion; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using NS of Plan +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +/// +/// Result produced by the . +/// +public class FunctionCallingStepwisePlannerResult +{ + /// + /// Final result message of the plan. + /// + public string FinalAnswer { get; internal set; } = string.Empty; + + /// + /// Chat history containing the planning process. + /// + public ChatHistory? ChatHistory { get; internal set; } + + /// + /// Number of iterations performed by the planner. + /// + public int Iterations { get; internal set; } = 0; +} diff --git a/dotnet/src/Planners/Planners.OpenAI/Stepwise/InitialPlanPrompt.txt b/dotnet/src/Planners/Planners.OpenAI/Stepwise/InitialPlanPrompt.txt new file mode 100644 index 000000000000..c7d679eaf1ee --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Stepwise/InitialPlanPrompt.txt @@ -0,0 +1,12 @@ +You are an expert at generating plans from a given GOAL. Think step by step and determine a plan to satisfy the specified GOAL using only the FUNCTIONS provided to you. You can also make use of your own knowledge while forming an answer but you must not use functions that are not provided. Once you have come to a final answer, use the UserInteraction_SendFinalAnswer function to communicate this back to the user. + +[FUNCTIONS] + +{{$available_functions}} + +[END FUNCTIONS] + +To create the plan, follow these steps: +0. Each step should be something that is capable of being done by the list of available functions. +1. Steps can use output from one or more previous steps as input, if appropriate. +2. The plan should be as short as possible. diff --git a/dotnet/src/Planners/Planners.OpenAI/Stepwise/StepPrompt.txt b/dotnet/src/Planners/Planners.OpenAI/Stepwise/StepPrompt.txt new file mode 100644 index 000000000000..1299d55d62e3 --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Stepwise/StepPrompt.txt @@ -0,0 +1,6 @@ +Original request: {{$goal}} + +You are in the process of helping the user fulfill this request using the following plan: +{{$initial_plan}} + +The user will ask you for help with each step. \ No newline at end of file diff --git a/dotnet/src/Planners/Planners.OpenAI/Utils/EmbeddedResource.cs b/dotnet/src/Planners/Planners.OpenAI/Utils/EmbeddedResource.cs new file mode 100644 index 000000000000..67a5a57bd6c7 --- /dev/null +++ b/dotnet/src/Planners/Planners.OpenAI/Utils/EmbeddedResource.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.IO; +using System.Reflection; + +#pragma warning disable IDE0130 +namespace Microsoft.SemanticKernel.Planners; +#pragma warning restore IDE0130 + +internal static class EmbeddedResource +{ + private static readonly string? s_namespace = typeof(EmbeddedResource).Namespace; + + internal static string Read(string name) + { + var assembly = typeof(EmbeddedResource).GetTypeInfo().Assembly; + if (assembly == null) { throw new FileNotFoundException($"[{s_namespace}] {name} assembly not found"); } + + using Stream? resource = assembly.GetManifestResourceStream($"{s_namespace}." + name); + if (resource == null) { throw new FileNotFoundException($"[{s_namespace}] {name} resource not found"); } + + using var reader = new StreamReader(resource); + return reader.ReadToEnd(); + } +} diff --git a/dotnet/src/SemanticKernel.Core/Extensions/FunctionViewExtensions.cs b/dotnet/src/SemanticKernel.Core/Extensions/FunctionViewExtensions.cs index c476f9a4a945..ba9e19fc4d21 100644 --- a/dotnet/src/SemanticKernel.Core/Extensions/FunctionViewExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/Extensions/FunctionViewExtensions.cs @@ -25,7 +25,7 @@ public static JsonSchemaFunctionManual ToJsonSchemaManual(this FunctionView func { var functionManual = new JsonSchemaFunctionManual { - Name = function.Name, + Name = $"{function.PluginName}_{function.Name}", Description = function.Description, };