From 08d9eb25ee59b9aadd93cc0bb622efa3b4a152b3 Mon Sep 17 00:00:00 2001 From: puchacz Date: Sat, 18 Jan 2025 17:05:22 +0100 Subject: [PATCH 1/4] fixed no max output tokens --- src/MicrosoftAi/AbstractionMapper.cs | 1 + test/AbstractionMapperTests.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MicrosoftAi/AbstractionMapper.cs b/src/MicrosoftAi/AbstractionMapper.cs index 7559a91..a8a46be 100644 --- a/src/MicrosoftAi/AbstractionMapper.cs +++ b/src/MicrosoftAi/AbstractionMapper.cs @@ -65,6 +65,7 @@ public static ChatRequest ToOllamaSharpChatRequest(IList chatMessag Temperature = options?.Temperature, TopP = options?.TopP, TopK = options?.TopK, + NumPredict = options?.MaxOutputTokens }, Stream = stream, Template = null, diff --git a/test/AbstractionMapperTests.cs b/test/AbstractionMapperTests.cs index 7a4fa6d..cbd28d4 100644 --- a/test/AbstractionMapperTests.cs +++ b/test/AbstractionMapperTests.cs @@ -435,6 +435,7 @@ public void Maps_Options() chatRequest.Options.Seed.Should().Be(11); chatRequest.Stream.Should().BeTrue(); chatRequest.Template.Should().BeNull(); + chatRequest.Options.NumPredict.Should().Be(1000); // not defined in ChatOptions chatRequest.CustomHeaders.Should().BeEmpty(); @@ -453,7 +454,6 @@ public void Maps_Options() chatRequest.Options.NumGpu.Should().BeNull(); chatRequest.Options.NumGqa.Should().BeNull(); chatRequest.Options.NumKeep.Should().BeNull(); - chatRequest.Options.NumPredict.Should().BeNull(); chatRequest.Options.NumThread.Should().BeNull(); chatRequest.Options.PenalizeNewline.Should().BeNull(); chatRequest.Options.RepeatLastN.Should().BeNull(); From 4f2e694150874cb28aaf24df3217b8f39e8728a1 Mon Sep 17 00:00:00 2001 From: puchacz Date: Mon, 20 Jan 2025 21:15:30 +0100 Subject: [PATCH 2/4] TryAddOllamaOption enhanced with MaxOutputTokens, added new test that validates everything is covered --- src/Constants/Application.cs | 1 + src/MicrosoftAi/AbstractionMapper.cs | 67 ++++++++++--------- src/Models/OllamaOption.cs | 6 ++ test/AbstractionMapperTests.cs | 99 ++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 34 deletions(-) diff --git a/src/Constants/Application.cs b/src/Constants/Application.cs index 1f715e8..dbca7ab 100644 --- a/src/Constants/Application.cs +++ b/src/Constants/Application.cs @@ -76,6 +76,7 @@ internal static class Application public const string Seed = "seed"; public const string TfsZ = "tfs_z"; public const string NumPredict = "num_predict"; + public const string MaxOutputTokens = "max_output_tokens"; public const string TopK = "top_k"; public const string TopP = "top_p"; public const string MinP = "min_p"; diff --git a/src/MicrosoftAi/AbstractionMapper.cs b/src/MicrosoftAi/AbstractionMapper.cs index a8a46be..b221370 100644 --- a/src/MicrosoftAi/AbstractionMapper.cs +++ b/src/MicrosoftAi/AbstractionMapper.cs @@ -72,40 +72,39 @@ public static ChatRequest ToOllamaSharpChatRequest(IList chatMessag Tools = ToOllamaSharpTools(options?.Tools) }; - if (options?.AdditionalProperties?.Any() ?? false) - { - TryAddOllamaOption(options, OllamaOption.F16kv, v => request.Options.F16kv = (bool?)v); - TryAddOllamaOption(options, OllamaOption.FrequencyPenalty, v => request.Options.FrequencyPenalty = (float?)v); - TryAddOllamaOption(options, OllamaOption.LogitsAll, v => request.Options.LogitsAll = (bool?)v); - TryAddOllamaOption(options, OllamaOption.LowVram, v => request.Options.LowVram = (bool?)v); - TryAddOllamaOption(options, OllamaOption.MainGpu, v => request.Options.MainGpu = (int?)v); - TryAddOllamaOption(options, OllamaOption.MinP, v => request.Options.MinP = (float?)v); - TryAddOllamaOption(options, OllamaOption.MiroStat, v => request.Options.MiroStat = (int?)v); - TryAddOllamaOption(options, OllamaOption.MiroStatEta, v => request.Options.MiroStatEta = (float?)v); - TryAddOllamaOption(options, OllamaOption.MiroStatTau, v => request.Options.MiroStatTau = (float?)v); - TryAddOllamaOption(options, OllamaOption.Numa, v => request.Options.Numa = (bool?)v); - TryAddOllamaOption(options, OllamaOption.NumBatch, v => request.Options.NumBatch = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumCtx, v => request.Options.NumCtx = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumGpu, v => request.Options.NumGpu = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumGqa, v => request.Options.NumGqa = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumKeep, v => request.Options.NumKeep = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumPredict, v => request.Options.NumPredict = (int?)v); - TryAddOllamaOption(options, OllamaOption.NumThread, v => request.Options.NumThread = (int?)v); - TryAddOllamaOption(options, OllamaOption.PenalizeNewline, v => request.Options.PenalizeNewline = (bool?)v); - TryAddOllamaOption(options, OllamaOption.PresencePenalty, v => request.Options.PresencePenalty = (float?)v); - TryAddOllamaOption(options, OllamaOption.RepeatLastN, v => request.Options.RepeatLastN = (int?)v); - TryAddOllamaOption(options, OllamaOption.RepeatPenalty, v => request.Options.RepeatPenalty = (float?)v); - TryAddOllamaOption(options, OllamaOption.Seed, v => request.Options.Seed = (int?)v); - TryAddOllamaOption(options, OllamaOption.Stop, v => request.Options.Stop = (v as IEnumerable)?.ToArray()); - TryAddOllamaOption(options, OllamaOption.Temperature, v => request.Options.Temperature = (float?)v); - TryAddOllamaOption(options, OllamaOption.TfsZ, v => request.Options.TfsZ = (float?)v); - TryAddOllamaOption(options, OllamaOption.TopK, v => request.Options.TopK = (int?)v); - TryAddOllamaOption(options, OllamaOption.TopP, v => request.Options.TopP = (float?)v); - TryAddOllamaOption(options, OllamaOption.TypicalP, v => request.Options.TypicalP = (float?)v); - TryAddOllamaOption(options, OllamaOption.UseMlock, v => request.Options.UseMlock = (bool?)v); - TryAddOllamaOption(options, OllamaOption.UseMmap, v => request.Options.UseMmap = (bool?)v); - TryAddOllamaOption(options, OllamaOption.VocabOnly, v => request.Options.VocabOnly = (bool?)v); - } + if (!(options?.AdditionalProperties?.Any() ?? false)) return request; + TryAddOllamaOption(options, OllamaOption.F16kv, v => request.Options.F16kv = (bool?)v); + TryAddOllamaOption(options, OllamaOption.FrequencyPenalty, v => request.Options.FrequencyPenalty = (float?)v); + TryAddOllamaOption(options, OllamaOption.LogitsAll, v => request.Options.LogitsAll = (bool?)v); + TryAddOllamaOption(options, OllamaOption.LowVram, v => request.Options.LowVram = (bool?)v); + TryAddOllamaOption(options, OllamaOption.MainGpu, v => request.Options.MainGpu = (int?)v); + TryAddOllamaOption(options, OllamaOption.MinP, v => request.Options.MinP = (float?)v); + TryAddOllamaOption(options, OllamaOption.MiroStat, v => request.Options.MiroStat = (int?)v); + TryAddOllamaOption(options, OllamaOption.MiroStatEta, v => request.Options.MiroStatEta = (float?)v); + TryAddOllamaOption(options, OllamaOption.MiroStatTau, v => request.Options.MiroStatTau = (float?)v); + TryAddOllamaOption(options, OllamaOption.Numa, v => request.Options.Numa = (bool?)v); + TryAddOllamaOption(options, OllamaOption.NumBatch, v => request.Options.NumBatch = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumCtx, v => request.Options.NumCtx = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumGpu, v => request.Options.NumGpu = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumGqa, v => request.Options.NumGqa = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumKeep, v => request.Options.NumKeep = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumPredict, v => request.Options.NumPredict = (int?)v); + TryAddOllamaOption(options, OllamaOption.MaxOutputTokens, v => request.Options.NumPredict = (int?)v); + TryAddOllamaOption(options, OllamaOption.NumThread, v => request.Options.NumThread = (int?)v); + TryAddOllamaOption(options, OllamaOption.PenalizeNewline, v => request.Options.PenalizeNewline = (bool?)v); + TryAddOllamaOption(options, OllamaOption.PresencePenalty, v => request.Options.PresencePenalty = (float?)v); + TryAddOllamaOption(options, OllamaOption.RepeatLastN, v => request.Options.RepeatLastN = (int?)v); + TryAddOllamaOption(options, OllamaOption.RepeatPenalty, v => request.Options.RepeatPenalty = (float?)v); + TryAddOllamaOption(options, OllamaOption.Seed, v => request.Options.Seed = (int?)v); + TryAddOllamaOption(options, OllamaOption.Stop, v => request.Options.Stop = (v as IEnumerable)?.ToArray()); + TryAddOllamaOption(options, OllamaOption.Temperature, v => request.Options.Temperature = (float?)v); + TryAddOllamaOption(options, OllamaOption.TfsZ, v => request.Options.TfsZ = (float?)v); + TryAddOllamaOption(options, OllamaOption.TopK, v => request.Options.TopK = (int?)v); + TryAddOllamaOption(options, OllamaOption.TopP, v => request.Options.TopP = (float?)v); + TryAddOllamaOption(options, OllamaOption.TypicalP, v => request.Options.TypicalP = (float?)v); + TryAddOllamaOption(options, OllamaOption.UseMlock, v => request.Options.UseMlock = (bool?)v); + TryAddOllamaOption(options, OllamaOption.UseMmap, v => request.Options.UseMmap = (bool?)v); + TryAddOllamaOption(options, OllamaOption.VocabOnly, v => request.Options.VocabOnly = (bool?)v); return request; } diff --git a/src/Models/OllamaOption.cs b/src/Models/OllamaOption.cs index 99e2bf0..7066fae 100644 --- a/src/Models/OllamaOption.cs +++ b/src/Models/OllamaOption.cs @@ -115,6 +115,12 @@ public class OllamaOption(string name) /// (Default: 128, -1 = infinite generation, -2 = fill context) /// public static OllamaOption NumPredict { get; } = new(Application.NumPredict); + + /// + /// The number of tokens to generate in the output. + /// (Default: -1, infinite generation) + /// + public static OllamaOption MaxOutputTokens { get; } = new(Application.MaxOutputTokens); /// /// Sets the number of threads to use during computation. By default, diff --git a/test/AbstractionMapperTests.cs b/test/AbstractionMapperTests.cs index cbd28d4..6c6e6bc 100644 --- a/test/AbstractionMapperTests.cs +++ b/test/AbstractionMapperTests.cs @@ -250,6 +250,105 @@ public void Maps_Messages_With_Tools() tool.Type.Should().Be("function"); } + [Test] + public void Maps_All_Options_With_AdditionalProperties() + { + // Arrange + List chatMessages = []; + + var options = new ChatOptions + { + AdditionalProperties = new AdditionalPropertiesDictionary() + { + // Boolean options + [OllamaOption.F16kv.Name] = true, + [OllamaOption.LogitsAll.Name] = true, + [OllamaOption.LowVram.Name] = true, + [OllamaOption.Numa.Name] = true, + [OllamaOption.PenalizeNewline.Name] = true, + [OllamaOption.UseMlock.Name] = true, + [OllamaOption.UseMmap.Name] = true, + [OllamaOption.VocabOnly.Name] = true, + + // Float options + [OllamaOption.FrequencyPenalty.Name] = 0.5f, + [OllamaOption.MinP.Name] = 0.1f, + [OllamaOption.MiroStatEta.Name] = 0.1f, + [OllamaOption.MiroStatTau.Name] = 0.2f, + [OllamaOption.PresencePenalty.Name] = 0.3f, + [OllamaOption.RepeatPenalty.Name] = 0.4f, + [OllamaOption.Temperature.Name] = 0.7f, + [OllamaOption.TfsZ.Name] = 0.8f, + [OllamaOption.TopP.Name] = 0.9f, + [OllamaOption.TypicalP.Name] = 0.95f, + + // Integer options + [OllamaOption.MainGpu.Name] = 0, + [OllamaOption.MiroStat.Name] = 1, + [OllamaOption.NumBatch.Name] = 512, + [OllamaOption.NumCtx.Name] = 4096, + [OllamaOption.NumGpu.Name] = 1, + [OllamaOption.NumGqa.Name] = 8, + [OllamaOption.NumKeep.Name] = 64, + [OllamaOption.NumPredict.Name] = 1024, + [OllamaOption.MaxOutputTokens.Name] = 2048, + [OllamaOption.NumThread.Name] = 8, + [OllamaOption.RepeatLastN.Name] = 64, + [OllamaOption.Seed.Name] = 42, + [OllamaOption.TopK.Name] = 40, + + // String array options + [OllamaOption.Stop.Name] = new[] { "stop1", "stop2" } + } + }; + + // Act + var chatRequest = AbstractionMapper.ToOllamaSharpChatRequest(chatMessages, options, stream: true, JsonSerializerOptions.Default); + + // Assert + chatRequest.Options.Should().NotBeNull(); + + // Boolean assertions + chatRequest.Options!.F16kv.Should().BeTrue(); + chatRequest.Options!.LogitsAll.Should().BeTrue(); + chatRequest.Options!.LowVram.Should().BeTrue(); + chatRequest.Options!.Numa.Should().BeTrue(); + chatRequest.Options!.PenalizeNewline.Should().BeTrue(); + chatRequest.Options!.UseMlock.Should().BeTrue(); + chatRequest.Options!.UseMmap.Should().BeTrue(); + chatRequest.Options!.VocabOnly.Should().BeTrue(); + + // Float assertions + chatRequest.Options!.FrequencyPenalty.Should().Be(0.5f); + chatRequest.Options!.MinP.Should().Be(0.1f); + chatRequest.Options!.MiroStatEta.Should().Be(0.1f); + chatRequest.Options!.MiroStatTau.Should().Be(0.2f); + chatRequest.Options!.PresencePenalty.Should().Be(0.3f); + chatRequest.Options!.RepeatPenalty.Should().Be(0.4f); + chatRequest.Options!.Temperature.Should().Be(0.7f); + chatRequest.Options!.TfsZ.Should().Be(0.8f); + chatRequest.Options!.TopP.Should().Be(0.9f); + chatRequest.Options!.TypicalP.Should().Be(0.95f); + + // Integer assertions + chatRequest.Options!.MainGpu.Should().Be(0); + chatRequest.Options!.MiroStat.Should().Be(1); + chatRequest.Options!.NumBatch.Should().Be(512); + chatRequest.Options!.NumCtx.Should().Be(4096); + chatRequest.Options!.NumGpu.Should().Be(1); + chatRequest.Options!.NumGqa.Should().Be(8); + chatRequest.Options!.NumKeep.Should().Be(64); + chatRequest.Options!.NumPredict.Should().Be(2048); + chatRequest.Options!.NumThread.Should().Be(8); + chatRequest.Options!.RepeatLastN.Should().Be(64); + chatRequest.Options!.Seed.Should().Be(42); + chatRequest.Options!.TopK.Should().Be(40); + + // String array assertions + chatRequest.Options!.Stop.Should().NotBeNull(); + chatRequest.Options!.Stop.Should().BeEquivalentTo("stop1", "stop2"); + } + [TestCaseSource(nameof(StopSequencesTestData))] public void Maps_Messages_With_IEnumerable_StopSequences(object? enumerable) { From 220f10d0752c127ec9eda7307b984fa8f8652bee Mon Sep 17 00:00:00 2001 From: puchacz Date: Mon, 20 Jan 2025 21:16:44 +0100 Subject: [PATCH 3/4] additional break added --- src/MicrosoftAi/AbstractionMapper.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/MicrosoftAi/AbstractionMapper.cs b/src/MicrosoftAi/AbstractionMapper.cs index b221370..09d0413 100644 --- a/src/MicrosoftAi/AbstractionMapper.cs +++ b/src/MicrosoftAi/AbstractionMapper.cs @@ -73,6 +73,7 @@ public static ChatRequest ToOllamaSharpChatRequest(IList chatMessag }; if (!(options?.AdditionalProperties?.Any() ?? false)) return request; + TryAddOllamaOption(options, OllamaOption.F16kv, v => request.Options.F16kv = (bool?)v); TryAddOllamaOption(options, OllamaOption.FrequencyPenalty, v => request.Options.FrequencyPenalty = (float?)v); TryAddOllamaOption(options, OllamaOption.LogitsAll, v => request.Options.LogitsAll = (bool?)v); From 74780922cee08af2fccde5c48bdeecb949c40efb Mon Sep 17 00:00:00 2001 From: puchacz Date: Tue, 21 Jan 2025 20:24:48 +0100 Subject: [PATCH 4/4] Check whether it has additional properties extracted to separate variable. --- src/MicrosoftAi/AbstractionMapper.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/MicrosoftAi/AbstractionMapper.cs b/src/MicrosoftAi/AbstractionMapper.cs index 09d0413..5e790d0 100644 --- a/src/MicrosoftAi/AbstractionMapper.cs +++ b/src/MicrosoftAi/AbstractionMapper.cs @@ -72,7 +72,9 @@ public static ChatRequest ToOllamaSharpChatRequest(IList chatMessag Tools = ToOllamaSharpTools(options?.Tools) }; - if (!(options?.AdditionalProperties?.Any() ?? false)) return request; + var hasAdditionalProperties = options?.AdditionalProperties?.Any() ?? false; + if (!hasAdditionalProperties) + return request; TryAddOllamaOption(options, OllamaOption.F16kv, v => request.Options.F16kv = (bool?)v); TryAddOllamaOption(options, OllamaOption.FrequencyPenalty, v => request.Options.FrequencyPenalty = (float?)v);