From 2d03818c7b5729c6025d1f553d15eca982b712d7 Mon Sep 17 00:00:00 2001 From: jarmatys Date: Sun, 22 Dec 2024 18:44:57 +0100 Subject: [PATCH] Task 03 - week 3 --- .../TaskRequestModel.cs | 2 +- .../TaskResponse.cs | 2 +- .../DatabaseModels/DatabaseRequestModel.cs | 15 +++ .../Models/DatabaseModels/DatabaseResponse.cs | 12 ++ API/ASSISTENTE.Playground/Playground.cs | 2 +- API/ASSISTENTE.Playground/Tasks/TaskBase.cs | 31 ++++- API/ASSISTENTE.Playground/Tasks/WeekThree.cs | 120 +++++++++++++++++- 7 files changed, 175 insertions(+), 9 deletions(-) rename API/ASSISTENTE.Playground/Models/{CentralModel => CentralModels}/TaskRequestModel.cs (85%) rename API/ASSISTENTE.Playground/Models/{CentralModel => CentralModels}/TaskResponse.cs (81%) create mode 100644 API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseRequestModel.cs create mode 100644 API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseResponse.cs diff --git a/API/ASSISTENTE.Playground/Models/CentralModel/TaskRequestModel.cs b/API/ASSISTENTE.Playground/Models/CentralModels/TaskRequestModel.cs similarity index 85% rename from API/ASSISTENTE.Playground/Models/CentralModel/TaskRequestModel.cs rename to API/ASSISTENTE.Playground/Models/CentralModels/TaskRequestModel.cs index f4b1a87..d1608e5 100644 --- a/API/ASSISTENTE.Playground/Models/CentralModel/TaskRequestModel.cs +++ b/API/ASSISTENTE.Playground/Models/CentralModels/TaskRequestModel.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace ASSISTENTE.Playground.Models.CentralModel; +namespace ASSISTENTE.Playground.Models.CentralModels; public class TaskRequestModel { diff --git a/API/ASSISTENTE.Playground/Models/CentralModel/TaskResponse.cs b/API/ASSISTENTE.Playground/Models/CentralModels/TaskResponse.cs similarity index 81% rename from API/ASSISTENTE.Playground/Models/CentralModel/TaskResponse.cs rename to API/ASSISTENTE.Playground/Models/CentralModels/TaskResponse.cs index 906af57..8c6a1cb 100644 --- a/API/ASSISTENTE.Playground/Models/CentralModel/TaskResponse.cs +++ b/API/ASSISTENTE.Playground/Models/CentralModels/TaskResponse.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace ASSISTENTE.Playground.Models.CentralModel; +namespace ASSISTENTE.Playground.Models.CentralModels; public class TaskResponse { diff --git a/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseRequestModel.cs b/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseRequestModel.cs new file mode 100644 index 0000000..e819a70 --- /dev/null +++ b/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseRequestModel.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; + +namespace ASSISTENTE.Playground.Models.DatabaseModels; + +public class DatabaseRequestModel +{ + [JsonPropertyName("task")] + public required string Task { get; set; } + + [JsonPropertyName("apikey")] + public required string ApiKey { get; set; } + + [JsonPropertyName("query")] + public required string Query { get; set; } +} \ No newline at end of file diff --git a/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseResponse.cs b/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseResponse.cs new file mode 100644 index 0000000..3ab136c --- /dev/null +++ b/API/ASSISTENTE.Playground/Models/DatabaseModels/DatabaseResponse.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace ASSISTENTE.Playground.Models.DatabaseModels; + +public class DatabaseResponse +{ + [JsonPropertyName("reply")] + public required object Reply { get; set; } + + [JsonPropertyName("error")] + public required string Error { get; set; } +} \ No newline at end of file diff --git a/API/ASSISTENTE.Playground/Playground.cs b/API/ASSISTENTE.Playground/Playground.cs index 70a31e7..5bab07c 100644 --- a/API/ASSISTENTE.Playground/Playground.cs +++ b/API/ASSISTENTE.Playground/Playground.cs @@ -41,7 +41,7 @@ public async Task LearnAsync() public async Task RunAsync() { - var result = await weekThree.Task_02() + var result = await weekThree.Task_03() .Tap(result => logger.LogInformation(result)) .TapError(error => logger.LogError(error)); } diff --git a/API/ASSISTENTE.Playground/Tasks/TaskBase.cs b/API/ASSISTENTE.Playground/Tasks/TaskBase.cs index 101e0f2..09da704 100644 --- a/API/ASSISTENTE.Playground/Tasks/TaskBase.cs +++ b/API/ASSISTENTE.Playground/Tasks/TaskBase.cs @@ -1,7 +1,8 @@ using System.Text; using System.Text.Json; using ASSISTENTE.Playground.Models; -using ASSISTENTE.Playground.Models.CentralModel; +using ASSISTENTE.Playground.Models.CentralModels; +using ASSISTENTE.Playground.Models.DatabaseModels; using CSharpFunctionalExtensions; namespace ASSISTENTE.Playground.Tasks; @@ -75,4 +76,32 @@ protected async Task> ReportResult(string taskName, object? taskR ? Result.Failure(deserializedContent?.Message) : Result.Success(deserializedContent!.Message); } + + protected async Task> DatabaseQuery(string taskName, string query) + { + const string databaseUrl = "https://centrala.ag3nts.org/apidb"; + + var request = new DatabaseRequestModel + { + Task = taskName, + ApiKey = ApiKey, + Query = query + }; + + var response = await httpClient.PostAsync( + databaseUrl, + new StringContent( + JsonSerializer.Serialize(request), + Encoding.UTF8, "application/json" + ) + ); + + var responseContent = await response.Content.ReadAsStringAsync(); + + var deserializedContent = JsonSerializer.Deserialize(responseContent); + + return !response.IsSuccessStatusCode + ? Result.Failure(deserializedContent?.Error) + : Result.Success(deserializedContent!.Reply.ToString())!; + } } \ No newline at end of file diff --git a/API/ASSISTENTE.Playground/Tasks/WeekThree.cs b/API/ASSISTENTE.Playground/Tasks/WeekThree.cs index 3019aee..5c775e2 100644 --- a/API/ASSISTENTE.Playground/Tasks/WeekThree.cs +++ b/API/ASSISTENTE.Playground/Tasks/WeekThree.cs @@ -149,15 +149,15 @@ public async Task> Task_02() const string question = "W raporcie, z którego dnia znajduje się wzmianka o kradzieży prototypu broni?"; var weaponsFiles = Directory.GetFiles(dataFilesWeapons); - + await qdrantService.DropCollectionAsync("aidevs"); await qdrantService.CreateCollectionAsync("aidevs"); - + foreach (var weaponFile in weaponsFiles) { var fileName = Path.GetFileName(weaponFile); var fileContent = await File.ReadAllTextAsync(weaponFile); - + await EmbeddingText.Create(fileContent) .Bind(async embeddingText => await embeddingClient.GetAsync(embeddingText)) .Bind(embedding => @@ -166,12 +166,16 @@ await EmbeddingText.Create(fileContent) { { "fileName", fileName } }; - + return DocumentDto.Create("aidevs", embedding.Embeddings, metadata); }) .Bind(async document => await qdrantService.UpsertAsync(document)); } + // TODO: Create library 'ASSISTENTE.Infrastructure.Search' which will be combined qdrant, embedding and other services like typsense, database simple query + // Implement hybrid search service which will be able to search in multiple sources and score them together + // Combined score formula: (1 / vector rank) + (1 / full text rank) + var date = await EmbeddingText.Create(question) .Bind(async embeddingText => await embeddingClient.GetAsync(embeddingText)) .Bind(embedding => VectorDto.Create("aidevs", embedding.Embeddings, elements: 1)) @@ -181,8 +185,114 @@ await EmbeddingText.Create(fileContent) .Map(Path.GetFileNameWithoutExtension) .Map(dateString => DateTime.Parse(dateString!.Replace("_", "-")).ToString("yyyy-MM-dd")) .GetValueOrDefault(x => x); - + return await ReportResult("wektory", date); } + + public async Task> Task_03() + { + // Dostępne tabele: users, datacenters & connections + + const string question = "które aktywne datacenter (DC_ID) są zarządzane przez pracowników, " + + "którzy są na urlopie (is_active=0)"; + + const string acceptableFormat = "1, 2, 3"; + + var context = new StringBuilder(); + + while (true) + { + var masterContext = context.ToString() == string.Empty + ? "BRAK INFORMACJI - NAPISZ ZAPYTANIE SQL" + : context.ToString(); + + if (context.ToString() != string.Empty) + { + var verifyPrompt = $""" + Twoim zadaniem jest zweryfikować czy w pozyskanych informacjach znajduje się + odpowiedź na zadane pytanie. + + + {question} + + + + {masterContext} + + + + Jeżeli nie znajdziesz odpowiedzi, zwróć "BRAK". + + Jeżeli znajdziesz odpowiedź, zwróć znalezioną informację. + + Zwrócona informacja powinna zawierać tylko i wyłącznie odpowiedź na pytanie + w formacie: {acceptableFormat} + """; + + var hasAnswer = await Prompt.Create(verifyPrompt) + .Bind(async prompt => await llmClient.GenerateAnswer(prompt)) + .GetValueOrDefault(x => x.Text, ""); + + if (hasAnswer != "BRAK") + { + var inActiveDataCenterIds = hasAnswer.Split(", ") + .Select(int.Parse) + .ToList(); + + return await ReportResult("database", inActiveDataCenterIds); + } + } + + var masterPrompt = $""" + Twoim zadaniem jest otrzymać odpowiedź na pytanie: + + {question} + + + + 1. Do dyspozycji masz dostęp do bazy danych. Musisz napisać zapytania SQL, które naprowadzą Cię na odpowiedź. + 2. Nie podawaj żadnych dodatkowych informacji, zwróć sam SQL lub odpowiedź na pytanie. + 3. Nie zwracaj żadnych tagów markdown, ani znaków specjalnych, same zapytanie SQL. + + + + `show tables` = zwraca listę tabel + `show create table ` = pokazuje, jak zbudowana jest konkretna tabela + + + Jeżeli nie jesteś pewny jak odpowiedzieć na zadane pytanie przygotuj kolejne zapytanie SQL aby uzyskać więcej szczegółów. + Jeżeli masz wszystko czego potrzebujesz zwróć rezultat w postaci liczb po przecinku np. 1, 2, 3. + + + select * from datacenters + + + + {masterContext} + + """; + + var sqlQuery = await Prompt.Create(masterPrompt) + .Bind(async prompt => await llmClient.GenerateAnswer(prompt)) + .GetValueOrDefault(x => x.Text, ""); + + var isSqlQuery = sqlQuery.Contains("select", StringComparison.OrdinalIgnoreCase) || + sqlQuery.Contains("show", StringComparison.OrdinalIgnoreCase); + + if (isSqlQuery) + { + var queryResult = await DatabaseQuery("database", sqlQuery); + + context.Append($"{sqlQuery}\n" + + $"{queryResult}\n\n"); + + continue; + } + + break; + } + + return Result.Success("Zadanie zakończone"); + } } \ No newline at end of file