move tokenizer to new project and fix UT

Fiona.sln

@@ -37,6 +37,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{2C5448CC
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fiona.IDE.Compiler.Tests", "ide\tests\Fiona.IDE.Compiler.Tests\Fiona.IDE.Compiler.Tests.csproj", "{0BCA4496-00BA-4F91-B725-79680AD4AF17}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fiona.IDE.Tokenizer", "ide\src\Fiona.IDE.Tokenizer\Fiona.IDE.Tokenizer.csproj", "{0FC41D88-1E95-4CBA-BCA5-7116C2619002}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -85,6 +87,10 @@ Global
 		{0BCA4496-00BA-4F91-B725-79680AD4AF17}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0BCA4496-00BA-4F91-B725-79680AD4AF17}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0BCA4496-00BA-4F91-B725-79680AD4AF17}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0FC41D88-1E95-4CBA-BCA5-7116C2619002}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0FC41D88-1E95-4CBA-BCA5-7116C2619002}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0FC41D88-1E95-4CBA-BCA5-7116C2619002}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0FC41D88-1E95-4CBA-BCA5-7116C2619002}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -105,6 +111,7 @@ Global
 		{7521761E-B0D2-4A52-BF3A-C719971680D0} = {9386A258-6F1D-4785-BD1B-0468D65DC3A6}
 		{2C5448CC-C600-46F1-978C-0972771275DC} = {D59629DE-5F27-45CE-900A-B57751CF077C}
 		{0BCA4496-00BA-4F91-B725-79680AD4AF17} = {2C5448CC-C600-46F1-978C-0972771275DC}
+		{0FC41D88-1E95-4CBA-BCA5-7116C2619002} = {9386A258-6F1D-4785-BD1B-0468D65DC3A6}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {5E1ADD6E-3523-4A6C-87B1-B8FB8A03ED19}

ide/src/Fiona.IDE.Compiler/Compiler.cs

@@ -1,8 +1,7 @@
 using Fiona.IDE.Compiler.Parser;
-using Fiona.IDE.Compiler.Tokens;
 using Fiona.IDE.ProjectManager;
 using Fiona.IDE.ProjectManager.Models;
-using System.Text;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler
 {
@@ -19,24 +18,12 @@ public async Task RunAsync()
 
         private async Task ParseFileAsync(ProjectFile projectFile)
         {
-
             await using FileStream file = File.Open(projectFile.Path, FileMode.Open);
-            try
-            {
-                using StreamReader reader = new(file);
-                IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
-                ReadOnlyMemory<byte> parsedContent =(await parser.ParseAsync(tokens, projectFile));
-                await file.WriteAsync(parsedContent);
-            }
-            catch (Exception e)
-            {
-                throw;
-            }
-            finally
-            {
-                file.Close();
-            }
+            using StreamReader reader = new(file);
+            IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
+            ReadOnlyMemory<byte> parsedContent = (await parser.ParseAsync(tokens, projectFile));
+            await file.WriteAsync(parsedContent);
         }
-        
+
     }
 }

ide/src/Fiona.IDE.Compiler/Extension.cs

@@ -1,5 +1,5 @@
 using Fiona.IDE.Compiler.Parser;
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 using Microsoft.Extensions.DependencyInjection;
 
 namespace Fiona.IDE.Compiler
@@ -12,14 +12,5 @@ public static IServiceCollection AddCompiler(this IServiceCollection services)
             services.AddSingleton<IParser, Parser.Parser>();
             return services;
         }
-
-
-        private static void AddTokens(this IServiceCollection services)
-        {
-            foreach (string tokenType in Enum.GetNames<TokenType>())
-            {
-                services.Add(new ServiceDescriptor(typeof(IToken), new Token(Enum.Parse<TokenType>(tokenType), tokenType), ServiceLifetime.Singleton));
-            }
-        }
     }
 }

ide/src/Fiona.IDE.Compiler/Fiona.IDE.Compiler.csproj

@@ -12,6 +12,7 @@
 
     <ItemGroup>
       <ProjectReference Include="..\Fiona.IDE.ProjectManager\Fiona.IDE.ProjectManager.csproj" />
+      <ProjectReference Include="..\Fiona.IDE.Tokenizer\Fiona.IDE.Tokenizer.csproj" />
     </ItemGroup>
 
 </Project>

ide/src/Fiona.IDE.Compiler/Parser/IParser.cs

@@ -1,5 +1,5 @@
-using Fiona.IDE.Compiler.Tokens;
 using Fiona.IDE.ProjectManager.Models;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler.Parser;
 

ide/src/Fiona.IDE.Compiler/Parser/Models/Class.cs

@@ -1,4 +1,3 @@
-using Fiona.IDE.Compiler.Tokens;
 using System.Text;
 
 namespace Fiona.IDE.Compiler.Parser.Models;

ide/src/Fiona.IDE.Compiler/Parser/Models/Dependency.cs

@@ -1,5 +1,5 @@
 using Fiona.IDE.Compiler.Parser.Exceptions;
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler.Parser.Models;
 

ide/src/Fiona.IDE.Compiler/Parser/Models/Endpoint.cs

@@ -1,4 +1,4 @@
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 using System.Text;
 
 namespace Fiona.IDE.Compiler.Parser.Models;

ide/src/Fiona.IDE.Compiler/Parser/Models/Parameter.cs

@@ -1,5 +1,5 @@
 using Fiona.IDE.Compiler.Parser.Exceptions;
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler.Parser.Models;
 

ide/src/Fiona.IDE.Compiler/Parser/Parser.cs

@@ -1,7 +1,7 @@
 using Fiona.IDE.Compiler.Parser.Exceptions;
 using Fiona.IDE.Compiler.Parser.Models;
-using Fiona.IDE.Compiler.Tokens;
 using Fiona.IDE.ProjectManager.Models;
+using Fiona.IDE.Tokenizer;
 using Microsoft.VisualBasic;
 using System.Text;
 using Class=Fiona.IDE.Compiler.Parser.Models.Class;

ide/src/Fiona.IDE.Compiler/Parser/Validator.cs

@@ -1,5 +1,5 @@
 using Fiona.IDE.Compiler.Parser.Exceptions;
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler.Parser;
 

ide/src/Fiona.IDE.ProjectManager/Models/ProjectFile.cs

@@ -28,7 +28,9 @@ internal static async Task<ProjectFile> Create(string path)
         {
             throw new FileAlreadyExistsException(path);
         }
-        File.Create(path);
+        FileStream fileHandler = File.Create(path);
+        // We have to close file after create
+        fileHandler.Close();
 
         ProjectFile projectFile = new(path);
 

ide/src/Fiona.IDE.Compiler/Tokens/Exceptons/EmptyInputStreamException.cs → ide/src/Fiona.IDE.Tokenizer/Exceptions/EmptyInputStreamException.cs

@@ -1,4 +1,4 @@
-namespace Fiona.IDE.Compiler.Tokens.Exceptons;
+namespace Fiona.IDE.Tokenizer.Exceptions;
 
 public class EmptyInputStreamException() : Exception("The input stream is null")
 {

ide/src/Fiona.IDE.Tokenizer/Fiona.IDE.Tokenizer.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net8.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+</Project>

ide/src/Fiona.IDE.Compiler/Tokens/IToken.cs → ide/src/Fiona.IDE.Tokenizer/IToken.cs

@@ -1,6 +1,6 @@
-namespace Fiona.IDE.Compiler.Tokens;
+namespace Fiona.IDE.Tokenizer;
 
-internal interface IToken
+public interface IToken
 {
     public string? Value { get; }
     public TokenType Type { get; }

ide/src/Fiona.IDE.Tokenizer/InternalVisiebleTo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly:InternalsVisibleTo("Fiona.IDE.Compiler.Tests")]

ide/src/Fiona.IDE.Compiler/Tokens/Token.cs → ide/src/Fiona.IDE.Tokenizer/Token.cs

@@ -1,4 +1,4 @@
-namespace Fiona.IDE.Compiler.Tokens;
+namespace Fiona.IDE.Tokenizer;
 
 internal sealed class Token : IToken
 {

ide/src/Fiona.IDE.Compiler/Tokens/TokenFactory.cs → ide/src/Fiona.IDE.Tokenizer/TokenFactory.cs

@@ -1,7 +1,7 @@
-using Fiona.IDE.Compiler.Parser.Exceptions;
+using Fiona.IDE.Tokenizer.Exceptions;
 using System.Text;
 
-namespace Fiona.IDE.Compiler.Tokens;
+namespace Fiona.IDE.Tokenizer;
 
 internal static class TokenFactory
 {

ide/src/Fiona.IDE.Compiler/Tokens/TokenType.cs → ide/src/Fiona.IDE.Tokenizer/TokenType.cs

@@ -1,6 +1,6 @@
-namespace Fiona.IDE.Compiler.Tokens;
+namespace Fiona.IDE.Tokenizer;
 
-internal enum TokenType
+public enum TokenType
 {
     UsingBegin,
     UsingEnd,
@@ -20,7 +20,7 @@ internal enum TokenType
 }
 
 
-internal static  class TokenTypeExtension {
+public static  class TokenTypeExtension {
 
     public static string GetTokenKeyword(this TokenType tokenType)
     {

ide/src/Fiona.IDE.Compiler/Tokens/Tokenizer.cs → ide/src/Fiona.IDE.Tokenizer/Tokenizer.cs

@@ -1,9 +1,9 @@
-using Fiona.IDE.Compiler.Tokens.Exceptons;
+using Fiona.IDE.Tokenizer.Exceptions;
 using System.Text;
 
-namespace Fiona.IDE.Compiler.Tokens;
+namespace Fiona.IDE.Tokenizer;
 
-internal static class Tokenizer
+public static class Tokenizer
 {
     private static readonly char[] SplitChars = [';', '*'];
 

ide/tests/Fiona.IDE.Compiler.Tests/Parser/ParserTests.cs

@@ -1,8 +1,7 @@
-using Fiona.IDE.Compiler.Parser;
 using Fiona.IDE.Compiler.Parser.Exceptions;
 using Fiona.IDE.Compiler.Tests.Shared;
-using Fiona.IDE.Compiler.Tokens;
 using Fiona.IDE.ProjectManager.Models;
+using Fiona.IDE.Tokenizer;
 using FluentAssertions;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -27,7 +26,7 @@ public async Task When_Given_TokenizedCode_Should_Return_Parsed_Code()
         using StreamReader reader = new(stream);
 
         // act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         ReadOnlyMemory<byte> result = await _parser.ParseAsync(tokens, projectFile);
         string resultReader = Encoding.UTF8.GetString(result.ToArray());
 
@@ -64,7 +63,7 @@ public async Task When_Given_TokenizedCodeWithParameter_Should_Return_Parsed_Cod
         using StreamReader reader = new(stream);
 
         // act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         ReadOnlyMemory<byte> result = await _parser.ParseAsync(tokens, projectFile);
         string resultReader = Encoding.UTF8.GetString(result.ToArray());
 
@@ -108,7 +107,7 @@ public async Task When_Given_TokenizedCodeWithParameterAndBody_Should_Return_Par
         using StreamReader reader = new(stream);
 
         // act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         ReadOnlyMemory<byte> result = await _parser.ParseAsync(tokens, projectFile);
         string resultReader = Encoding.UTF8.GetString(result.ToArray());
 

ide/tests/Fiona.IDE.Compiler.Tests/Parser/ParserTestsData.cs

@@ -1,4 +1,4 @@
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 
 namespace Fiona.IDE.Compiler.Tests.Parser;
 

ide/tests/Fiona.IDE.Compiler.Tests/Tokens/TokenizerTests.cs

@@ -1,5 +1,5 @@
 using Fiona.IDE.Compiler.Tests.Shared;
-using Fiona.IDE.Compiler.Tokens;
+using Fiona.IDE.Tokenizer;
 using FluentAssertions;
 using System.Text;
 
@@ -14,7 +14,7 @@ public async Task When_Given_String_With_Using_Clause_Should_Return_Using_Tokens
         using MemoryStream stream = new(Encoding.UTF8.GetBytes(SampleTestCode.UsingTokens!));
         using StreamReader reader = new(stream);
         // Act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         // Assert
         tokens.Count.Should().Be(5);
         tokens.First().Value.Should().Be(TokenType.UsingBegin.ToString());
@@ -36,7 +36,7 @@ public async Task When_Given_String_With_Controller_Clause_Should_Return_Control
         using MemoryStream stream = new(Encoding.UTF8.GetBytes(SampleTestCode.ControllerTokens));
         using StreamReader reader = new(stream);
         // Act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         // Assert
         tokens.Count.Should().Be(9);
         tokens.First().Value.Should().Be("Token.Test");
@@ -66,7 +66,7 @@ public async Task When_Given_String_With_Full_Endpoint_Definition_Clause_Should_
         using MemoryStream stream = new(Encoding.UTF8.GetBytes(SampleTestCode.FullTokensTest!));
         using StreamReader reader = new(stream);
         // Act
-        IReadOnlyCollection<IToken> tokens = await Tokenizer.GetTokensAsync(reader);
+        IReadOnlyCollection<IToken> tokens = await Tokenizer.Tokenizer.GetTokensAsync(reader);
         // Assert
         tokens.Count.Should().Be(14);
         tokens.First().Value.Should().Be(TokenType.UsingBegin.ToString());