feat: optimize the native library loading.

LLama.Examples/Program.cs

@@ -1,5 +1,6 @@
 using LLama.Native;
 using Spectre.Console;
+using System.Runtime.InteropServices;
 
 AnsiConsole.MarkupLineInterpolated(
     $"""
@@ -16,23 +17,24 @@ __       __                                       ____     __
 
     """);
 
-// Configure native library to use. This must be done before any other llama.cpp methods are called!
-NativeLibraryConfig
-   .Instance
-   .WithCuda();
-
 // Configure logging. Change this to `true` to see log messages from llama.cpp
 var showLLamaCppLogs = false;
 NativeLibraryConfig
-   .Instance
+   .All
    .WithLogCallback((level, message) =>
-    {
-        if (showLLamaCppLogs)
-            Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
-    });
+   {
+       if (showLLamaCppLogs)
+           Console.WriteLine($"[llama {level}]: {message.TrimEnd('\n')}");
+   });
+
+// Configure native library to use. This must be done before any other llama.cpp methods are called!
+NativeLibraryConfig
+   .All
+   .WithCuda()
+   //.WithAutoDownload() // An experimental feature
+   .DryRun(out var loadedllamaLibrary, out var loadedLLavaLibrary);
 
 // Calling this method forces loading to occur now.
 NativeApi.llama_empty_call();
 
-await ExampleRunner.Run();
-
+await ExampleRunner.Run();

LLama/Abstractions/INativeLibrary.cs

@@ -0,0 +1,29 @@
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions
+{
+    /// <summary>
+    /// Descriptor of a native library.
+    /// </summary>
+    public interface INativeLibrary
+    {
+        /// <summary>
+        /// Metadata of this library.
+        /// </summary>
+        NativeLibraryMetadata? Metadata { get; }
+
+        /// <summary>
+        /// Prepare the native library file and returns the local path of it.
+        /// If it's a relative path, LLamaSharp will search the path in the search directies you set.
+        /// </summary>
+        /// <param name="systemInfo">The system information of the current machine.</param>
+        /// <param name="logCallback">The log callback.</param>
+        /// <returns>
+        /// The relative paths of the library. You could return multiple paths to try them one by one. If no file is available, please return an empty array.
+        /// </returns>
+        IEnumerable<string> Prepare(SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null);
+    }
+}

LLama/Abstractions/INativeLibrarySelectingPolicy.cs

@@ -0,0 +1,24 @@
+using LLama.Native;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace LLama.Abstractions
+{
+#if NET6_0_OR_GREATER
+    /// <summary>
+    /// Decides the selected native library that should be loaded according to the configurations.
+    /// </summary>
+    public interface INativeLibrarySelectingPolicy
+    {
+        /// <summary>
+        /// Select the native library.
+        /// </summary>
+        /// <param name="description"></param>
+        /// <param name="systemInfo">The system information of the current machine.</param>
+        /// <param name="logCallback">The log callback.</param>
+        /// <returns>The information of the selected native library files, in order by priority from the beginning to the end.</returns>
+        IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback = null);
+    }
+#endif
+}

LLama/LLamaSharp.csproj

@@ -3,7 +3,7 @@
     <TargetFrameworks>netstandard2.0;net6.0;net8.0</TargetFrameworks>
     <RootNamespace>LLama</RootNamespace>
     <Nullable>enable</Nullable>
-    <LangVersion>10</LangVersion>
+    <LangVersion>12</LangVersion>
     <Platforms>AnyCPU;x64;Arm64</Platforms>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
 

LLama/Native/Load/DefaultNativeLibrarySelectingPolicy.cs

@@ -0,0 +1,69 @@
+using LLama.Abstractions;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace LLama.Native
+{
+#if NET6_0_OR_GREATER
+    /// <inheritdoc/>
+    public class DefaultNativeLibrarySelectingPolicy: INativeLibrarySelectingPolicy
+    {
+        /// <inheritdoc/>
+        public IEnumerable<INativeLibrary> Apply(NativeLibraryConfig.Description description, SystemInfo systemInfo, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            List<INativeLibrary> results = new();
+
+            // Show the configuration we're working with
+            Log(description.ToString(), LLamaLogLevel.Info, logCallback);
+
+            // If a specific path is requested, only use it, no fall back.
+            if (!string.IsNullOrEmpty(description.Path))
+            {
+                yield return new NativeLibraryFromPath(description.Path);
+            }
+            else
+            {
+                if (description.UseCuda)
+                {
+                    yield return new NativeLibraryWithCuda(systemInfo.CudaMajorVersion, description.Library, description.SkipCheck);
+                }
+
+                if(!description.UseCuda || description.AllowFallback)
+                {
+                    if (description.AllowFallback)
+                    {
+                        // Try all of the AVX levels we can support.
+                        if (description.AvxLevel >= AvxLevel.Avx512)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx512, description.SkipCheck);
+
+                        if (description.AvxLevel >= AvxLevel.Avx2)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx2, description.SkipCheck);
+
+                        if (description.AvxLevel >= AvxLevel.Avx)
+                            yield return new NativeLibraryWithAvx(description.Library, AvxLevel.Avx, description.SkipCheck);
+
+                        yield return new NativeLibraryWithAvx(description.Library, AvxLevel.None, description.SkipCheck);
+                    }
+                    else
+                    {
+                        yield return new NativeLibraryWithAvx(description.Library, description.AvxLevel, description.SkipCheck);
+                    }
+                }
+
+                if(systemInfo.OSPlatform == OSPlatform.OSX || description.AllowFallback)
+                {
+                    yield return new NativeLibraryWithMacOrFallback(description.Library, description.SkipCheck);
+                }
+            }
+        }
+
+        private void Log(string message, LLamaLogLevel level, NativeLogConfig.LLamaLogCallback? logCallback)
+        {
+            if (!message.EndsWith("\n"))
+                message += "\n";
+
+            logCallback?.Invoke(level, message);
+        }
+    }
+#endif
+}