Chore: Fixed SqLite docker tests

* Added Grate-in-Docker tests to mid-weight integration tests
* Made 'SetLogger' public, to be able to set from tests
* Some fixes to be able to run publish on the whole solution
* Added UnitTests solution filter file
* Build .slnf in integration test build step
* Disable Docker tests for now in pipeline, as we have to introduce a "build docker image" step before enabling it

.github/workflows/integration-tests.yml

@@ -7,6 +7,11 @@ on:
  - main
  paths-ignore:
  - docs/
+ pull_request:
+ branches:
+ - main
+ paths-ignore:
+ - docs/
  workflow_dispatch:
 
 permissions:
@@ -129,17 +134,8 @@ jobs:
 
  - name: Build Test DLLs ${{ matrix.arch }}
  run: |
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }}
-
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }}
+ dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
+ dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}
 
  env:
  VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
@@ -169,17 +165,8 @@ jobs:
 
  - name: Build Test DLLs ${{ matrix.arch }}
  run: |
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }}
-
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }}
- dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }}
+ dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf
+ dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }}
 
  env:
  VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }}
@@ -360,20 +347,34 @@ jobs:
  run: |
  dotnet test integration-tests/linux-x64/CommandLine.${{ matrix.database.type }}.dll `
  --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml"
-
  env:
  LogLevel: Information
  GrateExecutablePath: ${{ github.workspace }}/executables/linux-x64/grate
  TZ: UTC
  GrateTestConfig__DockerImage: ${{ matrix.database.image }} 
 
+ - name: Docker Test Docker
+ # TODO: We need to integrate build of a docker image to test into the pipeline, and push it to the local registry,
+ # and use that one, to run the integration tests. If not, we are testing the latest published released.
+ #if: ${{ !cancelled() }}
+ if: false
+ shell: pwsh
+ run: |
+ dotnet test integration-tests/linux-x64/Docker.${{ matrix.database.type }}.dll `
+ --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml"
+ env:
+ LogLevel: Information
+ TZ: UTC
+ GrateTestConfig__DockerImage: ${{ matrix.database.image }} 
+
  - name: Upload Unit Test Results
  if: always()
  uses: actions/upload-artifact@v4
  with:
  name: "docker ${{ matrix.database.name }} XML test results"
  path: |
  /tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml
+ /tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml
  retention-days: 1
 
  integration-test:
@@ -513,7 +514,8 @@ jobs:
  report:
  runs-on: ubuntu-latest
  name: Create test report
- needs: integration-test
+ #needs: integration-test
+ needs: docker-integration-tests
  if: always()
 
  steps:

src/grate.core/Migration/AnsiSqlDatabase.cs

@@ -21,11 +21,13 @@ public abstract record AnsiSqlDatabase : IDatabase
 
  protected GrateConfiguration? Config { get; private set; }
 
- protected ILogger Logger { get; }
+ protected ILogger Logger { get; private set; }
  private DbConnection? _connection;
  private DbConnection? _adminConnection;
  private readonly ISyntax _syntax;
 
+ public void SetLogger(ILogger logger) => Logger = logger;
+
  private IDictionary<string, string>? _scriptsRunCache;
 
  private readonly List<Func<DbConnection, Task>> _deferredWrites = new();

src/grate.core/Migration/DbMigrator.cs

@@ -9,12 +9,21 @@ namespace grate.Migration;
 
 internal record DbMigrator : IDbMigrator
 {
- public ILogger Logger { get; set; }
+ public ILogger Logger
+ {
+ get => _logger;
+ set
+ {
+ _logger = value;
+ Database.SetLogger(value);
+ }
+ }
+
  private readonly IHashGenerator _hashGenerator;
 
  public DbMigrator(IDatabase database, ILogger<DbMigrator> logger, IHashGenerator hashGenerator, GrateConfiguration? configuration = null)
  {
- Logger = logger;
+ _logger = logger;
  _hashGenerator = hashGenerator;
  Configuration = configuration ?? throw new ArgumentException("No configuration passed to DbMigrator. Container setup error?", nameof(configuration));
  Database = database;
@@ -247,6 +256,9 @@ private string GetHash(string sql)
  /// Lazily initialised only if needed.
  /// </summary>
  private Dictionary<string, string?>? _tokens;
+
+ private ILogger _logger;
+
  private string ReplaceTokensIn(string sql)
  {
  _tokens ??= new TokenProvider(Configuration, Database).GetTokens();

src/grate.core/Migration/IDatabase.cs

@@ -1,5 +1,6 @@
 using System.Data.Common;
 using grate.Configuration;
+using Microsoft.Extensions.Logging;
 
 namespace grate.Migration;
 
@@ -26,6 +27,8 @@ public interface IDatabase : IAsyncDisposable, ICloneable
  Task CreateDatabase();
  Task RestoreDatabase(string backupPath);
 
+ void SetLogger(ILogger logger);
+
  /// <summary>
  /// Drops the database if it exists, and does nothing if it doesn't.
  /// </summary>

unittests/Basic_tests/Basic_tests.csproj

@@ -6,8 +6,9 @@
  <Nullable>enable</Nullable>
 
  <IsPackable>false</IsPackable>
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsTestProject>true</IsTestProject>
+
  <RootNamespace>Basic_tests</RootNamespace>
  </PropertyGroup>
 

unittests/Basic_tests/GrateMigrator_.cs

@@ -12,7 +12,12 @@ namespace Basic_tests;
 public class GrateMigrator_
 {
  private readonly IDatabase _database = Substitute.For<IDatabase>();
- private readonly GrateConfiguration? _config = new GrateConfiguration();
+ private readonly GrateConfiguration? _config = new();
+
+ public GrateMigrator_()
+ {
+ _database.Clone().Returns(_database);
+ }
 
  [Fact]
  public void Setting_the_config_does_not_change_the_original()

unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj

@@ -5,10 +5,11 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>false</IsTestProject>
-
+ <NoWarn>IL2091;IL2026</NoWarn>
+
  <XunitStartupFullName>CommandLine.Common.Startup</XunitStartupFullName>
  </PropertyGroup>
 

unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/Docker/Docker.Common/Docker.Common.csproj

@@ -5,10 +5,11 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>false</IsTestProject>
-
+ <NoWarn>IL2091;IL2026</NoWarn>
+
  <XunitStartupFullName>Docker.Common.Startup</XunitStartupFullName>
  </PropertyGroup>
 

unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs

@@ -25,34 +25,65 @@ INetwork Network
 
  public async Task Migrate()
  {
+ // Sqlite doesn't like to be extracted to a tmpfs mount, so we need to use a bind mount to a temporary folder
+ // if not, we get an error like this:
+ // /home/app/.net/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object
+ // (similarly) /tmp/dotnet-bundle-extract/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object
+ var tmpFolder = Directory.CreateTempSubdirectory().ToString();
+
+ // Need to map the SQL files directory to the container
+ var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString();
+
+ Dictionary<string, string> bindMounts = new Dictionary<string, string>
+ {
+ { tmpFolder, "/home/app" },
+ { sqlFilesDirectory, sqlFilesDirectory }
+ };
+
+ // For Sqlite, the database file needs to be mapped as bind mounts, to be able to access the files
+ // both from the host and the container.
+ if (DatabaseType == DatabaseType.SQLite)
+ {
+ foreach (var connectionString in new [] {Configuration.ConnectionString, Configuration.AdminConnectionString})
+ {
+ var dbFile = connectionString?.Split("=", 2)[1];
+ var folder = Path.GetDirectoryName(dbFile)!;
+ bindMounts[folder] = folder;
+ }
+ }
+
  // Convert configuration to command-line arguments
  var convertToDockerArguments = ConvertToDockerArguments(Configuration);
  var dockerArguments = convertToDockerArguments.ToList();
 
  // Add the database type
  dockerArguments.Add("--databasetype=" + DatabaseType.ToString().ToLowerInvariant());
 
+ //dockerArguments.Add("--verbosity=debug");
+
  // Needed when overriding the entrypoint, not the command
  dockerArguments.Insert(0, "./grate");
-
- // Need to map the SQL files directory to the container
- var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString();
 
  var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));
  var token = cancellationTokenSource.Token;
 
- await using var container = new ContainerBuilder()
+ var containerBuilder = new ContainerBuilder()
  .WithImage(DockerImage)
- .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/tmp/dotnet-bundle-extract")
  .WithEntrypoint(dockerArguments.ToArray())
+ .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/home/app")
  // This is dependent on changing the image to only use grate as ENTRYPOINT, and the rest as CMD
  //.WithCommand(dockerArguments.ToArray())
- .WithBindMount(sqlFilesDirectory, sqlFilesDirectory)
  .WithCreateParameterModifier(param => param.HostConfig.ReadonlyRootfs = true)
  .WithTmpfsMount("/tmp")
  .WithNetwork(Network)
- .WithLogger(Logger)
- .Build();
+ .WithLogger(Logger);
+
+ foreach (var bindMount in bindMounts)
+ {
+ containerBuilder = containerBuilder.WithBindMount(bindMount.Key, bindMount.Value);
+ }
+
+ await using var container = containerBuilder.Build();
 
  try
  {

unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/Docker/Docker.Oracle/Docker.Oracle.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>

unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj

@@ -5,10 +5,10 @@
  <ImplicitUsings>enable</ImplicitUsings>
  <Nullable>enable</Nullable>
 
- <IsPublishable>false</IsPublishable>
+ <IsPublishable>true</IsPublishable>
  <IsPackable>false</IsPackable>
  <IsTestProject>true</IsTestProject>
-
+ 
  </PropertyGroup>
 
  <ItemGroup>