diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 90a2848a..d5c21a48 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -7,6 +7,11 @@ on: - main paths-ignore: - docs/ + pull_request: + branches: + - main + paths-ignore: + - docs/ workflow_dispatch: permissions: @@ -129,17 +134,8 @@ jobs: - name: Build Test DLLs ${{ matrix.arch }} run: | - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }} - - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }} + dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf + dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }} env: VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} @@ -169,17 +165,8 @@ jobs: - name: Build Test DLLs ${{ matrix.arch }} run: | - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.MariaDB -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.SqlServer -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Oracle -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.PostgreSQL -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/CommandLine/CommandLine.Sqlite -o ./integration-tests/${{ matrix.arch }} - - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/MariaDB -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/SqlServer -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Oracle -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/PostgreSQL -o ./integration-tests/${{ matrix.arch }} - dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release ./unittests/Sqlite -o ./integration-tests/${{ matrix.arch }} + dotnet restore -r ${{ matrix.arch }} unittests/UnitTests.slnf + dotnet publish -f net8.0 -r ${{ matrix.arch }} -c release --no-restore ./unittests/UnitTests.slnf -o ./integration-tests/${{ matrix.arch }} env: VERSION: ${{ needs.set-version-number.outputs.nuGetVersion }} @@ -360,13 +347,26 @@ jobs: run: | dotnet test integration-tests/linux-x64/CommandLine.${{ matrix.database.type }}.dll ` --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml" - env: LogLevel: Information GrateExecutablePath: ${{ github.workspace }}/executables/linux-x64/grate TZ: UTC GrateTestConfig__DockerImage: ${{ matrix.database.image }} + - name: Docker Test Docker + # TODO: We need to integrate build of a docker image to test into the pipeline, and push it to the local registry, + # and use that one, to run the integration tests. If not, we are testing the latest published released. + #if: ${{ !cancelled() }} + if: false + shell: pwsh + run: | + dotnet test integration-tests/linux-x64/Docker.${{ matrix.database.type }}.dll ` + --logger:"xunit;Environment=${{ matrix.database.name }} (docker);LogFilePath=/tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml" + env: + LogLevel: Information + TZ: UTC + GrateTestConfig__DockerImage: ${{ matrix.database.image }} + - name: Upload Unit Test Results if: always() uses: actions/upload-artifact@v4 @@ -374,6 +374,7 @@ jobs: name: "docker ${{ matrix.database.name }} XML test results" path: | /tmp/test-results/docker/CommandLine.${{ matrix.database.name }}.xml + /tmp/test-results/docker/Docker.${{ matrix.database.name }}.xml retention-days: 1 integration-test: @@ -513,7 +514,8 @@ jobs: report: runs-on: ubuntu-latest name: Create test report - needs: integration-test + #needs: integration-test + needs: docker-integration-tests if: always() steps: diff --git a/src/grate.core/Migration/AnsiSqlDatabase.cs b/src/grate.core/Migration/AnsiSqlDatabase.cs index bf4caf0a..57a6a6ed 100644 --- a/src/grate.core/Migration/AnsiSqlDatabase.cs +++ b/src/grate.core/Migration/AnsiSqlDatabase.cs @@ -21,11 +21,13 @@ public abstract record AnsiSqlDatabase : IDatabase protected GrateConfiguration? Config { get; private set; } - protected ILogger Logger { get; } + protected ILogger Logger { get; private set; } private DbConnection? _connection; private DbConnection? _adminConnection; private readonly ISyntax _syntax; + public void SetLogger(ILogger logger) => Logger = logger; + private IDictionary? _scriptsRunCache; private readonly List> _deferredWrites = new(); diff --git a/src/grate.core/Migration/DbMigrator.cs b/src/grate.core/Migration/DbMigrator.cs index d7b00ef8..7b588a93 100644 --- a/src/grate.core/Migration/DbMigrator.cs +++ b/src/grate.core/Migration/DbMigrator.cs @@ -9,12 +9,21 @@ namespace grate.Migration; internal record DbMigrator : IDbMigrator { - public ILogger Logger { get; set; } + public ILogger Logger + { + get => _logger; + set + { + _logger = value; + Database.SetLogger(value); + } + } + private readonly IHashGenerator _hashGenerator; public DbMigrator(IDatabase database, ILogger logger, IHashGenerator hashGenerator, GrateConfiguration? configuration = null) { - Logger = logger; + _logger = logger; _hashGenerator = hashGenerator; Configuration = configuration ?? throw new ArgumentException("No configuration passed to DbMigrator. Container setup error?", nameof(configuration)); Database = database; @@ -247,6 +256,9 @@ private string GetHash(string sql) /// Lazily initialised only if needed. /// private Dictionary? _tokens; + + private ILogger _logger; + private string ReplaceTokensIn(string sql) { _tokens ??= new TokenProvider(Configuration, Database).GetTokens(); diff --git a/src/grate.core/Migration/IDatabase.cs b/src/grate.core/Migration/IDatabase.cs index e881e6a6..147609d0 100644 --- a/src/grate.core/Migration/IDatabase.cs +++ b/src/grate.core/Migration/IDatabase.cs @@ -1,5 +1,6 @@ using System.Data.Common; using grate.Configuration; +using Microsoft.Extensions.Logging; namespace grate.Migration; @@ -26,6 +27,8 @@ public interface IDatabase : IAsyncDisposable, ICloneable Task CreateDatabase(); Task RestoreDatabase(string backupPath); + void SetLogger(ILogger logger); + /// /// Drops the database if it exists, and does nothing if it doesn't. /// diff --git a/unittests/Basic_tests/Basic_tests.csproj b/unittests/Basic_tests/Basic_tests.csproj index bbd76904..f0d120a8 100644 --- a/unittests/Basic_tests/Basic_tests.csproj +++ b/unittests/Basic_tests/Basic_tests.csproj @@ -6,8 +6,9 @@ enable false - false + true true + Basic_tests diff --git a/unittests/Basic_tests/GrateMigrator_.cs b/unittests/Basic_tests/GrateMigrator_.cs index 9226d5e6..e7f8cedf 100644 --- a/unittests/Basic_tests/GrateMigrator_.cs +++ b/unittests/Basic_tests/GrateMigrator_.cs @@ -12,7 +12,12 @@ namespace Basic_tests; public class GrateMigrator_ { private readonly IDatabase _database = Substitute.For(); - private readonly GrateConfiguration? _config = new GrateConfiguration(); + private readonly GrateConfiguration? _config = new(); + + public GrateMigrator_() + { + _database.Clone().Returns(_database); + } [Fact] public void Setting_the_config_does_not_change_the_original() diff --git a/unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj b/unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj index fc1fbc8a..404e224d 100644 --- a/unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj +++ b/unittests/CommandLine/CommandLine.Common/CommandLine.Common.csproj @@ -5,10 +5,11 @@ enable enable - false + true false false - + IL2091;IL2026 + CommandLine.Common.Startup diff --git a/unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj b/unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj index a162a899..15fd0fc4 100644 --- a/unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj +++ b/unittests/CommandLine/CommandLine.MariaDB/CommandLine.MariaDB.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj b/unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj index 915d93e8..7208f38f 100644 --- a/unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj +++ b/unittests/CommandLine/CommandLine.Oracle/CommandLine.Oracle.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj b/unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj index 841b2cf0..ea98d5a8 100644 --- a/unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj +++ b/unittests/CommandLine/CommandLine.PostgreSQL/CommandLine.PostgreSQL.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj b/unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj index 773c17a8..8b4a0c45 100644 --- a/unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj +++ b/unittests/CommandLine/CommandLine.SqlServer/CommandLine.SqlServer.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj b/unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj index d4a2db5e..179f2f49 100644 --- a/unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj +++ b/unittests/CommandLine/CommandLine.Sqlite/CommandLine.Sqlite.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/Docker/Docker.Common/Docker.Common.csproj b/unittests/Docker/Docker.Common/Docker.Common.csproj index b83dab4b..830d9215 100644 --- a/unittests/Docker/Docker.Common/Docker.Common.csproj +++ b/unittests/Docker/Docker.Common/Docker.Common.csproj @@ -5,10 +5,11 @@ enable enable - false + true false false - + IL2091;IL2026 + Docker.Common.Startup diff --git a/unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs b/unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs index 4597858f..d900de56 100644 --- a/unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs +++ b/unittests/Docker/Docker.Common/TestInfrastructure/DockerGrateMigrator.cs @@ -25,6 +25,33 @@ INetwork Network public async Task Migrate() { + // Sqlite doesn't like to be extracted to a tmpfs mount, so we need to use a bind mount to a temporary folder + // if not, we get an error like this: + // /home/app/.net/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object + // (similarly) /tmp/dotnet-bundle-extract/grate/fdcA3gxdjBiIcVt0mGoBJ5IgxSbD0kE=/libe_sqlite3.so: failed to map segment from shared object + var tmpFolder = Directory.CreateTempSubdirectory().ToString(); + + // Need to map the SQL files directory to the container + var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString(); + + Dictionary bindMounts = new Dictionary + { + { tmpFolder, "/home/app" }, + { sqlFilesDirectory, sqlFilesDirectory } + }; + + // For Sqlite, the database file needs to be mapped as bind mounts, to be able to access the files + // both from the host and the container. + if (DatabaseType == DatabaseType.SQLite) + { + foreach (var connectionString in new [] {Configuration.ConnectionString, Configuration.AdminConnectionString}) + { + var dbFile = connectionString?.Split("=", 2)[1]; + var folder = Path.GetDirectoryName(dbFile)!; + bindMounts[folder] = folder; + } + } + // Convert configuration to command-line arguments var convertToDockerArguments = ConvertToDockerArguments(Configuration); var dockerArguments = convertToDockerArguments.ToList(); @@ -32,27 +59,31 @@ public async Task Migrate() // Add the database type dockerArguments.Add("--databasetype=" + DatabaseType.ToString().ToLowerInvariant()); + //dockerArguments.Add("--verbosity=debug"); + // Needed when overriding the entrypoint, not the command dockerArguments.Insert(0, "./grate"); - - // Need to map the SQL files directory to the container - var sqlFilesDirectory = Configuration.SqlFilesDirectory.ToString(); var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20)); var token = cancellationTokenSource.Token; - await using var container = new ContainerBuilder() + var containerBuilder = new ContainerBuilder() .WithImage(DockerImage) - .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/tmp/dotnet-bundle-extract") .WithEntrypoint(dockerArguments.ToArray()) + .WithEnvironment("DOTNET_BUNDLE_EXTRACT_BASE_DIR", "/home/app") // This is dependent on changing the image to only use grate as ENTRYPOINT, and the rest as CMD //.WithCommand(dockerArguments.ToArray()) - .WithBindMount(sqlFilesDirectory, sqlFilesDirectory) .WithCreateParameterModifier(param => param.HostConfig.ReadonlyRootfs = true) .WithTmpfsMount("/tmp") .WithNetwork(Network) - .WithLogger(Logger) - .Build(); + .WithLogger(Logger); + + foreach (var bindMount in bindMounts) + { + containerBuilder = containerBuilder.WithBindMount(bindMount.Key, bindMount.Value); + } + + await using var container = containerBuilder.Build(); try { diff --git a/unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj b/unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj index fd3f5944..001a9a9c 100644 --- a/unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj +++ b/unittests/Docker/Docker.MariaDB/Docker.MariaDB.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/Docker/Docker.Oracle/Docker.Oracle.csproj b/unittests/Docker/Docker.Oracle/Docker.Oracle.csproj index f2416bf4..9b33919c 100644 --- a/unittests/Docker/Docker.Oracle/Docker.Oracle.csproj +++ b/unittests/Docker/Docker.Oracle/Docker.Oracle.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj b/unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj index 393b7474..8e82fd85 100644 --- a/unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj +++ b/unittests/Docker/Docker.PostgreSQL/Docker.PostgreSQL.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/Docker/Docker.SqlServer/Docker.SqlServer.csproj b/unittests/Docker/Docker.SqlServer/Docker.SqlServer.csproj index a06d9df3..a3d29dc4 100644 --- a/unittests/Docker/Docker.SqlServer/Docker.SqlServer.csproj +++ b/unittests/Docker/Docker.SqlServer/Docker.SqlServer.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/Docker/Docker.Sqlite/Docker.Sqlite.csproj b/unittests/Docker/Docker.Sqlite/Docker.Sqlite.csproj index 4ec396a7..fe0511e1 100644 --- a/unittests/Docker/Docker.Sqlite/Docker.Sqlite.csproj +++ b/unittests/Docker/Docker.Sqlite/Docker.Sqlite.csproj @@ -5,10 +5,10 @@ enable enable - false + true false true - + diff --git a/unittests/MariaDB/MariaDB.csproj b/unittests/MariaDB/MariaDB.csproj index b7a60bfa..3b3dc1ef 100644 --- a/unittests/MariaDB/MariaDB.csproj +++ b/unittests/MariaDB/MariaDB.csproj @@ -7,7 +7,8 @@ true false - false + true + diff --git a/unittests/Oracle/Oracle.csproj b/unittests/Oracle/Oracle.csproj index e5b720a1..9abb8456 100644 --- a/unittests/Oracle/Oracle.csproj +++ b/unittests/Oracle/Oracle.csproj @@ -6,8 +6,9 @@ enable true - false + true false + diff --git a/unittests/PostgreSQL/PostgreSQL.csproj b/unittests/PostgreSQL/PostgreSQL.csproj index f1c9528e..b4dc132b 100644 --- a/unittests/PostgreSQL/PostgreSQL.csproj +++ b/unittests/PostgreSQL/PostgreSQL.csproj @@ -6,8 +6,9 @@ enable true - false + true false + diff --git a/unittests/SqlServer/SqlServer.csproj b/unittests/SqlServer/SqlServer.csproj index c5c043be..30d037c0 100644 --- a/unittests/SqlServer/SqlServer.csproj +++ b/unittests/SqlServer/SqlServer.csproj @@ -6,8 +6,9 @@ enable true - false + true false + diff --git a/unittests/SqlServerCaseSensitive/SqlServerCaseSensitive.csproj b/unittests/SqlServerCaseSensitive/SqlServerCaseSensitive.csproj index fe3d7495..84b0c661 100644 --- a/unittests/SqlServerCaseSensitive/SqlServerCaseSensitive.csproj +++ b/unittests/SqlServerCaseSensitive/SqlServerCaseSensitive.csproj @@ -6,8 +6,9 @@ enable true - false + true false + diff --git a/unittests/Sqlite/Sqlite.csproj b/unittests/Sqlite/Sqlite.csproj index f4daa195..16a3ba82 100644 --- a/unittests/Sqlite/Sqlite.csproj +++ b/unittests/Sqlite/Sqlite.csproj @@ -6,8 +6,9 @@ enable true - false + true false + diff --git a/unittests/TestCommon/TestCommon.csproj b/unittests/TestCommon/TestCommon.csproj index 9ef4c946..ca3f75f3 100644 --- a/unittests/TestCommon/TestCommon.csproj +++ b/unittests/TestCommon/TestCommon.csproj @@ -5,9 +5,9 @@ enable enable false - false + true false - IL2072;IL2075;IL2026 + IL2072;IL2075;IL2026 diff --git a/unittests/UnitTests.slnf b/unittests/UnitTests.slnf new file mode 100644 index 00000000..b24e9c80 --- /dev/null +++ b/unittests/UnitTests.slnf @@ -0,0 +1,27 @@ +{ + "solution": { + "path": "..\\grate.sln", + "projects": [ + "unittests\\Basic_tests\\Basic_tests.csproj", + "unittests\\CommandLine\\CommandLine.Common\\CommandLine.Common.csproj", + "unittests\\CommandLine\\CommandLine.MariaDB\\CommandLine.MariaDB.csproj", + "unittests\\CommandLine\\CommandLine.Oracle\\CommandLine.Oracle.csproj", + "unittests\\CommandLine\\CommandLine.PostgreSQL\\CommandLine.PostgreSQL.csproj", + "unittests\\CommandLine\\CommandLine.Sqlite\\CommandLine.Sqlite.csproj", + "unittests\\CommandLine\\CommandLine.SqlServer\\CommandLine.SqlServer.csproj", + "unittests\\Docker\\Docker.Common\\Docker.Common.csproj", + "unittests\\Docker\\Docker.MariaDB\\Docker.MariaDB.csproj", + "unittests\\Docker\\Docker.Oracle\\Docker.Oracle.csproj", + "unittests\\Docker\\Docker.PostgreSQL\\Docker.PostgreSQL.csproj", + "unittests\\Docker\\Docker.Sqlite\\Docker.Sqlite.csproj", + "unittests\\Docker\\Docker.SqlServer\\Docker.SqlServer.csproj", + "unittests\\MariaDB\\MariaDB.csproj", + "unittests\\Oracle\\Oracle.csproj", + "unittests\\PostgreSQL\\PostgreSQL.csproj", + "unittests\\Sqlite\\Sqlite.csproj", + "unittests\\SqlServerCaseSensitive\\SqlServerCaseSensitive.csproj", + "unittests\\SqlServer\\SqlServer.csproj", + "unittests\\TestCommon\\TestCommon.csproj" + ] + } +} \ No newline at end of file