[WIP] chore: Create simple CI test for persistent, refreshed database

.github/workflows/create-dependencies-pr.yaml

@@ -7,7 +7,8 @@ on:
 
 jobs:
   generate:
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
 
     steps:
       - name: Checkout OpenActive.Server.NET

.github/workflows/openactive-test-suite.yml

@@ -8,7 +8,8 @@ on:
 
 jobs:
   test-server:
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
     steps:
       # Checks out this project ie OpenActive.Server.NET
     - name: Checkout
@@ -32,7 +33,8 @@ jobs:
       run: dotnet test ./OpenActive.Server.NET.Tests/OpenActive.Server.NET.Tests.csproj --configuration Release --no-build --verbosity normal
 
   test-fake-database:
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
     steps:
       # Checks out this project ie OpenActive.Server.NET
     - name: Checkout
@@ -55,13 +57,65 @@ jobs:
       # output
     - name: Run OpenActive.FakeDatabase.NET.Tests
       run: dotnet test ./Fakes/OpenActive.FakeDatabase.NET.Tests/OpenActive.FakeDatabase.NET.Tests.csproj --configuration Release --no-build --verbosity normal
-
+
+  # TODO document this job in .github/workflows/README.md
+  test-persistent-database:
+    needs:
+      - test-server
+      - test-fake-database
+    runs-on: ubuntu-latest
+    steps:
+      # INITIAL SET UP
+      - name: Checkout OpenActive.Server.NET
+        uses: actions/checkout@v2
+        with:
+          path: server
+
+      # checks if the workflow is triggered by a branch starting with `coverage/`. If so, it sets an output variable named
+      # `mirror_ref` with the branch name, allowing later steps to use it
+      - name: Use matching coverage/* branch ${{ github.head_ref }} in OpenActive Test Suite 
+        if: ${{ startsWith(github.head_ref, 'coverage/') }}
+        id: refs
+        run: echo "::set-output name=mirror_ref::${{ github.head_ref }}"
+
+      # checks out Test Suite and places it in a directory named "tests." The branch used for checkout is determined by
+      # the `mirror_ref` set in the previous step  
+      - name: Checkout OpenActive Test Suite ${{ steps.refs.outputs.mirror_ref }}
+        uses: actions/checkout@v2
+        with:
+          repository: openactive/openactive-test-suite
+          ref: ${{ steps.refs.outputs.mirror_ref }}
+          path: tests
+
+      # sets up .NET for running OpenActive.Server.NET
+      - name: Setup .NET Core SDK 3.1.419
+        uses: actions/setup-dotnet@v1
+        with:
+          dotnet-version: 3.1.419
+      # sets up Node.js for running Test Suite
+      - name: Setup Node.js 18.17.1
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18.17.1
+
+      - name: Install OpenActive Test Suite
+        run: npm install
+        working-directory: tests
+
+      - name: Build .NET Core Booking Server Reference Implementation
+        run: dotnet build ./server/Examples/BookingSystem.AspNetCore/BookingSystem.AspNetCore.csproj --configuration Release
+
+      # Run tests
+      - name: Run tests
+        run: node ./scripts/testPersistentDatabase.js
+
   core:
     # Specifies that this job depends on the successful completion of two other jobs: "test-server" and "test-fake-database"
     needs:
       - test-server
       - test-fake-database
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
 
     # Defines a matrix strategy for running tests with different combinations of parameters, allowing for parallel test
     # runs with various configurations. For more information about these parameters, see `.github/README.md` in Test
@@ -107,6 +161,7 @@ jobs:
 
     # runs `dotnet restore` to install the dependencies for the "OpenActive.Server.NET" project. It is conditional and
     # depends on the "profile" value not being 'no-auth' or 'single-seller'
+    # LW: Why is it conditional?
     - name: Install OpenActive.Server.NET dependencies
       if: ${{ matrix.profile != 'no-auth' && matrix.profile != 'single-seller' }}
       run: dotnet restore ./server/
@@ -154,7 +209,7 @@ jobs:
 
     # uploads the test output as an artifact, which can be used for reference or debugging later  
     - name: Upload test output for ${{ matrix.mode }} mode as artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       if: ${{ success() || failure() }}
       with:
         name: core.${{ matrix.mode }}.${{ matrix.profile }}
@@ -234,7 +289,7 @@ jobs:
         NODE_APP_INSTANCE: framework
       working-directory: tests
     - name: Upload test output for ${{ matrix.mode }} mode as artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       if: ${{ success() || failure() }}
       with:
         name: framework.${{ matrix.mode }}.${{ matrix.profile }}
@@ -247,7 +302,8 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' }}
     needs:
       - core
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
     steps:
       # Checkout the repo
       - uses: actions/checkout@master
@@ -291,7 +347,8 @@ jobs:
     needs:
       - core
       - framework
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -342,7 +399,8 @@ jobs:
     needs:
       - core
       - framework
-    runs-on: ubuntu-latest
+    # ubuntu-latest's SSL library no longer supports .NET 3.1
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2

.gitignore

@@ -336,3 +336,6 @@ ASALocalRun/
 
 # Fake database
 *fakedatabase.db
+
+# Output path for app publishing
+/web-app-package/

Examples/BookingSystem.AspNetCore.IdentityServer/Startup.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Configuration;
 using OpenActive.FakeDatabase.NET;
+using Microsoft.Extensions.Logging;
 
 namespace IdentityServer
 {
@@ -27,7 +28,11 @@ public Startup(IWebHostEnvironment environment, IConfiguration configuration)
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddTransient<IClientStore, ClientStore>();
-            services.AddSingleton(x => new FakeBookingSystem(AppSettings.FeatureFlags.FacilityUseHasSlots));
+            services.AddSingleton(x => new FakeBookingSystem
+            (
+                AppSettings.FeatureFlags.FacilityUseHasSlots,
+                x.GetRequiredService<ILogger<FakeBookingSystem>>()
+            ));
 
             var builder = services.AddIdentityServer(options =>
                 {

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+using BookingSystem.AspNetCore.Services;
+
+
+namespace BookingSystem
+{
+    /// <summary>
+    /// A background task which periodically refreshes the data in the
+    /// FakeBookingSystem. This means that past data is deleted and new copies
+    /// are created in the future.
+    ///
+    /// More information on background tasks here:
+    /// https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+    /// </summary>
+    public class FakeDataRefresherService : BackgroundService
+    {
+        private readonly ILogger<FakeDataRefresherService> _logger;
+        private readonly AppSettings _settings;
+        private readonly FakeBookingSystem _bookingSystem;
+        private readonly DataRefresherStatusService _statusService;
+
+        public FakeDataRefresherService(
+            AppSettings settings, 
+            ILogger<FakeDataRefresherService> logger,
+            FakeBookingSystem bookingSystem,
+            DataRefresherStatusService statusService)
+        {
+            _settings = settings;
+            _logger = logger;
+            _bookingSystem = bookingSystem;
+            _statusService = statusService;
+
+            // Indicate that the refresher service is configured to run
+            _statusService.SetRefresherConfigured(true);
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            var interval = TimeSpan.FromHours(_settings.DataRefresherIntervalHours);
+
+            stoppingToken.Register(() =>
+                _logger.LogInformation($"FakeDataRefresherService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                _logger.LogInformation($"FakeDataRefresherService is starting..");
+                var (numDeletedOccurrences, numDeletedSlots) = await _bookingSystem
+                  .Database
+                  .HardDeleteOldSoftDeletedOccurrencesAndSlots();
+                _logger.LogInformation($"FakeDataRefresherService hard deleted {numDeletedOccurrences} occurrences and {numDeletedSlots} slots that were previously old and soft-deleted.");
+
+                var (numRefreshedOccurrences, numRefreshedSlots) = await _bookingSystem
+                  .Database
+                  .SoftDeletePastOpportunitiesAndInsertNewAtEdgeOfWindow();
+                _logger.LogInformation($"FakeDataRefresherService, for {numRefreshedOccurrences} old occurrences and {numRefreshedSlots} old slots, inserted new copies into the future and soft-deleted the old ones.");
+
+                _logger.LogInformation($"FakeDataRefresherService is finished");
+
+                // Signal that a cycle has completed
+                _statusService.SignalCycleCompletion();
+
+                await Task.Delay(interval, stoppingToken);
+            }
+        }
+    }
+}

Examples/BookingSystem.AspNetCore/Controllers/InitWaitController.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Threading.Tasks;
+using BookingSystem.AspNetCore.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Endpoints which are used to wait for components within
+/// BookingSystem.AspNetCore to be initialized.
+/// </summary>
+namespace BookingSystem.AspNetCore.Controllers
+{
+    [ApiController]
+    [Route("init-wait")]
+    public class InitWaitController : ControllerBase
+    {
+        private readonly DataRefresherStatusService _statusService;
+        private readonly ILogger<InitWaitController> _logger;
+        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(5);
+
+        public InitWaitController(
+            DataRefresherStatusService statusService,
+            ILogger<InitWaitController> logger)
+        {
+            _statusService = statusService;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Wait for the data refresher to complete its first cycle.
+        ///
+        /// This makes it possible to write scripts (for CI) which don't start
+        /// until the data refresher has completed at least one cycle.
+        ///
+        /// - Returns 204 when the data refresher has completed its first cycle.
+        /// - Returns 503 if the data refresher is not configured to run.
+        /// - Returns 504 if the data refresher fails to complete a cycle within
+        ///   the default timeout.
+        /// </summary>
+        [HttpGet("data-refresher")]
+        public async Task<IActionResult> WaitForDataRefresher()
+        {
+            _logger.LogDebug("Received request to wait for data refresher completion");
+
+            // Check if the data refresher is configured to run
+            if (!_statusService.IsRefresherConfigured())
+            {
+                _logger.LogWarning("Data refresher is not configured to run");
+                return StatusCode(503, "Data refresher service is not configured to run");
+            }
+
+            // If it has already completed a cycle, return immediately
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher has already completed a cycle");
+                return NoContent();
+            }
+
+            _logger.LogDebug("Waiting for data refresher to complete a cycle...");
+
+            // Wait for the cycle to complete, with a timeout
+            await _statusService.WaitForCycleCompletion(_defaultTimeout);
+
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher completed a cycle, returning 204");
+                return NoContent();
+            }
+            else
+            {
+                _logger.LogWarning("Timed out waiting for data refresher to complete a cycle");
+                return StatusCode(504, "Timed out waiting for data refresher to complete a cycle");
+            }
+        }
+    }
+} 

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -1,5 +1,5 @@
-using Bogus;
-using BookingSystem.AspNetCore.Helpers;
+using Bogus;
+using BookingSystem.AspNetCore.Helpers;
 using OpenActive.DatasetSite.NET;
 using OpenActive.FakeDatabase.NET;
 using OpenActive.NET;
@@ -26,6 +26,7 @@ public AcmeFacilityUseRpdeGenerator(AppSettings appSettings, FakeBookingSystem f
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             var facilityTypeId = Environment.GetEnvironmentVariable("FACILITY_TYPE_ID") ?? "https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651";
@@ -204,6 +205,7 @@ public AcmeFacilityUseSlotRpdeGenerator(AppSettings appSettings, FakeBookingSyst
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             using (var db = _fakeBookingSystem.Database.Mem.Database.Open())

Examples/BookingSystem.AspNetCore/README.md

@@ -33,6 +33,15 @@ ASPNETCORE_ENVIRONMENT=no-auth dotnet run --no-launch-profile --project ./Bookin
 
 The above example starts the BookingSystem.AspNetCore in `no-auth` mode.
 
+## Env Vars
+
+BookingSystem.AspNetCore uses the following environment variables (note: this list is not exhaustive):
+
+- `PERIODICALLY_REFRESH_DATA`: (optional - default `false`) If set to `true`,
+  the database will be periodically refreshed, deleting past data and replacing
+  it with future data.
+- `IS_LOREM_FITSUM_MODE`: (optional - default `false`) If set to `true`, data is created in ["Lorem Fitsum" mode](#lorem-fitsum-mode).
+
 ## BookingSystem.AspNetCore Data Generation
 
 BookingSystem.AspNetCore has three main uses that make it very important in the OpenActive ecosystem:
@@ -64,6 +73,6 @@ In the CLI this can be done by running the following command for example:
 IS_LOREM_FITSUM_MODE=true dotnet run --no-launch-profile --project ./BookingSystem.AspNetCore.csproj --configuration Release --no-build
 ```
 
-### Golden Records
+#### Golden Records
 Golden records are randomly generated records that have maximally enriched properties in the generated data. For example where a record might have one image normally, a golden record will have four.