feat: Support persisting SQLite DB, and data refresher (#233)

Author: lukehesluke

.github/workflows/openactive-test-suite.yml

@@ -57,7 +57,7 @@ jobs:
       # output
     - name: Run OpenActive.FakeDatabase.NET.Tests
       run: dotnet test ./Fakes/OpenActive.FakeDatabase.NET.Tests/OpenActive.FakeDatabase.NET.Tests.csproj --configuration Release --no-build --verbosity normal
-  
+
   core:
     # Specifies that this job depends on the successful completion of two other jobs: "test-server" and "test-fake-database"
     needs:
@@ -110,6 +110,7 @@ jobs:
 
     # runs `dotnet restore` to install the dependencies for the "OpenActive.Server.NET" project. It is conditional and
     # depends on the "profile" value not being 'no-auth' or 'single-seller'
+    # LW: Why is it conditional?
     - name: Install OpenActive.Server.NET dependencies
       if: ${{ matrix.profile != 'no-auth' && matrix.profile != 'single-seller' }}
       run: dotnet restore ./server/

.gitignore

@@ -336,3 +336,6 @@ ASALocalRun/
 
 # Fake database
 *fakedatabase.db
+
+# Output path for app publishing
+/web-app-package/

Examples/BookingSystem.AspNetCore.IdentityServer/Startup.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Configuration;
 using OpenActive.FakeDatabase.NET;
+using Microsoft.Extensions.Logging;
 
 namespace IdentityServer
 {
@@ -27,7 +28,11 @@ public Startup(IWebHostEnvironment environment, IConfiguration configuration)
         public void ConfigureServices(IServiceCollection services)
         {
             services.AddTransient<IClientStore, ClientStore>();
-            services.AddSingleton(x => new FakeBookingSystem(AppSettings.FeatureFlags.FacilityUseHasSlots));
+            services.AddSingleton(x => new FakeBookingSystem
+            (
+                AppSettings.FeatureFlags.FacilityUseHasSlots,
+                x.GetRequiredService<ILogger<FakeBookingSystem>>()
+            ));
 
             var builder = services.AddIdentityServer(options =>
                 {

Examples/BookingSystem.AspNetCore/BackgroundServices/FakeDataRefresherService.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenActive.FakeDatabase.NET;
+using BookingSystem.AspNetCore.Services;
+
+
+namespace BookingSystem
+{
+    /// <summary>
+    /// A background task which periodically refreshes the data in the
+    /// FakeBookingSystem. This means that past data is deleted and new copies
+    /// are created in the future.
+    ///
+    /// More information on background tasks here:
+    /// https://docs.microsoft.com/en-us/dotnet/architecture/microservices/multi-container-microservice-net-applications/background-tasks-with-ihostedservice#implementing-ihostedservice-with-a-custom-hosted-service-class-deriving-from-the-backgroundservice-base-class
+    /// </summary>
+    public class FakeDataRefresherService : BackgroundService
+    {
+        private readonly ILogger<FakeDataRefresherService> _logger;
+        private readonly AppSettings _settings;
+        private readonly FakeBookingSystem _bookingSystem;
+        private readonly DataRefresherStatusService _statusService;
+
+        public FakeDataRefresherService(
+            AppSettings settings, 
+            ILogger<FakeDataRefresherService> logger,
+            FakeBookingSystem bookingSystem,
+            DataRefresherStatusService statusService)
+        {
+            _settings = settings;
+            _logger = logger;
+            _bookingSystem = bookingSystem;
+            _statusService = statusService;
+
+            // Indicate that the refresher service is configured to run
+            _statusService.SetRefresherConfigured(true);
+        }
+
+        protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+        {
+            var interval = TimeSpan.FromHours(_settings.DataRefresherIntervalHours);
+
+            stoppingToken.Register(() =>
+                _logger.LogInformation($"FakeDataRefresherService background task is stopping."));
+
+            while (!stoppingToken.IsCancellationRequested)
+            {
+                _logger.LogInformation($"FakeDataRefresherService is starting..");
+                var (numDeletedOccurrences, numDeletedSlots) = await _bookingSystem
+                  .Database
+                  .HardDeleteOldSoftDeletedOccurrencesAndSlots();
+                _logger.LogInformation($"FakeDataRefresherService hard deleted {numDeletedOccurrences} occurrences and {numDeletedSlots} slots that were previously old and soft-deleted.");
+
+                var (numRefreshedOccurrences, numRefreshedSlots) = await _bookingSystem
+                  .Database
+                  .SoftDeletePastOpportunitiesAndInsertNewAtEdgeOfWindow();
+                _logger.LogInformation($"FakeDataRefresherService, for {numRefreshedOccurrences} old occurrences and {numRefreshedSlots} old slots, inserted new copies into the future and soft-deleted the old ones.");
+
+                _logger.LogInformation($"FakeDataRefresherService is finished");
+
+                // Signal that a cycle has completed
+                _statusService.SignalCycleCompletion();
+
+                await Task.Delay(interval, stoppingToken);
+            }
+        }
+    }
+}

Examples/BookingSystem.AspNetCore/Controllers/InitWaitController.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Threading.Tasks;
+using BookingSystem.AspNetCore.Services;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Endpoints which are used to wait for components within
+/// BookingSystem.AspNetCore to be initialized.
+/// </summary>
+namespace BookingSystem.AspNetCore.Controllers
+{
+    [ApiController]
+    [Route("init-wait")]
+    public class InitWaitController : ControllerBase
+    {
+        private readonly DataRefresherStatusService _statusService;
+        private readonly ILogger<InitWaitController> _logger;
+        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromMinutes(5);
+
+        public InitWaitController(
+            DataRefresherStatusService statusService,
+            ILogger<InitWaitController> logger)
+        {
+            _statusService = statusService;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Wait for the data refresher to complete its first cycle.
+        ///
+        /// This makes it possible to write scripts (for CI) which don't start
+        /// until the data refresher has completed at least one cycle.
+        ///
+        /// - Returns 204 when the data refresher has completed its first cycle.
+        /// - Returns 503 if the data refresher is not configured to run.
+        /// - Returns 504 if the data refresher fails to complete a cycle within
+        ///   the default timeout.
+        /// </summary>
+        [HttpGet("data-refresher")]
+        public async Task<IActionResult> WaitForDataRefresher()
+        {
+            _logger.LogDebug("Received request to wait for data refresher completion");
+
+            // Check if the data refresher is configured to run
+            if (!_statusService.IsRefresherConfigured())
+            {
+                _logger.LogWarning("Data refresher is not configured to run");
+                return StatusCode(503, "Data refresher service is not configured to run");
+            }
+
+            // If it has already completed a cycle, return immediately
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher has already completed a cycle");
+                return NoContent();
+            }
+
+            _logger.LogDebug("Waiting for data refresher to complete a cycle...");
+            
+            // Wait for the cycle to complete, with a timeout
+            await _statusService.WaitForCycleCompletion(_defaultTimeout);
+            
+            if (_statusService.HasCompletedCycle())
+            {
+                _logger.LogDebug("Data refresher completed a cycle, returning 204");
+                return NoContent();
+            }
+            else
+            {
+                _logger.LogWarning("Timed out waiting for data refresher to complete a cycle");
+                return StatusCode(504, "Timed out waiting for data refresher to complete a cycle");
+            }
+        }
+    }
+} 

Examples/BookingSystem.AspNetCore/Feeds/FacilitiesFeeds.cs

@@ -1,5 +1,5 @@
-using Bogus;
-using BookingSystem.AspNetCore.Helpers;
+using Bogus;
+using BookingSystem.AspNetCore.Helpers;
 using OpenActive.DatasetSite.NET;
 using OpenActive.FakeDatabase.NET;
 using OpenActive.NET;
@@ -26,6 +26,7 @@ public AcmeFacilityUseRpdeGenerator(AppSettings appSettings, FakeBookingSystem f
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<FacilityUse>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             var facilityTypeId = Environment.GetEnvironmentVariable("FACILITY_TYPE_ID") ?? "https://openactive.io/facility-types#a1f82b7a-1258-4d9a-8dc5-bfc2ae961651";
@@ -204,6 +205,7 @@ public AcmeFacilityUseSlotRpdeGenerator(AppSettings appSettings, FakeBookingSyst
             this._fakeBookingSystem = fakeBookingSystem;
         }
 
+        // TODO this method should use async queries so as not to block the main thread
         protected override async Task<List<RpdeItem<Slot>>> GetRpdeItems(long? afterTimestamp, long? afterId)
         {
             using (var db = _fakeBookingSystem.Database.Mem.Database.Open())

Examples/BookingSystem.AspNetCore/README.md

@@ -33,6 +33,15 @@ ASPNETCORE_ENVIRONMENT=no-auth dotnet run --no-launch-profile --project ./Bookin
 
 The above example starts the BookingSystem.AspNetCore in `no-auth` mode.
 
+## Env Vars
+
+BookingSystem.AspNetCore uses the following environment variables (note: this list is not exhaustive):
+
+- `PERIODICALLY_REFRESH_DATA`: (optional - default `false`) If set to `true`,
+  the database will be periodically refreshed, deleting past data and replacing
+  it with future data.
+- `IS_LOREM_FITSUM_MODE`: (optional - default `false`) If set to `true`, data is created in ["Lorem Fitsum" mode](#lorem-fitsum-mode).
+
 ## BookingSystem.AspNetCore Data Generation
 
 BookingSystem.AspNetCore has three main uses that make it very important in the OpenActive ecosystem:
@@ -64,6 +73,6 @@ In the CLI this can be done by running the following command for example:
 IS_LOREM_FITSUM_MODE=true dotnet run --no-launch-profile --project ./BookingSystem.AspNetCore.csproj --configuration Release --no-build
 ```
 
-### Golden Records
+#### Golden Records
 Golden records are randomly generated records that have maximally enriched properties in the generated data. For example where a record might have one image normally, a golden record will have four.
 

Examples/BookingSystem.AspNetCore/Services/DataRefresherStatusService.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BookingSystem.AspNetCore.Services
+{
+    /// <summary>
+    /// A service which tracks the status of the data refresher, including
+    /// whether it is configured to run and whether a cycle has completed.
+    /// </summary>
+    public class DataRefresherStatusService
+    {
+        private readonly SemaphoreSlim _completionSemaphore = new SemaphoreSlim(0, 1);
+        private bool _isRefresherConfigured = false;
+        private bool _hasCompletedCycle = false;
+
+        public void SetRefresherConfigured(bool isConfigured)
+        {
+            _isRefresherConfigured = isConfigured;
+        }
+
+        public bool IsRefresherConfigured()
+        {
+            return _isRefresherConfigured;
+        }
+
+        public void SignalCycleCompletion()
+        {
+            _hasCompletedCycle = true;
+            
+            // Release the semaphore if someone is waiting on it
+            if (_completionSemaphore.CurrentCount == 0)
+            {
+                _completionSemaphore.Release();
+            }
+        }
+
+        /// <summary>
+        /// Has the data refresher completed a cycle?
+        ///
+        /// This makes it possible to write scripts (for CI) which don't start
+        /// until the data refresher has completed at least one cycle.
+        /// </summary>
+        public bool HasCompletedCycle()
+        {
+            return _hasCompletedCycle;
+        }
+
+        public async Task WaitForCycleCompletion(TimeSpan timeout)
+        {
+            if (_hasCompletedCycle)
+            {
+                return;
+            }
+
+            await _completionSemaphore.WaitAsync(timeout);
+        }
+    }
+} 

Examples/BookingSystem.AspNetCore/Settings/AppSettings.cs

@@ -1,3 +1,5 @@
+using System;
+
 namespace BookingSystem
 {
     public class AppSettings
@@ -6,6 +8,7 @@ public class AppSettings
         public string OpenIdIssuerUrl { get; set; }
         public FeatureSettings FeatureFlags { get; set; }
         public PaymentSettings Payment { get; set; }
+        public int DataRefresherIntervalHours { get; set; } = 6;
     }
 
     /**