Implement configurable message body storage

src/Directory.Packages.props

@@ -6,9 +6,11 @@
   <ItemGroup Label="Versions for direct package references">
     <PackageVersion Include="Autofac" Version="9.3.0" />
     <PackageVersion Include="AWSSDK.CloudWatch" Version="4.0.10.5" />
+    <PackageVersion Include="AWSSDK.S3" Version="4.0.101.4" />
     <PackageVersion Include="Azure.Identity" Version="1.21.0" />
     <PackageVersion Include="Azure.ResourceManager.Monitor" Version="1.3.1" />
     <PackageVersion Include="Azure.ResourceManager.ServiceBus" Version="1.1.0" />
+    <PackageVersion Include="Azure.Storage.Blobs" Version="12.27.0" />
     <PackageVersion Include="ByteSize" Version="2.1.2" />
     <PackageVersion Include="Caliburn.Micro" Version="5.0.258" />
     <PackageVersion Include="DnsClient" Version="1.8.0" />
@@ -76,6 +78,8 @@
     <PackageVersion Include="PropertyChanging.Fody" Version="1.31.0" />
     <PackageVersion Include="PublicApiGenerator" Version="11.5.4" />
     <PackageVersion Include="RavenDB.Embedded" Version="6.2.17" />
+    <PackageVersion Include="Testcontainers.Azurite" Version="4.13.0" />
+    <PackageVersion Include="Testcontainers.LocalStack" Version="4.13.0" />
     <PackageVersion Include="Testcontainers.MsSql" Version="4.13.0" />
     <PackageVersion Include="Testcontainers.PostgreSql" Version="4.13.0" />
     <PackageVersion Include="ReactiveUI.WPF" Version="22.3.1" />

src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs

@@ -12,7 +12,7 @@ public void AddPersistence(IServiceCollection services)
     {
         RegisterSettings(services);
         ConfigureDbContext(services);
-        RegisterDataStores(services);
+        RegisterDataStores(services, settings);
 
         services.AddSingleton<IIngestionSqlDialect, PostgreSqlIngestionSqlDialect>();
     }
@@ -23,6 +23,7 @@ public void AddInstaller(IServiceCollection services)
         ConfigureDbContext(services);
 
         services.AddScoped<IDatabaseMigrator, PostgreSqlDatabaseMigrator>();
+        RegisterBodyStorageInstaller(services, settings);
     }
 
     void RegisterSettings(IServiceCollection services)

src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistenceConfiguration.cs

@@ -15,6 +15,6 @@ public override IPersistence Create(PersistenceSettings settings)
         return new PostgreSqlPersistence((PostgreSqlPersisterSettings)settings);
     }
 
-    protected override EFPersisterSettings CreateSettings(string connectionString) =>
-        new PostgreSqlPersisterSettings { ConnectionString = connectionString };
+    protected override EFPersisterSettings CreateSettings(string connectionString, BodyStorageSettings bodyStorage) =>
+        new PostgreSqlPersisterSettings { ConnectionString = connectionString, BodyStorage = bodyStorage };
 }

src/ServiceControl.Persistence.EFCore.PostgreSql/persistence.manifest

@@ -13,10 +13,58 @@
       "Name": "ServiceControl/Database/CommandTimeout",
       "Mandatory": false
     },
+    {
+      "Name": "ServiceControl/MessageBody/StorageType",
+      "Mandatory": true
+    },
     {
       "Name": "ServiceControl/MessageBody/StoragePath",
       "Mandatory": false
     },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ConnectionString",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ServiceUri",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ManagedIdentityClientId",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/AuthorityHost",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ContainerName",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/BucketName",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/KeyPrefix",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/Region",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/ServiceUrl",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/AccessKeyId",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/SecretAccessKey",
+      "Mandatory": false
+    },
     {
       "Name": "ServiceControl/MessageBody/MinCompressionSize",
       "Mandatory": false

src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs

@@ -12,7 +12,7 @@ public void AddPersistence(IServiceCollection services)
     {
         RegisterSettings(services);
         ConfigureDbContext(services);
-        RegisterDataStores(services);
+        RegisterDataStores(services, settings);
 
         services.AddSingleton<IIngestionSqlDialect, SqlServerIngestionSqlDialect>();
     }
@@ -23,6 +23,7 @@ public void AddInstaller(IServiceCollection services)
         ConfigureDbContext(services);
 
         services.AddScoped<IDatabaseMigrator, SqlServerDatabaseMigrator>();
+        RegisterBodyStorageInstaller(services, settings);
     }
 
     void RegisterSettings(IServiceCollection services)

src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistenceConfiguration.cs

@@ -15,6 +15,6 @@ public override IPersistence Create(PersistenceSettings settings)
         return new SqlServerPersistence((SqlServerPersisterSettings)settings);
     }
 
-    protected override EFPersisterSettings CreateSettings(string connectionString) =>
-        new SqlServerPersisterSettings { ConnectionString = connectionString };
+    protected override EFPersisterSettings CreateSettings(string connectionString, BodyStorageSettings bodyStorage) =>
+        new SqlServerPersisterSettings { ConnectionString = connectionString, BodyStorage = bodyStorage };
 }

src/ServiceControl.Persistence.EFCore.SqlServer/persistence.manifest

@@ -13,10 +13,58 @@
       "Name": "ServiceControl/Database/CommandTimeout",
       "Mandatory": false
     },
+    {
+      "Name": "ServiceControl/MessageBody/StorageType",
+      "Mandatory": true
+    },
     {
       "Name": "ServiceControl/MessageBody/StoragePath",
       "Mandatory": false
     },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ConnectionString",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ServiceUri",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ManagedIdentityClientId",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/AuthorityHost",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/Azure/ContainerName",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/BucketName",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/KeyPrefix",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/Region",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/ServiceUrl",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/AccessKeyId",
+      "Mandatory": false
+    },
+    {
+      "Name": "ServiceControl/MessageBody/S3/SecretAccessKey",
+      "Mandatory": false
+    },
     {
       "Name": "ServiceControl/MessageBody/MinCompressionSize",
       "Mandatory": false

src/ServiceControl.Persistence.EFCore/Abstractions/AzureBlobBodyStorageSettings.cs

@@ -0,0 +1,28 @@
+namespace ServiceControl.Persistence.EFCore.Abstractions;
+
+public sealed class AzureBlobBodyStorageSettings : BodyStorageSettings
+{
+    public const string DefaultContainerName = "error-bodies";
+
+    public required AzureBlobAuthentication Authentication { get; set; }
+    public string ContainerName { get; set; } = DefaultContainerName;
+}
+
+// Shared-key and managed-identity auth are mutually exclusive, and the managed identity options are
+// meaningless alongside a connection string.
+public abstract class AzureBlobAuthentication;
+
+public sealed class AzureBlobSharedKeyAuthentication : AzureBlobAuthentication
+{
+    public required string ConnectionString { get; set; }
+}
+
+public sealed class AzureBlobManagedIdentityAuthentication : AzureBlobAuthentication
+{
+    public required Uri ServiceUri { get; set; }
+    public string? ClientId { get; set; }
+
+    // Steers the login endpoint for sovereign clouds; when unset the SDK honours the
+    // AZURE_AUTHORITY_HOST environment variable.
+    public Uri? AuthorityHost { get; set; }
+}

src/ServiceControl.Persistence.EFCore/Abstractions/BasePersistence.cs

@@ -1,10 +1,12 @@
 namespace ServiceControl.Persistence.EFCore.Abstractions;
 
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
 using NServiceBus.Unicast.Subscriptions.MessageDrivenSubscriptions;
 using Particular.LicensingComponent.Persistence;
 using ServiceControl.Operations.BodyStorage;
 using ServiceControl.Persistence.EFCore.Implementation;
+using ServiceControl.Persistence.EFCore.Implementation.BodyStorage;
 using ServiceControl.Persistence.EFCore.Implementation.UnitOfWork;
 using ServiceControl.Persistence.EFCore.Infrastructure;
 using ServiceControl.Persistence.MessageRedirects;
@@ -13,7 +15,7 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 
 public abstract class BasePersistence
 {
-    protected static void RegisterDataStores(IServiceCollection services)
+    protected static void RegisterDataStores(IServiceCollection services, EFPersisterSettings settings)
     {
         services.AddSingleton(TimeProvider.System);
         services.AddSingleton<MinimumRequiredStorageState>();
@@ -51,6 +53,50 @@ protected static void RegisterDataStores(IServiceCollection services)
 
         services.AddSingleton<ILicensingDataStore, LicensingDataStore>();
 
-        services.AddSingleton<IBodyStoragePersistence, FakeBodyStoragePersistence>();
+        RegisterBodyStorage(services, settings);
+    }
+
+    // Settings are registered under their concrete type so each store resolves only what it can act on.
+    static void RegisterBodyStorage(IServiceCollection services, EFPersisterSettings settings)
+    {
+        switch (settings.BodyStorage)
+        {
+            case FileSystemBodyStorageSettings fileSystem:
+                services.TryAddSingleton(fileSystem);
+                services.AddSingleton<IBodyStoragePersistence, FileSystemBodyStoragePersistence>();
+                break;
+            case AzureBlobBodyStorageSettings azureBlob:
+                services.TryAddSingleton(azureBlob);
+                services.AddSingleton<IBodyStoragePersistence, AzureBlobBodyStoragePersistence>();
+                break;
+            case S3BodyStorageSettings s3:
+                services.TryAddSingleton(s3);
+                services.AddSingleton<IBodyStoragePersistence, S3BodyStoragePersistence>();
+                break;
+            default:
+                throw new ArgumentOutOfRangeException(nameof(settings), settings.BodyStorage, "Unknown body storage type.");
+        }
+    }
+
+    // Only stores needing setup-time provisioning register an installer; SetupCommand skips when none is.
+    protected static void RegisterBodyStorageInstaller(IServiceCollection services, EFPersisterSettings settings)
+    {
+        switch (settings.BodyStorage)
+        {
+            case FileSystemBodyStorageSettings fileSystem:
+                services.TryAddSingleton(fileSystem);
+                services.AddScoped<IBodyStorageInstaller, FileSystemBodyStorageInstaller>();
+                break;
+            case AzureBlobBodyStorageSettings azureBlob:
+                services.TryAddSingleton(azureBlob);
+                services.AddScoped<IBodyStorageInstaller, AzureBlobBodyStorageInstaller>();
+                break;
+            case S3BodyStorageSettings s3:
+                services.TryAddSingleton(s3);
+                services.AddScoped<IBodyStorageInstaller, S3BodyStorageInstaller>();
+                break;
+            default:
+                throw new ArgumentOutOfRangeException(nameof(settings), settings.BodyStorage, "Unknown body storage type.");
+        }
     }
 }

src/ServiceControl.Persistence.EFCore/Abstractions/BodyStorageSettings.cs

@@ -0,0 +1,17 @@
+namespace ServiceControl.Persistence.EFCore.Abstractions;
+
+/// <summary>
+/// Settings for the selected body storage type.
+/// </summary>
+/// <remarks>
+/// One subclass per storage type, so a store only ever receives the settings it can act on and the
+/// configuration layer's validation is carried by the type rather than re-asserted at the point of use.
+/// </remarks>
+public abstract class BodyStorageSettings
+{
+    public const int DefaultMinCompressionSize = 4096;
+    public const int DefaultMaxBodySizeToStore = 102400; // 100 kb
+
+    public int MinCompressionSize { get; set; } = DefaultMinCompressionSize;
+    public int MaxBodySizeToStore { get; set; } = DefaultMaxBodySizeToStore;
+}

src/ServiceControl.Persistence.EFCore/Abstractions/BodyStorageType.cs

@@ -0,0 +1,8 @@
+namespace ServiceControl.Persistence.EFCore.Abstractions;
+
+public enum BodyStorageType
+{
+    FileSystem,
+    AzureBlob,
+    S3
+}