Add operational health, encrypted backups and isolated restore drills

This commit is contained in:
wolf-demon
2026-09-21 09:10:54 +01:00
parent bc7cbc1492
commit 39751c5f1a
18 changed files with 542 additions and 7 deletions
+29
View File
@@ -0,0 +1,29 @@
using Microsoft.AspNetCore.DataProtection;
namespace GuestOps.Web;
public sealed class WorkerHeartbeat
{
[MongoDB.Bson.Serialization.Attributes.BsonId] public string Id {get;set;}="worker";
public DateTime At {get;set;}=DateTime.UtcNow;
}
public static class BackupProbe
{
const string Value="GuestOps backup key verification v1";
public static string Create(IDataProtectionProvider protection)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Protect(Value);
public static bool Verify(IDataProtectionProvider protection,string value)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Unprotect(value)==Value;
}
public static class Operations
{
public static void Map(WebApplication app,RouteGroupBuilder api,bool preview)
{
app.MapGet("/health/ready",async(IStore store,HttpContext c)=>
{
c.Response.Headers.CacheControl="no-store";
try{await store.Ping();return Results.Ok(new{status="ready"});}catch{return Results.Json(new{status="unavailable"},statusCode:503);}
});
api.MapGet("/operations",async(HttpContext c,IStore store)=>
{
var id=Session.Hotel(c);var boxes=await store.List<Mailbox>(id);var messages=await store.List<Conversation>(id);var seen=await store.WorkerLastSeen();
return Results.Ok(new{checkedAt=DateTime.UtcNow,preview,database="Reachable",worker=new{lastSeenAt=seen,state=preview?"Preview":seen==null?"NotSeen":seen<DateTime.UtcNow.AddMinutes(-3)?"Stale":"Reporting"},mailboxes=new{total=boxes.Count,connected=boxes.Count(x=>x.Status=="Connected"),attention=boxes.Count(x=>x.Status!="Connected"||x.SyncError.Length>0)},replies=new{sampleSize=messages.Count,sampleLimit=500,pending=messages.Count(x=>x.Delivery?.State is "Pending" or "Sending"),uncertain=messages.Count(x=>x.Delivery?.State=="NeedsReview"),rejected=messages.Count(x=>x.Delivery?.State=="Rejected")}});
}).RequireAuthorization("Owner");
}
}
+11 -1
View File
@@ -13,7 +13,8 @@ using System.Net;
var bootstrap = args.Contains("--bootstrap");
var recoverOwner = args.Contains("--recover-owner");
var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner").ToArray());
var backupProbe=args.Contains("--backup-probe");var verifyBackupProbe=args.Contains("--verify-backup-probe");
var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner" && arg != "--backup-probe" && arg != "--verify-backup-probe").ToArray());
if (builder.Configuration["Pms:ConfigFile"] is { Length: > 0 } pmsConfigFile) builder.Configuration.AddJsonFile(pmsConfigFile,optional:false,reloadOnChange:false);
if (builder.Configuration["Payments:ConfigFile"] is { Length: > 0 } paymentConfigFile) builder.Configuration.AddJsonFile(paymentConfigFile,optional:false,reloadOnChange:false);
builder.Logging.ClearProviders(); builder.Logging.AddConsole();
@@ -71,6 +72,14 @@ builder.Services.AddRateLimiter(o =>
o.AddPolicy("login", context => RateLimitPartition.GetFixedWindowLimiter(context.Connection.RemoteIpAddress?.ToString() ?? "unknown", _ => new() { PermitLimit = 10, Window = TimeSpan.FromMinutes(1), QueueLimit = 0 }));
});
var app = builder.Build();
if(backupProbe||verifyBackupProbe)
{
var provider=app.Services.GetRequiredService<IDataProtectionProvider>();
if(backupProbe)Console.WriteLine(BackupProbe.Create(provider));
else if(!BackupProbe.Verify(provider,Environment.GetEnvironmentVariable("BACKUP_PROBE")??""))throw new InvalidOperationException("Backup keys failed verification.");
else Console.WriteLine("Backup keys verified.");
return;
}
app.UseForwardedHeaders();
var store = app.Services.GetRequiredService<IStore>();
await store.Initialize();
@@ -148,6 +157,7 @@ PaymentEndpoints.Map(api,preview);
AutoReplyEndpoints.Map(api,preview);
TeamEndpoints.Map(app,api,preview);
MailboxManagement.Map(api,preview);
Operations.Map(app,api,preview);
api.MapGet("/hotel", async (HttpContext c, CancellationToken _) => Results.Ok(await store.Get<Hotel>(Session.Hotel(c), Session.Hotel(c))));
api.MapPut("/hotel", async (SettingsInput input, HttpContext c) =>
{
+9
View File
@@ -7,6 +7,9 @@ namespace GuestOps.Web;
public interface IStore
{
Task Initialize();
Task Ping();
Task<DateTime?> WorkerLastSeen();
Task RecordWorkerHeartbeat();
Task<List<T>> List<T>(string hotel) where T : TenantDocument;
Task<T?> Get<T>(string hotel, string id) where T : TenantDocument;
Task Insert<T>(T document) where T : TenantDocument;
@@ -32,6 +35,9 @@ public interface IStore
}
public sealed class MongoStore : IStore
{
public async Task Ping()=>await db.RunCommandAsync<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument("ping",1));
public async Task<DateTime?> WorkerLastSeen()=>(await db.GetCollection<WorkerHeartbeat>("workerheartbeat").Find(x=>x.Id=="worker").FirstOrDefaultAsync())?.At;
public async Task RecordWorkerHeartbeat()=>await db.GetCollection<WorkerHeartbeat>("workerheartbeat").ReplaceOneAsync(x=>x.Id=="worker",new WorkerHeartbeat(),new ReplaceOptions{IsUpsert=true});
public Task<List<Conversation>> Deliveries() => Collection<Conversation>().Find(x => x.Delivery != null && (x.Delivery.State == "Pending" || x.Delivery.State == "Sending")).SortBy(x => x.Delivery!.UpdatedAt).Limit(100).ToListAsync();
private readonly IMongoDatabase db;
public MongoStore(IConfiguration config)
@@ -141,6 +147,9 @@ public sealed class MongoStore : IStore
// Explicit Development-only preview store. Production never falls back to this.
public sealed class PreviewStore : IStore
{
public Task Ping()=>Task.CompletedTask;
public Task<DateTime?> WorkerLastSeen()=>Task.FromResult<DateTime?>(null);
public Task RecordWorkerHeartbeat()=>Task.CompletedTask;
public async Task<List<Conversation>> AutoReplyCandidates(string hotel,string mailbox,DateTime since)=>(await List<Conversation>(hotel)).Where(x=>x.MailboxId==mailbox&&x.AutoReplyCheckedAt==null&&x.ReceivedAt>=since).OrderBy(x=>x.ReceivedAt).Take(100).ToList();
public Task<bool> TryAutoReplyClaim(AutoReplyClaim claim)
{
+13
View File
@@ -17,8 +17,21 @@ builder.Services.AddTransient<AutoReplyWork>();
builder.Services.AddHostedService<AutoReplyWorker>();
builder.Services.AddHostedService<DeliveryWorker>();
builder.Services.AddHostedService<MailboxWorker>();
builder.Services.AddHostedService<HeartbeatWorker>();
await builder.Build().RunAsync();
sealed class HeartbeatWorker(IStore store,ILogger<HeartbeatWorker> log):BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while(!stoppingToken.IsCancellationRequested)
{
try{await store.RecordWorkerHeartbeat();}catch(Exception ex){log.LogWarning("Worker heartbeat unavailable ({Type})",ex.GetType().Name);}
await Task.Delay(TimeSpan.FromSeconds(30),stoppingToken);
}
}
}
sealed class MailboxWorker(IStore store, IServiceScopeFactory factory, ILogger<MailboxWorker> log) : BackgroundService
{
readonly string owner = Guid.NewGuid().ToString("N");