Add operational health, encrypted backups and isolated restore drills
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
using Microsoft.AspNetCore.DataProtection;
|
||||
namespace GuestOps.Web;
|
||||
public sealed class WorkerHeartbeat
|
||||
{
|
||||
[MongoDB.Bson.Serialization.Attributes.BsonId] public string Id {get;set;}="worker";
|
||||
public DateTime At {get;set;}=DateTime.UtcNow;
|
||||
}
|
||||
public static class BackupProbe
|
||||
{
|
||||
const string Value="GuestOps backup key verification v1";
|
||||
public static string Create(IDataProtectionProvider protection)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Protect(Value);
|
||||
public static bool Verify(IDataProtectionProvider protection,string value)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Unprotect(value)==Value;
|
||||
}
|
||||
public static class Operations
|
||||
{
|
||||
public static void Map(WebApplication app,RouteGroupBuilder api,bool preview)
|
||||
{
|
||||
app.MapGet("/health/ready",async(IStore store,HttpContext c)=>
|
||||
{
|
||||
c.Response.Headers.CacheControl="no-store";
|
||||
try{await store.Ping();return Results.Ok(new{status="ready"});}catch{return Results.Json(new{status="unavailable"},statusCode:503);}
|
||||
});
|
||||
api.MapGet("/operations",async(HttpContext c,IStore store)=>
|
||||
{
|
||||
var id=Session.Hotel(c);var boxes=await store.List<Mailbox>(id);var messages=await store.List<Conversation>(id);var seen=await store.WorkerLastSeen();
|
||||
return Results.Ok(new{checkedAt=DateTime.UtcNow,preview,database="Reachable",worker=new{lastSeenAt=seen,state=preview?"Preview":seen==null?"NotSeen":seen<DateTime.UtcNow.AddMinutes(-3)?"Stale":"Reporting"},mailboxes=new{total=boxes.Count,connected=boxes.Count(x=>x.Status=="Connected"),attention=boxes.Count(x=>x.Status!="Connected"||x.SyncError.Length>0)},replies=new{sampleSize=messages.Count,sampleLimit=500,pending=messages.Count(x=>x.Delivery?.State is "Pending" or "Sending"),uncertain=messages.Count(x=>x.Delivery?.State=="NeedsReview"),rejected=messages.Count(x=>x.Delivery?.State=="Rejected")}});
|
||||
}).RequireAuthorization("Owner");
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,8 @@ using System.Net;
|
||||
|
||||
var bootstrap = args.Contains("--bootstrap");
|
||||
var recoverOwner = args.Contains("--recover-owner");
|
||||
var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner").ToArray());
|
||||
var backupProbe=args.Contains("--backup-probe");var verifyBackupProbe=args.Contains("--verify-backup-probe");
|
||||
var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner" && arg != "--backup-probe" && arg != "--verify-backup-probe").ToArray());
|
||||
if (builder.Configuration["Pms:ConfigFile"] is { Length: > 0 } pmsConfigFile) builder.Configuration.AddJsonFile(pmsConfigFile,optional:false,reloadOnChange:false);
|
||||
if (builder.Configuration["Payments:ConfigFile"] is { Length: > 0 } paymentConfigFile) builder.Configuration.AddJsonFile(paymentConfigFile,optional:false,reloadOnChange:false);
|
||||
builder.Logging.ClearProviders(); builder.Logging.AddConsole();
|
||||
@@ -71,6 +72,14 @@ builder.Services.AddRateLimiter(o =>
|
||||
o.AddPolicy("login", context => RateLimitPartition.GetFixedWindowLimiter(context.Connection.RemoteIpAddress?.ToString() ?? "unknown", _ => new() { PermitLimit = 10, Window = TimeSpan.FromMinutes(1), QueueLimit = 0 }));
|
||||
});
|
||||
var app = builder.Build();
|
||||
if(backupProbe||verifyBackupProbe)
|
||||
{
|
||||
var provider=app.Services.GetRequiredService<IDataProtectionProvider>();
|
||||
if(backupProbe)Console.WriteLine(BackupProbe.Create(provider));
|
||||
else if(!BackupProbe.Verify(provider,Environment.GetEnvironmentVariable("BACKUP_PROBE")??""))throw new InvalidOperationException("Backup keys failed verification.");
|
||||
else Console.WriteLine("Backup keys verified.");
|
||||
return;
|
||||
}
|
||||
app.UseForwardedHeaders();
|
||||
var store = app.Services.GetRequiredService<IStore>();
|
||||
await store.Initialize();
|
||||
@@ -148,6 +157,7 @@ PaymentEndpoints.Map(api,preview);
|
||||
AutoReplyEndpoints.Map(api,preview);
|
||||
TeamEndpoints.Map(app,api,preview);
|
||||
MailboxManagement.Map(api,preview);
|
||||
Operations.Map(app,api,preview);
|
||||
api.MapGet("/hotel", async (HttpContext c, CancellationToken _) => Results.Ok(await store.Get<Hotel>(Session.Hotel(c), Session.Hotel(c))));
|
||||
api.MapPut("/hotel", async (SettingsInput input, HttpContext c) =>
|
||||
{
|
||||
|
||||
@@ -7,6 +7,9 @@ namespace GuestOps.Web;
|
||||
public interface IStore
|
||||
{
|
||||
Task Initialize();
|
||||
Task Ping();
|
||||
Task<DateTime?> WorkerLastSeen();
|
||||
Task RecordWorkerHeartbeat();
|
||||
Task<List<T>> List<T>(string hotel) where T : TenantDocument;
|
||||
Task<T?> Get<T>(string hotel, string id) where T : TenantDocument;
|
||||
Task Insert<T>(T document) where T : TenantDocument;
|
||||
@@ -32,6 +35,9 @@ public interface IStore
|
||||
}
|
||||
public sealed class MongoStore : IStore
|
||||
{
|
||||
public async Task Ping()=>await db.RunCommandAsync<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument("ping",1));
|
||||
public async Task<DateTime?> WorkerLastSeen()=>(await db.GetCollection<WorkerHeartbeat>("workerheartbeat").Find(x=>x.Id=="worker").FirstOrDefaultAsync())?.At;
|
||||
public async Task RecordWorkerHeartbeat()=>await db.GetCollection<WorkerHeartbeat>("workerheartbeat").ReplaceOneAsync(x=>x.Id=="worker",new WorkerHeartbeat(),new ReplaceOptions{IsUpsert=true});
|
||||
public Task<List<Conversation>> Deliveries() => Collection<Conversation>().Find(x => x.Delivery != null && (x.Delivery.State == "Pending" || x.Delivery.State == "Sending")).SortBy(x => x.Delivery!.UpdatedAt).Limit(100).ToListAsync();
|
||||
private readonly IMongoDatabase db;
|
||||
public MongoStore(IConfiguration config)
|
||||
@@ -141,6 +147,9 @@ public sealed class MongoStore : IStore
|
||||
// Explicit Development-only preview store. Production never falls back to this.
|
||||
public sealed class PreviewStore : IStore
|
||||
{
|
||||
public Task Ping()=>Task.CompletedTask;
|
||||
public Task<DateTime?> WorkerLastSeen()=>Task.FromResult<DateTime?>(null);
|
||||
public Task RecordWorkerHeartbeat()=>Task.CompletedTask;
|
||||
public async Task<List<Conversation>> AutoReplyCandidates(string hotel,string mailbox,DateTime since)=>(await List<Conversation>(hotel)).Where(x=>x.MailboxId==mailbox&&x.AutoReplyCheckedAt==null&&x.ReceivedAt>=since).OrderBy(x=>x.ReceivedAt).Take(100).ToList();
|
||||
public Task<bool> TryAutoReplyClaim(AutoReplyClaim claim)
|
||||
{
|
||||
|
||||
@@ -17,8 +17,21 @@ builder.Services.AddTransient<AutoReplyWork>();
|
||||
builder.Services.AddHostedService<AutoReplyWorker>();
|
||||
builder.Services.AddHostedService<DeliveryWorker>();
|
||||
builder.Services.AddHostedService<MailboxWorker>();
|
||||
builder.Services.AddHostedService<HeartbeatWorker>();
|
||||
await builder.Build().RunAsync();
|
||||
|
||||
sealed class HeartbeatWorker(IStore store,ILogger<HeartbeatWorker> log):BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while(!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
try{await store.RecordWorkerHeartbeat();}catch(Exception ex){log.LogWarning("Worker heartbeat unavailable ({Type})",ex.GetType().Name);}
|
||||
await Task.Delay(TimeSpan.FromSeconds(30),stoppingToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sealed class MailboxWorker(IStore store, IServiceScopeFactory factory, ILogger<MailboxWorker> log) : BackgroundService
|
||||
{
|
||||
readonly string owner = Guid.NewGuid().ToString("N");
|
||||
|
||||
Reference in New Issue
Block a user