Add operational health, encrypted backups and isolated restore drills

This commit is contained in:
wolf-demon
2026-09-21 09:10:54 +01:00
parent bc7cbc1492
commit 39751c5f1a
18 changed files with 542 additions and 7 deletions
+10
View File
@@ -15,6 +15,13 @@ IStore store = uri == null ? new PreviewStore() : new MongoStore(new Configurati
await store.Initialize();
try
{
var proofProvider=new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider();
var proof=BackupProbe.Create(proofProvider);
Check("Backup proof decrypts with the original key provider",BackupProbe.Verify(proofProvider,proof));
bool wrongKeys=false;try{BackupProbe.Verify(new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider(),proof);}catch(System.Security.Cryptography.CryptographicException){wrongKeys=true;}
Check("Backup proof rejects unrelated encryption keys",wrongKeys);
await store.Ping();
if(uri!=null){await store.RecordWorkerHeartbeat();Check("Worker heartbeat persists in MongoDB",await store.WorkerLastSeen()>DateTime.UtcNow.AddMinutes(-1));}
await MailboxTests.Run(Check, store);
await TeamTests.Run(Check, store);
await ReplyTests.Run(Check, store);
@@ -73,12 +80,15 @@ try
async Task<JsonElement> Read(HttpClient h,string path) => JsonDocument.Parse(await h.GetStringAsync(path)).RootElement.Clone();
async Task SetCsrf(HttpClient h) { var s=await Read(h,"/api/session");h.DefaultRequestHeaders.Remove("X-CSRF-TOKEN");h.DefaultRequestHeaders.Add("X-CSRF-TOKEN",s.GetProperty("csrfToken").GetString()); }
Check("Anonymous inbox access blocked", (await one.GetAsync("/api/conversations")).StatusCode == HttpStatusCode.Unauthorized);
Check("Anonymous readiness returns only status",(await Read(one,"/health/ready")).GetRawText()=="{\"status\":\"ready\"}");
Check("Unsafe requests require CSRF token", (await one.PostAsJsonAsync("/api/preview/start",new {})).StatusCode == HttpStatusCode.BadRequest);
await SetCsrf(one); await SetCsrf(two);
Check("Preview creates signed-in workspace", (await one.PostAsJsonAsync("/api/preview/start",new {})).IsSuccessStatusCode);
await two.PostAsJsonAsync("/api/preview/start",new {}); await SetCsrf(one); await SetCsrf(two);
var h1=await Read(one,"/api/hotel");var h2=await Read(two,"/api/hotel");
Check("Different preview sessions have different hotels", h1.GetProperty("id").GetString()!=h2.GetProperty("id").GetString());
var health=await Read(one,"/api/operations");
Check("Health overview is hotel scoped and labels preview worker",health.GetProperty("mailboxes").GetProperty("total").GetInt32()==1&&health.GetProperty("worker").GetProperty("state").GetString()=="Preview");
var messages=await Read(one,"/api/conversations"); var id=messages[0].GetProperty("id").GetString();
Check("Guessed message ID cannot be edited across hotels", (await two.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="stolen",version=0})).StatusCode==HttpStatusCode.NotFound);
Check("Draft save succeeds", (await one.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="Hello guest",version=0})).IsSuccessStatusCode);
+1
View File
@@ -63,6 +63,7 @@ public static class TeamTests
await Csrf(staff);check("Invited colleague can sign in",(await staff.PostAsJsonAsync("/api/auth/login",new{email,password})).IsSuccessStatusCode);await Csrf(staff);
check("Staff cannot list or invite team members",(await staff.GetAsync("/api/team")).StatusCode==HttpStatusCode.Forbidden&&(await staff.PostAsJsonAsync("/api/team/invite",new{name="No",email="no@example.invalid"})).StatusCode==HttpStatusCode.Forbidden);
check("Staff cannot change hotel settings",(await staff.PutAsJsonAsync("/api/hotel",new{name="No",signature="",timezone="UTC",version=0})).StatusCode==HttpStatusCode.Forbidden);
check("Operational health is owner only",(await staff.GetAsync("/api/operations")).StatusCode==HttpStatusCode.Forbidden);
foreach(var action in new[]{"disconnect","reconnect","retry"})check("Staff cannot "+action+" mailboxes",(await staff.PostAsJsonAsync($"/api/mailboxes/unknown/{action}",new{version=0})).StatusCode==HttpStatusCode.Forbidden);
check("Foreign hotel cannot reset staff",(await other.PostAsJsonAsync($"/api/team/{id}/reset",new{version=2})).StatusCode==HttpStatusCode.NotFound);
var list=await Read(await owner.GetAsync("/api/team"));var member=list.EnumerateArray().Single(x=>x.GetProperty("id").GetString()==id);var version=member.GetProperty("version").GetInt64();
+9
View File
@@ -6,6 +6,7 @@ against an existing hotel database. Cookie values and credentials are not logged
import json
import os
import sys
import time
from http.cookies import SimpleCookie
from urllib.request import Request, urlopen
from urllib.error import HTTPError
@@ -43,6 +44,13 @@ request("/api/auth/login", "POST", {"email": os.environ["BOOTSTRAP_EMAIL"], "pas
session = request("/api/session")
assert session["user"]["role"] == "Owner"
csrf = session["csrfToken"]
assert request("/health/ready") == {"status": "ready"}
for attempt in range(10):
health = request("/api/operations")
if health["worker"]["state"] == "Reporting":
break
time.sleep(1)
assert health["worker"]["state"] == "Reporting" and health["database"] == "Reachable"
auto_status = request("/api/auto-replies/status")
assert auto_status["liveConfigured"] is False
request("/api/auto-replies/mode", "PUT", {"mode": "Live", "version": 0, "acceptanceConfirmed": True}, expected=400)
@@ -68,3 +76,4 @@ else:
request("/api/auth/logout", "POST")
request("/api/hotel", expected=401)
print("Production smoke checks passed: built UI, secure cookies, owner login, persisted settings and staff invitation, onboarding and logout.")
+90
View File
@@ -0,0 +1,90 @@
import contextlib
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
import tarfile
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location("ops", Path(__file__).resolve().parents[1] / "deploy" / "ops.py")
ops = importlib.util.module_from_spec(spec)
spec.loader.exec_module(ops)
class ArchiveTests(unittest.TestCase):
def bundle(self, root, change=None):
files = {name: b"fixture backup data" for name in ops.FILES - {"manifest.json"}}
files["manifest.json"] = json.dumps({"format": 1, "sha256": {name: hashlib.sha256(data).hexdigest() for name, data in files.items()}}).encode()
target = root / "bundle.tar"
with tarfile.open(target, "w") as archive:
for name, data in files.items():
member = tarfile.TarInfo(name); member.size = len(data)
if change:
change(member)
archive.addfile(member, io.BytesIO(data) if member.isfile() else None)
return target
def test_checked_archive_roundtrip(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp); dest = root / "dest"; dest.mkdir()
self.assertEqual(ops.unpack(self.bundle(root), dest)["format"], 1)
def test_path_escape_rejected(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp); dest = root / "dest"; dest.mkdir()
def corrupt(member):
if member.name == "configuration.json": member.name = "../outside"
with self.assertRaisesRegex(RuntimeError, "Unexpected backup members"):
ops.unpack(self.bundle(root, corrupt), dest)
self.assertFalse((root / "outside").exists())
def test_symlink_rejected(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp); dest = root / "dest"; dest.mkdir()
def corrupt(member):
if member.name == "configuration.json": member.type = tarfile.SYMTYPE; member.linkname = "/etc/passwd"; member.size = 0
with self.assertRaisesRegex(RuntimeError, "Invalid or oversized"):
ops.unpack(self.bundle(root, corrupt), dest)
def test_checksum_mismatch_rejected(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp); dest = root / "dest"; dest.mkdir()
with patch.object(ops, "digest", return_value="changed"):
with self.assertRaisesRegex(RuntimeError, "checksum"):
ops.unpack(self.bundle(root), dest)
def test_backup_requires_explicit_maintenance(self):
with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"):
ops.backup(type("Args", (), {"confirm_maintenance": False})())
@unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
def test_dump_failure_restarts_services_and_ttl(self):
with tempfile.TemporaryDirectory() as temp:
calls = []
def compose(*args, **kwargs):
calls.append(args)
if args[0] == "ps": return b"a" * 64
if "sh" in args: raise RuntimeError("Simulated dump failure")
return b""
def mongo(script):
calls.append((script,))
if "getParameter" in script: return b"true"
if "storageSize" in script: return b'{"bytes":1,"collections":{}}'
return b""
def run(args, **kwargs):
return b'[{"Image":"sha256:fixture"}]' if "inspect" in args else b""
args = type("Args", (), {"confirm_maintenance": True, "recipient": "A" * 40, "output": str(Path(temp) / "backup.gpg")})()
with patch.object(ops, "configuration", return_value={}), patch.object(ops, "run", side_effect=run), patch.object(ops, "compose", side_effect=compose), patch.object(ops, "mongo", side_effect=mongo), patch.object(ops, "maintenance_lock", return_value=contextlib.nullcontext()):
with self.assertRaisesRegex(RuntimeError, "Simulated dump failure"):
ops.backup(args)
self.assertIn(("start", "api", "worker"), calls)
self.assertTrue(any("ttlMonitorEnabled:true" in call[0] for call in calls))
self.assertFalse(Path(args.output).exists())
if __name__ == "__main__":
unittest.main()