Add operational health, encrypted backups and isolated restore drills
This commit is contained in:
@@ -15,6 +15,13 @@ IStore store = uri == null ? new PreviewStore() : new MongoStore(new Configurati
|
||||
await store.Initialize();
|
||||
try
|
||||
{
|
||||
var proofProvider=new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider();
|
||||
var proof=BackupProbe.Create(proofProvider);
|
||||
Check("Backup proof decrypts with the original key provider",BackupProbe.Verify(proofProvider,proof));
|
||||
bool wrongKeys=false;try{BackupProbe.Verify(new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider(),proof);}catch(System.Security.Cryptography.CryptographicException){wrongKeys=true;}
|
||||
Check("Backup proof rejects unrelated encryption keys",wrongKeys);
|
||||
await store.Ping();
|
||||
if(uri!=null){await store.RecordWorkerHeartbeat();Check("Worker heartbeat persists in MongoDB",await store.WorkerLastSeen()>DateTime.UtcNow.AddMinutes(-1));}
|
||||
await MailboxTests.Run(Check, store);
|
||||
await TeamTests.Run(Check, store);
|
||||
await ReplyTests.Run(Check, store);
|
||||
@@ -73,12 +80,15 @@ try
|
||||
async Task<JsonElement> Read(HttpClient h,string path) => JsonDocument.Parse(await h.GetStringAsync(path)).RootElement.Clone();
|
||||
async Task SetCsrf(HttpClient h) { var s=await Read(h,"/api/session");h.DefaultRequestHeaders.Remove("X-CSRF-TOKEN");h.DefaultRequestHeaders.Add("X-CSRF-TOKEN",s.GetProperty("csrfToken").GetString()); }
|
||||
Check("Anonymous inbox access blocked", (await one.GetAsync("/api/conversations")).StatusCode == HttpStatusCode.Unauthorized);
|
||||
Check("Anonymous readiness returns only status",(await Read(one,"/health/ready")).GetRawText()=="{\"status\":\"ready\"}");
|
||||
Check("Unsafe requests require CSRF token", (await one.PostAsJsonAsync("/api/preview/start",new {})).StatusCode == HttpStatusCode.BadRequest);
|
||||
await SetCsrf(one); await SetCsrf(two);
|
||||
Check("Preview creates signed-in workspace", (await one.PostAsJsonAsync("/api/preview/start",new {})).IsSuccessStatusCode);
|
||||
await two.PostAsJsonAsync("/api/preview/start",new {}); await SetCsrf(one); await SetCsrf(two);
|
||||
var h1=await Read(one,"/api/hotel");var h2=await Read(two,"/api/hotel");
|
||||
Check("Different preview sessions have different hotels", h1.GetProperty("id").GetString()!=h2.GetProperty("id").GetString());
|
||||
var health=await Read(one,"/api/operations");
|
||||
Check("Health overview is hotel scoped and labels preview worker",health.GetProperty("mailboxes").GetProperty("total").GetInt32()==1&&health.GetProperty("worker").GetProperty("state").GetString()=="Preview");
|
||||
var messages=await Read(one,"/api/conversations"); var id=messages[0].GetProperty("id").GetString();
|
||||
Check("Guessed message ID cannot be edited across hotels", (await two.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="stolen",version=0})).StatusCode==HttpStatusCode.NotFound);
|
||||
Check("Draft save succeeds", (await one.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="Hello guest",version=0})).IsSuccessStatusCode);
|
||||
|
||||
@@ -63,6 +63,7 @@ public static class TeamTests
|
||||
await Csrf(staff);check("Invited colleague can sign in",(await staff.PostAsJsonAsync("/api/auth/login",new{email,password})).IsSuccessStatusCode);await Csrf(staff);
|
||||
check("Staff cannot list or invite team members",(await staff.GetAsync("/api/team")).StatusCode==HttpStatusCode.Forbidden&&(await staff.PostAsJsonAsync("/api/team/invite",new{name="No",email="no@example.invalid"})).StatusCode==HttpStatusCode.Forbidden);
|
||||
check("Staff cannot change hotel settings",(await staff.PutAsJsonAsync("/api/hotel",new{name="No",signature="",timezone="UTC",version=0})).StatusCode==HttpStatusCode.Forbidden);
|
||||
check("Operational health is owner only",(await staff.GetAsync("/api/operations")).StatusCode==HttpStatusCode.Forbidden);
|
||||
foreach(var action in new[]{"disconnect","reconnect","retry"})check("Staff cannot "+action+" mailboxes",(await staff.PostAsJsonAsync($"/api/mailboxes/unknown/{action}",new{version=0})).StatusCode==HttpStatusCode.Forbidden);
|
||||
check("Foreign hotel cannot reset staff",(await other.PostAsJsonAsync($"/api/team/{id}/reset",new{version=2})).StatusCode==HttpStatusCode.NotFound);
|
||||
var list=await Read(await owner.GetAsync("/api/team"));var member=list.EnumerateArray().Single(x=>x.GetProperty("id").GetString()==id);var version=member.GetProperty("version").GetInt64();
|
||||
|
||||
@@ -6,6 +6,7 @@ against an existing hotel database. Cookie values and credentials are not logged
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from http.cookies import SimpleCookie
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import HTTPError
|
||||
@@ -43,6 +44,13 @@ request("/api/auth/login", "POST", {"email": os.environ["BOOTSTRAP_EMAIL"], "pas
|
||||
session = request("/api/session")
|
||||
assert session["user"]["role"] == "Owner"
|
||||
csrf = session["csrfToken"]
|
||||
assert request("/health/ready") == {"status": "ready"}
|
||||
for attempt in range(10):
|
||||
health = request("/api/operations")
|
||||
if health["worker"]["state"] == "Reporting":
|
||||
break
|
||||
time.sleep(1)
|
||||
assert health["worker"]["state"] == "Reporting" and health["database"] == "Reachable"
|
||||
auto_status = request("/api/auto-replies/status")
|
||||
assert auto_status["liveConfigured"] is False
|
||||
request("/api/auto-replies/mode", "PUT", {"mode": "Live", "version": 0, "acceptanceConfirmed": True}, expected=400)
|
||||
@@ -68,3 +76,4 @@ else:
|
||||
request("/api/auth/logout", "POST")
|
||||
request("/api/hotel", expected=401)
|
||||
print("Production smoke checks passed: built UI, secure cookies, owner login, persisted settings and staff invitation, onboarding and logout.")
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import contextlib
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location("ops", Path(__file__).resolve().parents[1] / "deploy" / "ops.py")
|
||||
ops = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(ops)
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def bundle(self, root, change=None):
|
||||
files = {name: b"fixture backup data" for name in ops.FILES - {"manifest.json"}}
|
||||
files["manifest.json"] = json.dumps({"format": 1, "sha256": {name: hashlib.sha256(data).hexdigest() for name, data in files.items()}}).encode()
|
||||
target = root / "bundle.tar"
|
||||
with tarfile.open(target, "w") as archive:
|
||||
for name, data in files.items():
|
||||
member = tarfile.TarInfo(name); member.size = len(data)
|
||||
if change:
|
||||
change(member)
|
||||
archive.addfile(member, io.BytesIO(data) if member.isfile() else None)
|
||||
return target
|
||||
|
||||
def test_checked_archive_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp); dest = root / "dest"; dest.mkdir()
|
||||
self.assertEqual(ops.unpack(self.bundle(root), dest)["format"], 1)
|
||||
|
||||
def test_path_escape_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp); dest = root / "dest"; dest.mkdir()
|
||||
def corrupt(member):
|
||||
if member.name == "configuration.json": member.name = "../outside"
|
||||
with self.assertRaisesRegex(RuntimeError, "Unexpected backup members"):
|
||||
ops.unpack(self.bundle(root, corrupt), dest)
|
||||
self.assertFalse((root / "outside").exists())
|
||||
|
||||
def test_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp); dest = root / "dest"; dest.mkdir()
|
||||
def corrupt(member):
|
||||
if member.name == "configuration.json": member.type = tarfile.SYMTYPE; member.linkname = "/etc/passwd"; member.size = 0
|
||||
with self.assertRaisesRegex(RuntimeError, "Invalid or oversized"):
|
||||
ops.unpack(self.bundle(root, corrupt), dest)
|
||||
|
||||
def test_checksum_mismatch_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp); dest = root / "dest"; dest.mkdir()
|
||||
with patch.object(ops, "digest", return_value="changed"):
|
||||
with self.assertRaisesRegex(RuntimeError, "checksum"):
|
||||
ops.unpack(self.bundle(root), dest)
|
||||
|
||||
def test_backup_requires_explicit_maintenance(self):
|
||||
with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"):
|
||||
ops.backup(type("Args", (), {"confirm_maintenance": False})())
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
|
||||
def test_dump_failure_restarts_services_and_ttl(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
calls = []
|
||||
def compose(*args, **kwargs):
|
||||
calls.append(args)
|
||||
if args[0] == "ps": return b"a" * 64
|
||||
if "sh" in args: raise RuntimeError("Simulated dump failure")
|
||||
return b""
|
||||
def mongo(script):
|
||||
calls.append((script,))
|
||||
if "getParameter" in script: return b"true"
|
||||
if "storageSize" in script: return b'{"bytes":1,"collections":{}}'
|
||||
return b""
|
||||
def run(args, **kwargs):
|
||||
return b'[{"Image":"sha256:fixture"}]' if "inspect" in args else b""
|
||||
args = type("Args", (), {"confirm_maintenance": True, "recipient": "A" * 40, "output": str(Path(temp) / "backup.gpg")})()
|
||||
with patch.object(ops, "configuration", return_value={}), patch.object(ops, "run", side_effect=run), patch.object(ops, "compose", side_effect=compose), patch.object(ops, "mongo", side_effect=mongo), patch.object(ops, "maintenance_lock", return_value=contextlib.nullcontext()):
|
||||
with self.assertRaisesRegex(RuntimeError, "Simulated dump failure"):
|
||||
ops.backup(args)
|
||||
self.assertIn(("start", "api", "worker"), calls)
|
||||
self.assertTrue(any("ttlMonitorEnabled:true" in call[0] for call in calls))
|
||||
self.assertFalse(Path(args.output).exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user