Add operational health, encrypted backups and isolated restore drills

.dockerignore

@@ -11,3 +11,9 @@ tests
 **/pms.local.json
 
 **/payments.local.json
+.guestops-maintenance.lock
+.guestops-test-keyring
+**/__pycache__
+*.tar.gpg
+guestops-backup-*
+guestops-restore-*

.github/workflows/web.yml

@@ -24,6 +24,8 @@ jobs:
         with: { node-version: '22', cache: npm, cache-dependency-path: web/package-lock.json }
       - name: Build services
         run: dotnet build src/GuestOps.Worker/GuestOps.Worker.csproj -c Release
+      - name: Verify backup validation and failure recovery
+        run: python3 -m unittest discover -s tests -p test_ops.py
       - name: Build interface
         working-directory: web
         run: npm ci && npm run build
@@ -64,6 +66,17 @@ jobs:
           docker compose restart api worker
           curl --retry 30 --retry-delay 2 --retry-all-errors --fail http://127.0.0.1:8080/health
           python3 tests/production_smoke.py --read
+          install -m 600 /dev/null .env
+          python3 deploy/ops.py preflight --offline
+          mkdir -m 700 .guestops-test-keyring
+          export GNUPGHOME="$PWD/.guestops-test-keyring"
+          gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key 'GuestOps CI <ci@example.invalid>' rsa2048 encr 1d
+          BACKUP_RECIPIENT=$(gpg --batch --with-colons --list-keys | awk -F: '$1=="fpr" {print $10; exit}')
+          backup_dir=$(mktemp -d)
+          python3 deploy/ops.py backup --recipient "$BACKUP_RECIPIENT" --output "$backup_dir/fixture.tar.gpg" --confirm-maintenance
+          python3 deploy/ops.py restore-drill "$backup_dir/fixture.tar.gpg" --api-image "$GUESTOPS_API_IMAGE"
+          curl --retry 30 --retry-delay 2 --retry-all-errors --fail http://127.0.0.1:8080/health/ready
+          python3 tests/production_smoke.py --read
       - uses: actions/upload-artifact@v4
         if: github.event_name != 'pull_request'
         with:

.gitignore

@@ -15,3 +15,9 @@
 **/pms.local.json
 
 **/payments.local.json
+.guestops-maintenance.lock
+.guestops-test-keyring/
+guestops-backup-*/
+guestops-restore-*/
+*.tar.gpg
+__pycache__/

README.md

@@ -53,6 +53,8 @@ Open http://127.0.0.1:5173 and select **Open preview workspace**. Each preview l
 
 For MongoDB-backed operation, disable Preview and set `Mongo__ConnectionString`, `Mongo__Database`, and a persistent private `Keys__Path`. See [deployment](docs/deployment.md).
 
+Operational tooling includes an owner-only Workspace health page and Linux deployment preflight, encrypted backup and isolated restore drill. See [operations and recovery](docs/operations.md) before the hotel pilot.
+
 ## Verification
 
 ```sh

deploy/ops.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""GuestOps dedicated-Compose operations. Never prints credentials or provider data."""
+import argparse
+import contextlib
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tarfile
+import tempfile
+import time
+import urllib.request
+import uuid
+
+ROOT = Path(__file__).resolve().parents[1]
+FILES = {"mongo.archive.gz", "keys.tar.gz", "configuration.json", "manifest.json"}
+LIMIT = 8 * 1024**3
+
+
+def require(condition, message):
+    if not condition:
+        raise RuntimeError(message)
+
+
+def run(args, *, output=None, input_file=None, timeout=900):
+    # Provider output may contain secrets; errors identify only the program.
+    result = subprocess.run(args, cwd=ROOT, stdin=input_file or subprocess.DEVNULL,
+                            stdout=output or subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
+    require(result.returncode == 0, f"{args[0]} step failed. Check the private operator environment; no command output was logged.")
+    return result.stdout or b""
+
+
+def compose(*args, **kwargs):
+    return run(["docker", "compose", *args], **kwargs)
+
+
+def digest(path):
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def image_id(image):
+    require(not image.startswith("-"), "Invalid image reference.")
+    return run(["docker", "image", "inspect", "--format", "{{.Id}}", image]).decode().strip()
+
+
+AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);'
+INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));'
+
+
+def mongo(script):
+    return compose("exec", "-T", "mongo", "mongosh", "--quiet", "--nodb", "--eval", AUTH + script)
+
+
+def configuration():
+    config = json.loads(compose("config", "--format", "json"))
+    services = config["services"]
+    require(set(services) == {"api", "worker", "mongo"}, "This tool supports the dedicated three-service GuestOps Compose deployment only.")
+    require(not services["mongo"].get("ports"), "MongoDB must not have published ports.")
+    ports = services["api"].get("ports", [])
+    require(len(ports) == 1 and ports[0].get("host_ip") == "127.0.0.1" and int(ports[0]["target"]) == 8080, "API must publish only port 8080 on loopback.")
+    require(not services["worker"].get("ports"), "Worker must not publish ports.")
+    for name in ("api", "worker"):
+        env = services[name]["environment"]
+        require(env.get("Mongo__Database") == "guestops" and "@mongo:27017/guestops?" in env.get("Mongo__ConnectionString", ""), "Backup supports only the dedicated Compose guestops database.")
+        require(str(env.get("Preview", "false")).lower() != "true", "Production preview is forbidden.")
+        require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.")
+    require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.")
+    require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.")
+    key_sources = []
+    for name in ("api", "worker"):
+        keys = [v for v in services[name].get("volumes", []) if v["target"] == "/var/lib/guestops/keys"]
+        require(len(keys) == 1 and keys[0]["type"] == "volume", "API and worker require a shared persistent key volume.")
+        key_sources.append(keys[0]["source"])
+    require(key_sources[0] == key_sources[1], "API and worker key volumes differ.")
+    return config
+
+
+def preflight(args):
+    config = configuration()
+    env_file = ROOT / ".env"
+    require(env_file.is_file() and not env_file.is_symlink(), "Create a private .env file before deployment.")
+    require(stat.S_IMODE(env_file.stat().st_mode) & 0o077 == 0, ".env must not be accessible to group or other users.")
+    require(shutil.disk_usage(ROOT).free >= 8 * 1024**3, "Keep at least 8 GiB free before deployment; allow extra room for backups.")
+    for name, service in config["services"].items():
+        image_id(service["image"])
+        for volume in service.get("volumes", []):
+            if volume["type"] == "bind":
+                require(Path(volume["source"]).exists(), f"A required {name} bind mount is missing.")
+                if volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"):
+                    provider = Path(volume["source"])
+                    if any(json.loads(provider.read_text()).values()):
+                        require(stat.S_IMODE(provider.stat().st_mode) & 0o077 == 0, "Configured provider files must not be accessible to group or other users.")
+    if not args.offline:
+        url = config["services"]["api"]["environment"]["PublicUrl"]
+        require(re.fullmatch(r"https://[A-Za-z0-9.-]+", url), "PublicUrl must be an HTTPS hostname without a path.")
+        with urllib.request.urlopen(url + "/health/ready", timeout=15) as response:
+            require(response.url == url + "/health/ready" and json.load(response) == {"status": "ready"}, "Public HTTPS readiness failed.")
+    print("Preflight passed: private database, loopback API, production settings, images, mounts and disk headroom" + ("; HTTPS not checked." if args.offline else "; public HTTPS and database readiness checked."))
+
+
+@contextlib.contextmanager
+def maintenance_lock():
+    import fcntl
+    with (ROOT / ".guestops-maintenance.lock").open("a") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        yield
+
+
+def backup(args):
+    require(args.confirm_maintenance, "Backup requires --confirm-maintenance: API and workers will briefly stop.")
+    require(re.fullmatch(r"[A-Fa-f0-9]{40}", args.recipient), "Use a verified full 40-character GPG recipient fingerprint.")
+    run(["gpg", "--batch", "--list-keys", args.recipient])
+    destination = Path(args.output).resolve()
+    require(not destination.exists(), "Backup output already exists; choose a new filename.")
+    require(destination.parent.is_dir(), "Create the private backup directory first.")
+    require(stat.S_IMODE(destination.parent.stat().st_mode) & 0o077 == 0, "Backup directory must be private (mode 700).")
+    config = configuration()
+    runtime = {}
+    for service in ("api", "worker", "mongo"):
+        cid = compose("ps", "--status", "running", "-q", service).decode().strip()
+        require(re.fullmatch(r"[a-f0-9]{12,64}", cid), f"Exactly one running {service} container is required.")
+        runtime[service] = json.loads(run(["docker", "inspect", cid]))[0]
+    require(shutil.disk_usage(destination.parent).free >= 3 * json.loads(mongo(INVENTORY))["bytes"] + 1024**3, "Insufficient free space for a consistent encrypted backup.")
+    partial = destination.with_name(destination.name + ".partial-" + uuid.uuid4().hex)
+    with maintenance_lock(), tempfile.TemporaryDirectory(prefix="guestops-backup-", dir=destination.parent) as folder:
+        folder = Path(folder)
+        stopped = False
+        ttl = None
+        try:
+            # Set before stopping so partial stop failures also attempt recovery.
+            stopped = True
+            compose("stop", "-t", "150", "api", "worker")
+            ttl = json.loads(mongo('print(JSON.stringify(c.getDB("admin").runCommand({getParameter:1,ttlMonitorEnabled:1}).ttlMonitorEnabled));'))
+            require(isinstance(ttl, bool), "Cannot determine MongoDB TTL monitor state.")
+            mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:false});if(!r.ok)quit(1);')
+            inventory = json.loads(mongo(INVENTORY))
+            dump = 'set -eu; case "$MONGO_INITDB_ROOT_PASSWORD" in ""|*[!0-9a-fA-F]*) exit 1;; esac; umask 077; cfg=$(mktemp); trap \'rm -f "$cfg"\' EXIT; printf \'password: "%s"\\n\' "$MONGO_INITDB_ROOT_PASSWORD" > "$cfg"; mongodump --config "$cfg" --username "$MONGO_INITDB_ROOT_USERNAME" --authenticationDatabase admin --db guestops --archive --gzip'
+            with (folder / "mongo.archive.gz").open("wb") as output:
+                compose("exec", "-T", "mongo", "sh", "-c", dump, output=output)
+            probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
+            require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
+            with (folder / "keys.tar.gz").open("wb") as output:
+                compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "tar", "api", "-C", "/var/lib/guestops/keys", "-czf", "-", ".", output=output)
+            mounted = {}
+            for volume in config["services"]["api"].get("volumes", []):
+                if volume["type"] == "bind" and volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"):
+                    mounted[volume["target"]] = Path(volume["source"]).read_text()
+            (folder / "configuration.json").write_text(json.dumps({"compose": config, "runtime": runtime, "providerFiles": mounted}))
+            manifest = {"format": 1, "createdAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "inventory": inventory, "probe": probe, "apiImageId": runtime["api"]["Image"], "mongoImageId": runtime["mongo"]["Image"], "sha256": {name: digest(folder / name) for name in FILES - {"manifest.json"}}}
+            (folder / "manifest.json").write_text(json.dumps(manifest))
+        finally:
+            try:
+                if ttl is not None:
+                    mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:' + str(ttl).lower() + '});if(!r.ok)quit(1);')
+            finally:
+                if stopped:
+                    compose("start", "api", "worker")
+        try:
+            with tarfile.open(folder / "bundle.tar", "w") as archive:
+                for name in sorted(FILES):
+                    archive.add(folder / name, arcname=name, recursive=False)
+            run(["gpg", "--batch", "--trust-model", "always", "--recipient", args.recipient, "--output", str(partial), "--encrypt", str(folder / "bundle.tar")])
+            os.link(partial, destination)  # Atomic publication, never overwrite an existing backup.
+        finally:
+            partial.unlink(missing_ok=True)
+    print("Encrypted backup complete. Copy it off-server and perform a restore drill.")
+
+
+def unpack(bundle, folder):
+    with tarfile.open(bundle, "r:") as archive:
+        members = archive.getmembers()
+        require(len(members) == len(FILES) and {m.name for m in members} == FILES, "Unexpected backup members.")
+        require(all(m.isfile() and 0 <= m.size <= LIMIT for m in members) and sum(m.size for m in members) <= LIMIT, "Invalid or oversized backup members.")
+        for member in members:
+            with archive.extractfile(member) as source, (folder / member.name).open("xb") as output:
+                shutil.copyfileobj(source, output)
+    manifest = json.loads((folder / "manifest.json").read_text())
+    require(manifest.get("format") == 1 and set(manifest.get("sha256", {})) == FILES - {"manifest.json"}, "Unsupported backup format.")
+    for name, expected in manifest["sha256"].items():
+        require(digest(folder / name) == expected, "Backup checksum verification failed.")
+    return manifest
+
+
+def restore_drill(args):
+    backup_file = Path(args.backup).resolve()
+    require(backup_file.is_file(), "Backup file is missing.")
+    with tempfile.TemporaryDirectory(prefix="guestops-restore-", dir=ROOT) as temp:
+        folder = Path(temp)
+        run(["gpg", "--batch", "--max-output", str(LIMIT), "--output", str(folder / "bundle.tar"), "--decrypt", str(backup_file)])
+        require((folder / "bundle.tar").stat().st_size <= LIMIT, "Decrypted bundle exceeds the 8 GiB pilot limit.")
+        manifest = unpack(folder / "bundle.tar", folder)
+        require(image_id(args.api_image) == manifest["apiImageId"] and image_id(args.mongo_image) == manifest["mongoImageId"], "Load the exact trusted API and MongoDB images used for this backup.")
+        require(shutil.disk_usage(ROOT).free > 3 * manifest["inventory"]["bytes"] + 1024**3, "Insufficient restore drill space.")
+        keys = folder / "keys"
+        keys.mkdir(mode=0o700)
+        with tarfile.open(folder / "keys.tar.gz", "r:gz") as archive:
+            total = 0
+            for member in archive:
+                if member.name in (".", "./") and member.isdir():
+                    continue
+                name = member.name.removeprefix("./")
+                total += member.size
+                require(member.isfile() and re.fullmatch(r"[A-Za-z0-9_-]+\.xml", name) and member.size < 1024**2 and total < 16 * 1024**2, "Invalid key archive.")
+                with archive.extractfile(member) as source, (keys / name).open("xb") as output:
+                    shutil.copyfileobj(source, output)
+        run(["docker", "run", "--rm", "--pull", "never", "--network", "none", "--user", "0:0", "--read-only", "--mount", f"type=bind,src={keys},dst=/var/lib/guestops/keys,readonly", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + manifest["probe"], args.api_image, "--verify-backup-probe"])
+        cid = run(["docker", "run", "-d", "--pull", "never", "--network", "none", "--memory", "1g", "--label", "guestops.restore-drill=true", args.mongo_image, "--bind_ip", "127.0.0.1", "--setParameter", "ttlMonitorEnabled=false"]).decode().strip()
+        require(re.fullmatch(r"[a-f0-9]{64}", cid), "Unexpected restore container identifier.")
+        try:
+            for attempt in range(30):
+                try:
+                    run(["docker", "exec", cid, "mongosh", "--quiet", "--eval", "db.adminCommand({ping:1})"], timeout=10)
+                    break
+                except RuntimeError:
+                    if attempt == 29:
+                        raise
+                    time.sleep(1)
+            with (folder / "mongo.archive.gz").open("rb") as source:
+                run(["docker", "exec", "-i", cid, "mongorestore", "--archive", "--gzip", "--nsInclude", "guestops.*"], input_file=source)
+            restored = json.loads(run(["docker", "exec", cid, "mongosh", "--quiet", "--nodb", "--eval", 'const c=new Mongo("mongodb://127.0.0.1");' + INVENTORY]))
+            require(restored["collections"] == manifest["inventory"]["collections"], "Restored collection counts or indexes differ.")
+        finally:
+            run(["docker", "rm", "-f", "-v", cid])  # Only the exact container created above and its anonymous volumes.
+    print("Restore drill passed: collection counts, indexes and actual key decryption verified in isolated containers. Production was not restored or modified.")
+
+
+def main():
+    require(os.name == "posix", "Run deployment operations on Linux.")
+    os.umask(0o077)
+    parser = argparse.ArgumentParser(description=__doc__)
+    subs = parser.add_subparsers(dest="command", required=True)
+    check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
+    save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
+    drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
+    args = parser.parse_args()
+    {"preflight": preflight, "backup": backup, "restore-drill": restore_drill}[args.command](args)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as error:
+        # Never include subprocess output, config values or parsed guest data.
+        print(str(error) if isinstance(error, RuntimeError) else "Operation failed (" + type(error).__name__ + "). No sensitive details were logged.", file=sys.stderr)
+        sys.exit(1)

docs/deployment.md

@@ -44,7 +44,7 @@ docker compose run --rm --no-deps -e BOOTSTRAP_EMAIL -e BOOTSTRAP_HOTEL -e BOOTS
 unset BOOTSTRAP_EMAIL BOOTSTRAP_HOTEL BOOTSTRAP_PASSWORD
 ```
 
-There is no development/demo account in production. Staff invitation and password recovery UI are follow-on work; do not treat this as a public self-service service yet.
+There is no development/demo account in production. Owners can issue staff invitation and assisted recovery links through Team; see [account setup](accounts.md). Passwords must be 14–128 characters. Public self-registration is not enabled.
 
 ## 4. Configure HTTPS
 
@@ -72,4 +72,4 @@ A multi-hotel production launch using Gmail restricted scopes requires planning
 
 Verify separate hotels cannot read or edit each other's records; save and reload settings; restart services and confirm persistence; import test messages twice without duplicates; check the worker resumes a paginated import; confirm no mail is sent without explicit staff approval and that default-disabled sending remains blocked. Review the activity log and Google account used by the connection.
 
-Keep reviewed image tags for rollback and backups of both MongoDB and the key volume. Do not remove named volumes to fix application errors. The initial release has no automatic schema migration that destroys data. Establish retention, off-server backup and a restore drill before importing real guest data.
+Keep reviewed image IDs and release images for rollback and backups of both MongoDB and the key volume. Do not remove named volumes to fix application errors. The initial release has no automatic schema migration that destroys data. Use the [operational preflight, encrypted backup and isolated restore drill](operations.md), and establish retention and off-server copies before importing real guest data. `/health/ready` checks database reachability; the owner's Workspace health page also reports worker heartbeat and mailbox/reply exceptions.

docs/migration.md

@@ -40,6 +40,10 @@ Implemented owner-issued single-use invitation and recovery links, staff disable
 
 Implemented owner-only disconnect/reconnect and import restart controls, synchronization health, revoked-access recovery, retry delays, page recovery and connection-bound reply approvals. Local disconnect removes saved credentials; provider grant revocation is a separate Google account action. See [mailbox operation and acceptance](mailboxes.md).
 
+## Milestone 8: operational readiness
+
+Implemented owner-only workspace health, database readiness, worker heartbeat, Linux deployment preflight, maintenance-window encrypted database/key/configuration backup and an isolated restore drill. The drill verifies collection counts, indexes and actual key decryption. Scheduling, off-server copies and production disaster cutover remain operator tasks; the Debian server rehearsal is still required. See [operations and recovery](operations.md).
+
 ## Remaining milestones
 
 1. Run dedicated Google test-mailbox acceptance, add full thread aggregation and history repair, and rehearse server backup/restore before the first hotel pilot.

docs/operations.md

@@ -0,0 +1,70 @@
+# Operations and recovery
+
+The owner-only **Workspace health** page reports database reachability, the worker's last heartbeat, mailbox recovery counts and reply exceptions. A heartbeat older than three minutes is marked stale. It proves that the worker process recently reached MongoDB, not that every provider or job succeeded. Reply totals cover at most the latest 500 conversations in this hotel. The page does not verify backups. `/health/ready` returns only readiness status and HTTP 503 when the database cannot be reached.
+
+## Deployment preflight
+
+Run from the dedicated GuestOps checkout on Linux with Python 3.11 or later, Docker with Compose, and GnuPG installed. The tool supports the supplied three-service Compose deployment and the `guestops` database only. Docker access is administrator-equivalent; use the designated server operator account. Load the reviewed API, worker and MongoDB images first, configure `.env` with mode 600, and follow [deployment](deployment.md).
+
+```sh
+python3 deploy/ops.py preflight --offline
+# After Nginx, DNS, TLS and services are running:
+python3 deploy/ops.py preflight
+```
+
+Preflight checks production settings, shared persistent keys, unpublished MongoDB/worker ports, a loopback API port, required images and mounts, private secret files, and at least 8 GiB free disk. The online check also verifies the configured HTTPS readiness URL. This is not a firewall, capacity or provider acceptance test. Allow additional disk space for the database, images and temporary backup/restore files; the supplied server initially had 18 GiB free.
+
+## Encrypted backup
+
+Keep the recovery private key on an administrator-controlled recovery machine, with a securely stored passphrase and a second protected recovery copy. Import only its public key on the server and verify its full 40-character fingerprint through a trusted channel. The tool selects that exact fingerprint; it does not establish who owns the key. Do not put private keys, decrypted backups or provider secrets in GitHub.
+
+```sh
+gpg --import /secure/path/recovery-public.asc
+gpg --fingerprint
+install -d -m 700 "$HOME/guestops-backups"
+read -r -p 'Verified recovery key fingerprint: ' BACKUP_RECIPIENT
+python3 deploy/ops.py backup --recipient "$BACKUP_RECIPIENT" \
+  --output "$HOME/guestops-backups/guestops-$(date -u +%Y%m%dT%H%M%SZ).tar.gpg" \
+  --confirm-maintenance
+unset BACKUP_RECIPIENT
+```
+
+This command causes a maintenance interruption. It stops the API and worker, temporarily pauses MongoDB's TTL expiry monitor, dumps MongoDB, and saves the shared Data Protection keys and deployed configuration. The configuration includes credentials and provider files, so the entire bundle is encrypted. Services and the previous TTL setting are restored in a `finally` block before encryption finishes. A successful backup message does not prove that restarted services are healthy; run the online preflight afterwards.
+
+No other application or administrator may write to this database during the snapshot. Standalone MongoDB dumps need coordinated writes for consistency; see the [MongoDB backup guidance](https://www.mongodb.com/docs/v8.0/tutorial/backup-and-restore-tools/). The tool is intentionally limited to the dedicated stack. It requires the hexadecimal MongoDB root password generated in the deployment guide.
+
+Copy the encrypted file off-server to restricted storage after every successful backup. Retain the exact API, worker and MongoDB images with the release: an image tag alone can change, and the drill requires matching image IDs. Agree the backup schedule, retention and tolerated data loss before a hotel pilot. Scheduling, off-server transfer, deletion and alerting are operator responsibilities in this milestone; none is silently installed.
+
+Temporary plaintext files are held in private directories and removed on normal completion or exceptions. Process termination or power loss can leave temporary data, stopped services or TTL expiry disabled. After an interrupted run, inspect the dedicated project and remove only its identified abandoned temporary directory after securing any recovery material. Restore the recorded TTL setting (normally true) and restart the services:
+
+```sh
+docker compose exec -T mongo mongosh --quiet --nodb --eval 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD); const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:true}); if(!r.ok)quit(1);'
+docker compose start api worker
+python3 deploy/ops.py preflight
+```
+
+Use the previous value instead of true if TTL expiry was deliberately disabled beforehand. Do not remove production volumes to recover from a failed backup.
+
+## Isolated restore drill
+
+Use a trusted encrypted backup and the recovery key on a Linux recovery machine. Unlock the key through the local GPG agent before running the batch command; never pass its passphrase on the command line. Load the exact reviewed images from the backed-up release first.
+
+```sh
+python3 deploy/ops.py restore-drill /secure/path/guestops-backup.tar.gpg \
+  --api-image guestops-api:REVIEWED_RELEASE \
+  --mongo-image mongo:8.0
+```
+
+The drill decrypts into a private temporary directory, checks the four expected files and their hashes, rejects unsafe archive entries, and verifies that the restored keys decrypt a protected test value. It then restores MongoDB into a newly created container with no external network or published ports and compares every restored collection's count and indexes. TTL expiry is disabled in this disposable database so expired token records do not disappear during comparison. The exact temporary container and its volumes are removed afterwards. No worker or production application is started.
+
+The decrypted bundle is capped at 8 GiB for this pilot tool. Leave room for the encrypted file, decrypted archive, extracted files and restored database. GPG's [output limit](https://www.gnupg.org/documentation/manuals/gnupg/GPG-Input-and-Output.html) bounds decrypted output; checksums detect corruption, not the identity of the backup creator. Use backups from the trusted operator only. A drill checks counts, indexes and key decryption, not every document's business meaning or live provider connectivity.
+
+## Actual disaster recovery
+
+This milestone implements a rehearsal, not an automatic production restore or cutover command. Before real guest data is accepted, rehearse a separate recovery deployment and document its precise image IDs, volume names, secret locations and operator responsibilities.
+
+Restore into new isolated MongoDB and key volumes; preserve the damaged original for investigation. Reconstruct reviewed configuration from the encrypted bundle without copying old Docker container IDs or blindly executing archived configuration. Restore the `guestops` database with the compatible MongoDB tools and restore the matching key files with the application's required ownership. Check hotel/account records and saved settings before exposing the recovered API. Keep the worker stopped, provider writes disabled and external network access restricted throughout this process.
+
+**A restored database can predate emails, invoices and PMS changes that providers already completed.** Review pending, sending and uncertain records against provider evidence before enabling any worker, including automatic FAQ rules. Do not replay an older approval merely because the restored record says it is pending. Reconcile external effects, validate account sessions and mailbox authorization, and explicitly approve the cutover only after these checks. Rotate credentials if compromise prompted the recovery. Keep the old deployment stopped when enabling the replacement.
+
+CI exercises a synthetic encrypted backup and isolated restore drill, including actual key decryption and database comparison. A successful CI drill is separate from the required rehearsal on the Debian server with its actual deployment configuration.

src/GuestOps.Api/Operations.cs

@@ -0,0 +1,29 @@
+using Microsoft.AspNetCore.DataProtection;
+namespace GuestOps.Web;
+public sealed class WorkerHeartbeat
+{
+    [MongoDB.Bson.Serialization.Attributes.BsonId] public string Id {get;set;}="worker";
+    public DateTime At {get;set;}=DateTime.UtcNow;
+}
+public static class BackupProbe
+{
+    const string Value="GuestOps backup key verification v1";
+    public static string Create(IDataProtectionProvider protection)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Protect(Value);
+    public static bool Verify(IDataProtectionProvider protection,string value)=>protection.CreateProtector("GuestOps.BackupProbe.v1").Unprotect(value)==Value;
+}
+public static class Operations
+{
+    public static void Map(WebApplication app,RouteGroupBuilder api,bool preview)
+    {
+        app.MapGet("/health/ready",async(IStore store,HttpContext c)=>
+        {
+            c.Response.Headers.CacheControl="no-store";
+            try{await store.Ping();return Results.Ok(new{status="ready"});}catch{return Results.Json(new{status="unavailable"},statusCode:503);}
+        });
+        api.MapGet("/operations",async(HttpContext c,IStore store)=>
+        {
+            var id=Session.Hotel(c);var boxes=await store.List<Mailbox>(id);var messages=await store.List<Conversation>(id);var seen=await store.WorkerLastSeen();
+            return Results.Ok(new{checkedAt=DateTime.UtcNow,preview,database="Reachable",worker=new{lastSeenAt=seen,state=preview?"Preview":seen==null?"NotSeen":seen<DateTime.UtcNow.AddMinutes(-3)?"Stale":"Reporting"},mailboxes=new{total=boxes.Count,connected=boxes.Count(x=>x.Status=="Connected"),attention=boxes.Count(x=>x.Status!="Connected"||x.SyncError.Length>0)},replies=new{sampleSize=messages.Count,sampleLimit=500,pending=messages.Count(x=>x.Delivery?.State is "Pending" or "Sending"),uncertain=messages.Count(x=>x.Delivery?.State=="NeedsReview"),rejected=messages.Count(x=>x.Delivery?.State=="Rejected")}});
+        }).RequireAuthorization("Owner");
+    }
+}

src/GuestOps.Api/Program.cs

@@ -13,7 +13,8 @@ using System.Net;
 
 var bootstrap = args.Contains("--bootstrap");
 var recoverOwner = args.Contains("--recover-owner");
-var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner").ToArray());
+var backupProbe=args.Contains("--backup-probe");var verifyBackupProbe=args.Contains("--verify-backup-probe");
+var builder = WebApplication.CreateBuilder(args.Where(arg => arg != "--bootstrap" && arg != "--recover-owner" && arg != "--backup-probe" && arg != "--verify-backup-probe").ToArray());
 if (builder.Configuration["Pms:ConfigFile"] is { Length: > 0 } pmsConfigFile) builder.Configuration.AddJsonFile(pmsConfigFile,optional:false,reloadOnChange:false);
 if (builder.Configuration["Payments:ConfigFile"] is { Length: > 0 } paymentConfigFile) builder.Configuration.AddJsonFile(paymentConfigFile,optional:false,reloadOnChange:false);
 builder.Logging.ClearProviders(); builder.Logging.AddConsole();
@@ -71,6 +72,14 @@ builder.Services.AddRateLimiter(o =>
     o.AddPolicy("login", context => RateLimitPartition.GetFixedWindowLimiter(context.Connection.RemoteIpAddress?.ToString() ?? "unknown", _ => new() { PermitLimit = 10, Window = TimeSpan.FromMinutes(1), QueueLimit = 0 }));
 });
 var app = builder.Build();
+if(backupProbe||verifyBackupProbe)
+{
+    var provider=app.Services.GetRequiredService<IDataProtectionProvider>();
+    if(backupProbe)Console.WriteLine(BackupProbe.Create(provider));
+    else if(!BackupProbe.Verify(provider,Environment.GetEnvironmentVariable("BACKUP_PROBE")??""))throw new InvalidOperationException("Backup keys failed verification.");
+    else Console.WriteLine("Backup keys verified.");
+    return;
+}
 app.UseForwardedHeaders();
 var store = app.Services.GetRequiredService<IStore>();
 await store.Initialize();
@@ -148,6 +157,7 @@ PaymentEndpoints.Map(api,preview);
 AutoReplyEndpoints.Map(api,preview);
 TeamEndpoints.Map(app,api,preview);
 MailboxManagement.Map(api,preview);
+Operations.Map(app,api,preview);
 api.MapGet("/hotel", async (HttpContext c, CancellationToken _) => Results.Ok(await store.Get<Hotel>(Session.Hotel(c), Session.Hotel(c))));
 api.MapPut("/hotel", async (SettingsInput input, HttpContext c) =>
 {

src/GuestOps.Api/Store.cs

@@ -7,6 +7,9 @@ namespace GuestOps.Web;
 public interface IStore
 {
     Task Initialize();
+    Task Ping();
+    Task<DateTime?> WorkerLastSeen();
+    Task RecordWorkerHeartbeat();
     Task<List<T>> List<T>(string hotel) where T : TenantDocument;
     Task<T?> Get<T>(string hotel, string id) where T : TenantDocument;
     Task Insert<T>(T document) where T : TenantDocument;
@@ -32,6 +35,9 @@ public interface IStore
 }
 public sealed class MongoStore : IStore
 {
+    public async Task Ping()=>await db.RunCommandAsync<MongoDB.Bson.BsonDocument>(new MongoDB.Bson.BsonDocument("ping",1));
+    public async Task<DateTime?> WorkerLastSeen()=>(await db.GetCollection<WorkerHeartbeat>("workerheartbeat").Find(x=>x.Id=="worker").FirstOrDefaultAsync())?.At;
+    public async Task RecordWorkerHeartbeat()=>await db.GetCollection<WorkerHeartbeat>("workerheartbeat").ReplaceOneAsync(x=>x.Id=="worker",new WorkerHeartbeat(),new ReplaceOptions{IsUpsert=true});
     public Task<List<Conversation>> Deliveries() => Collection<Conversation>().Find(x => x.Delivery != null && (x.Delivery.State == "Pending" || x.Delivery.State == "Sending")).SortBy(x => x.Delivery!.UpdatedAt).Limit(100).ToListAsync();
     private readonly IMongoDatabase db;
     public MongoStore(IConfiguration config)
@@ -141,6 +147,9 @@ public sealed class MongoStore : IStore
 // Explicit Development-only preview store. Production never falls back to this.
 public sealed class PreviewStore : IStore
 {
+    public Task Ping()=>Task.CompletedTask;
+    public Task<DateTime?> WorkerLastSeen()=>Task.FromResult<DateTime?>(null);
+    public Task RecordWorkerHeartbeat()=>Task.CompletedTask;
     public async Task<List<Conversation>> AutoReplyCandidates(string hotel,string mailbox,DateTime since)=>(await List<Conversation>(hotel)).Where(x=>x.MailboxId==mailbox&&x.AutoReplyCheckedAt==null&&x.ReceivedAt>=since).OrderBy(x=>x.ReceivedAt).Take(100).ToList();
     public Task<bool> TryAutoReplyClaim(AutoReplyClaim claim)
     {

src/GuestOps.Worker/Program.cs

@@ -17,8 +17,21 @@ builder.Services.AddTransient<AutoReplyWork>();
 builder.Services.AddHostedService<AutoReplyWorker>();
 builder.Services.AddHostedService<DeliveryWorker>();
 builder.Services.AddHostedService<MailboxWorker>();
+builder.Services.AddHostedService<HeartbeatWorker>();
 await builder.Build().RunAsync();
 
+sealed class HeartbeatWorker(IStore store,ILogger<HeartbeatWorker> log):BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        while(!stoppingToken.IsCancellationRequested)
+        {
+            try{await store.RecordWorkerHeartbeat();}catch(Exception ex){log.LogWarning("Worker heartbeat unavailable ({Type})",ex.GetType().Name);}
+            await Task.Delay(TimeSpan.FromSeconds(30),stoppingToken);
+        }
+    }
+}
+
 sealed class MailboxWorker(IStore store, IServiceScopeFactory factory, ILogger<MailboxWorker> log) : BackgroundService
 {
     readonly string owner = Guid.NewGuid().ToString("N");

tests/GuestOps.Tests/Program.cs

@@ -15,6 +15,13 @@ IStore store = uri == null ? new PreviewStore() : new MongoStore(new Configurati
 await store.Initialize();
 try
 {
+    var proofProvider=new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider();
+    var proof=BackupProbe.Create(proofProvider);
+    Check("Backup proof decrypts with the original key provider",BackupProbe.Verify(proofProvider,proof));
+    bool wrongKeys=false;try{BackupProbe.Verify(new Microsoft.AspNetCore.DataProtection.EphemeralDataProtectionProvider(),proof);}catch(System.Security.Cryptography.CryptographicException){wrongKeys=true;}
+    Check("Backup proof rejects unrelated encryption keys",wrongKeys);
+    await store.Ping();
+    if(uri!=null){await store.RecordWorkerHeartbeat();Check("Worker heartbeat persists in MongoDB",await store.WorkerLastSeen()>DateTime.UtcNow.AddMinutes(-1));}
     await MailboxTests.Run(Check, store);
     await TeamTests.Run(Check, store);
     await ReplyTests.Run(Check, store);
@@ -73,12 +80,15 @@ try
         async Task<JsonElement> Read(HttpClient h,string path) => JsonDocument.Parse(await h.GetStringAsync(path)).RootElement.Clone();
         async Task SetCsrf(HttpClient h) { var s=await Read(h,"/api/session");h.DefaultRequestHeaders.Remove("X-CSRF-TOKEN");h.DefaultRequestHeaders.Add("X-CSRF-TOKEN",s.GetProperty("csrfToken").GetString()); }
         Check("Anonymous inbox access blocked", (await one.GetAsync("/api/conversations")).StatusCode == HttpStatusCode.Unauthorized);
+        Check("Anonymous readiness returns only status",(await Read(one,"/health/ready")).GetRawText()=="{\"status\":\"ready\"}");
         Check("Unsafe requests require CSRF token", (await one.PostAsJsonAsync("/api/preview/start",new {})).StatusCode == HttpStatusCode.BadRequest);
         await SetCsrf(one); await SetCsrf(two);
         Check("Preview creates signed-in workspace", (await one.PostAsJsonAsync("/api/preview/start",new {})).IsSuccessStatusCode);
         await two.PostAsJsonAsync("/api/preview/start",new {}); await SetCsrf(one); await SetCsrf(two);
         var h1=await Read(one,"/api/hotel");var h2=await Read(two,"/api/hotel");
         Check("Different preview sessions have different hotels", h1.GetProperty("id").GetString()!=h2.GetProperty("id").GetString());
+        var health=await Read(one,"/api/operations");
+        Check("Health overview is hotel scoped and labels preview worker",health.GetProperty("mailboxes").GetProperty("total").GetInt32()==1&&health.GetProperty("worker").GetProperty("state").GetString()=="Preview");
         var messages=await Read(one,"/api/conversations"); var id=messages[0].GetProperty("id").GetString();
         Check("Guessed message ID cannot be edited across hotels", (await two.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="stolen",version=0})).StatusCode==HttpStatusCode.NotFound);
         Check("Draft save succeeds", (await one.PutAsJsonAsync($"/api/conversations/{id}/draft",new {draft="Hello guest",version=0})).IsSuccessStatusCode);

tests/GuestOps.Tests/TeamTests.cs

@@ -63,6 +63,7 @@ public static class TeamTests
         await Csrf(staff);check("Invited colleague can sign in",(await staff.PostAsJsonAsync("/api/auth/login",new{email,password})).IsSuccessStatusCode);await Csrf(staff);
         check("Staff cannot list or invite team members",(await staff.GetAsync("/api/team")).StatusCode==HttpStatusCode.Forbidden&&(await staff.PostAsJsonAsync("/api/team/invite",new{name="No",email="no@example.invalid"})).StatusCode==HttpStatusCode.Forbidden);
         check("Staff cannot change hotel settings",(await staff.PutAsJsonAsync("/api/hotel",new{name="No",signature="",timezone="UTC",version=0})).StatusCode==HttpStatusCode.Forbidden);
+        check("Operational health is owner only",(await staff.GetAsync("/api/operations")).StatusCode==HttpStatusCode.Forbidden);
         foreach(var action in new[]{"disconnect","reconnect","retry"})check("Staff cannot "+action+" mailboxes",(await staff.PostAsJsonAsync($"/api/mailboxes/unknown/{action}",new{version=0})).StatusCode==HttpStatusCode.Forbidden);
         check("Foreign hotel cannot reset staff",(await other.PostAsJsonAsync($"/api/team/{id}/reset",new{version=2})).StatusCode==HttpStatusCode.NotFound);
         var list=await Read(await owner.GetAsync("/api/team"));var member=list.EnumerateArray().Single(x=>x.GetProperty("id").GetString()==id);var version=member.GetProperty("version").GetInt64();

tests/production_smoke.py

@@ -6,6 +6,7 @@ against an existing hotel database. Cookie values and credentials are not logged
 import json
 import os
 import sys
+import time
 from http.cookies import SimpleCookie
 from urllib.request import Request, urlopen
 from urllib.error import HTTPError
@@ -43,6 +44,13 @@ request("/api/auth/login", "POST", {"email": os.environ["BOOTSTRAP_EMAIL"], "pas
 session = request("/api/session")
 assert session["user"]["role"] == "Owner"
 csrf = session["csrfToken"]
+assert request("/health/ready") == {"status": "ready"}
+for attempt in range(10):
+    health = request("/api/operations")
+    if health["worker"]["state"] == "Reporting":
+        break
+    time.sleep(1)
+assert health["worker"]["state"] == "Reporting" and health["database"] == "Reachable"
 auto_status = request("/api/auto-replies/status")
 assert auto_status["liveConfigured"] is False
 request("/api/auto-replies/mode", "PUT", {"mode": "Live", "version": 0, "acceptanceConfirmed": True}, expected=400)
@@ -68,3 +76,4 @@ else:
 request("/api/auth/logout", "POST")
 request("/api/hotel", expected=401)
 print("Production smoke checks passed: built UI, secure cookies, owner login, persisted settings and staff invitation, onboarding and logout.")
+

tests/test_ops.py

@@ -0,0 +1,90 @@
+import contextlib
+import hashlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import tarfile
+import tempfile
+import unittest
+from unittest.mock import patch
+
+spec = importlib.util.spec_from_file_location("ops", Path(__file__).resolve().parents[1] / "deploy" / "ops.py")
+ops = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(ops)
+
+
+class ArchiveTests(unittest.TestCase):
+    def bundle(self, root, change=None):
+        files = {name: b"fixture backup data" for name in ops.FILES - {"manifest.json"}}
+        files["manifest.json"] = json.dumps({"format": 1, "sha256": {name: hashlib.sha256(data).hexdigest() for name, data in files.items()}}).encode()
+        target = root / "bundle.tar"
+        with tarfile.open(target, "w") as archive:
+            for name, data in files.items():
+                member = tarfile.TarInfo(name); member.size = len(data)
+                if change:
+                    change(member)
+                archive.addfile(member, io.BytesIO(data) if member.isfile() else None)
+        return target
+
+    def test_checked_archive_roundtrip(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); dest = root / "dest"; dest.mkdir()
+            self.assertEqual(ops.unpack(self.bundle(root), dest)["format"], 1)
+
+    def test_path_escape_rejected(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); dest = root / "dest"; dest.mkdir()
+            def corrupt(member):
+                if member.name == "configuration.json": member.name = "../outside"
+            with self.assertRaisesRegex(RuntimeError, "Unexpected backup members"):
+                ops.unpack(self.bundle(root, corrupt), dest)
+            self.assertFalse((root / "outside").exists())
+
+    def test_symlink_rejected(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); dest = root / "dest"; dest.mkdir()
+            def corrupt(member):
+                if member.name == "configuration.json": member.type = tarfile.SYMTYPE; member.linkname = "/etc/passwd"; member.size = 0
+            with self.assertRaisesRegex(RuntimeError, "Invalid or oversized"):
+                ops.unpack(self.bundle(root, corrupt), dest)
+
+    def test_checksum_mismatch_rejected(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp); dest = root / "dest"; dest.mkdir()
+            with patch.object(ops, "digest", return_value="changed"):
+                with self.assertRaisesRegex(RuntimeError, "checksum"):
+                    ops.unpack(self.bundle(root), dest)
+
+    def test_backup_requires_explicit_maintenance(self):
+        with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"):
+            ops.backup(type("Args", (), {"confirm_maintenance": False})())
+
+    @unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
+    def test_dump_failure_restarts_services_and_ttl(self):
+        with tempfile.TemporaryDirectory() as temp:
+            calls = []
+            def compose(*args, **kwargs):
+                calls.append(args)
+                if args[0] == "ps": return b"a" * 64
+                if "sh" in args: raise RuntimeError("Simulated dump failure")
+                return b""
+            def mongo(script):
+                calls.append((script,))
+                if "getParameter" in script: return b"true"
+                if "storageSize" in script: return b'{"bytes":1,"collections":{}}'
+                return b""
+            def run(args, **kwargs):
+                return b'[{"Image":"sha256:fixture"}]' if "inspect" in args else b""
+            args = type("Args", (), {"confirm_maintenance": True, "recipient": "A" * 40, "output": str(Path(temp) / "backup.gpg")})()
+            with patch.object(ops, "configuration", return_value={}), patch.object(ops, "run", side_effect=run), patch.object(ops, "compose", side_effect=compose), patch.object(ops, "mongo", side_effect=mongo), patch.object(ops, "maintenance_lock", return_value=contextlib.nullcontext()):
+                with self.assertRaisesRegex(RuntimeError, "Simulated dump failure"):
+                    ops.backup(args)
+            self.assertIn(("start", "api", "worker"), calls)
+            self.assertTrue(any("ttlMonitorEnabled:true" in call[0] for call in calls))
+            self.assertFalse(Path(args.output).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

web/src/OperationsPage.tsx

@@ -0,0 +1,10 @@
+import { useEffect, useState } from 'react';
+import { api } from './api';
+type Health={checkedAt:string;preview:boolean;database:string;worker:{state:string;lastSeenAt:string|null};mailboxes:{total:number;connected:number;attention:number};replies:{sampleSize:number;sampleLimit:number;pending:number;uncertain:number;rejected:number}};
+export function OperationsPage({owner,go}:{owner:boolean;go:(path:string)=>void}){
+  const [data,setData]=useState<Health|null>(null),[error,setError]=useState(''),[busy,setBusy]=useState(false);
+  async function refresh(){setBusy(true);setError('');try{setData(await api<Health>('/operations'));}catch(e){setError(e instanceof Error?e.message:'Unable to check workspace health.');}finally{setBusy(false);}}
+  useEffect(()=>{if(owner)void refresh();},[owner]);
+  if(!owner)return <div className="page"><h1>Workspace health</h1><p>Your hotel owner can review operational health.</p></div>;
+  return <div className="page settings-page"><div className="heading-row"><div className="page-heading"><span className="eyebrow">Keep your workspace running</span><h1>Workspace health</h1><p>Spot connection and delivery issues before they affect your team.</p></div><button className="button secondary" disabled={busy} onClick={refresh}>{busy?'Checking…':'Refresh health'}</button></div>{error&&<div className="alert" role="alert">{error} The information below may be out of date.</div>}{data&&<><p className="small muted">Checked {new Date(data.checkedAt).toLocaleString()}{data.preview?' · Sample workspace':''}</p><section className="settings-card"><h2>Application and worker</h2><div className="mailbox-health"><div><span>Database connection</span><strong>{data.preview?'Temporary preview storage':data.database}</strong></div><div><span>Background worker</span><strong>{data.worker.state==='Reporting'?'Reporting normally':data.worker.state==='Preview'?'Unavailable in preview':data.worker.state==='Stale'?'Heartbeat overdue':'No heartbeat received'}</strong></div><div><span>Last worker heartbeat</span><strong>{data.worker.lastSeenAt?new Date(data.worker.lastSeenAt).toLocaleString():'Not yet'}</strong></div></div>{['Stale','NotSeen'].includes(data.worker.state)&&<p className="mailbox-explanation">Ask your server administrator to check the worker container and its database connection. Imports and queued replies may be delayed.</p>}<p className="small muted">A heartbeat confirms the worker process can reach storage. It does not prove that Google, payment or PMS requests are succeeding.</p></section><section className="settings-card"><h2>Mailbox connections</h2><div className="mailbox-health"><div><span>Total</span><strong>{data.mailboxes.total}</strong></div><div><span>Connected</span><strong>{data.mailboxes.connected}</strong></div><div><span>Need attention</span><strong>{data.mailboxes.attention}</strong></div></div><button className="button secondary" onClick={()=>go('/settings')}>Review mailbox status</button></section><section className="settings-card"><h2>Reply delivery</h2><div className="mailbox-health"><div><span>Queued or submitting</span><strong>{data.replies.pending}</strong></div><div><span>Need verification</span><strong>{data.replies.uncertain}</strong></div><div><span>Stopped before sending</span><strong>{data.replies.rejected}</strong></div></div><p className="small muted">Based on {data.replies.sampleSize} recent conversations, up to {data.replies.sampleLimit}. Older deliveries may exist. Verify uncertain results in Gmail before taking further action.</p><div className="form-actions"><button className="button secondary" onClick={()=>go('/inbox')}>Review the inbox</button></div></section><section className="settings-card"><h2>Backups and recovery</h2><p className="muted">Your server administrator runs encrypted backups and isolated restore drills. Ask them to confirm the latest off-server backup and successful restore test.</p><p className="small muted">This page does not claim a backup exists or that the server is ready for production. Provider acceptance and recovery checks are separate.</p></section></>}</div>;
+}

web/src/main.tsx

@@ -2,7 +2,8 @@ import React, { useEffect, useState } from 'react';
 import { createRoot } from 'react-dom/client';
 import { Inbox, BookOpen, Settings, Activity as ActivityIcon, Search, ArrowUpRight, ChevronDown, Check, CheckCheck, Clock3, FileText, LogOut, RefreshCw, ArrowLeft, Plus, X, Mail, ShieldCheck, Save, CircleHelp, Banknote, Building2, ChevronRight } from 'lucide-react';
 import { api, session, type Session, type Hotel, type Conversation, type Knowledge, type Activity, type Mailboxes } from './api';
-import './style.css';
+import './style.css';
+import { OperationsPage } from './OperationsPage';
 import { MailboxPanel } from './MailboxPanel';
 import { TeamPage, OnboardingPage, AccountPage } from './TeamPage';
 import { AutomationPage } from './AutomationPage';
@@ -42,13 +43,13 @@ function App() {
       <a href="/inbox" aria-label="GuestOps inbox" className="brand" onClick={e=>{e.preventDefault();go('/inbox');}}><span className="brand-mark">g</span><span>guestops<span className="brand-dot">.</span></span></a>
       <div className="hotel-switch"><span className="hotel-icon"><Building2 size={19}/></span><div><strong>{hotel?.name||'Your hotel'}</strong><small>Hotel workspace</small></div></div>
       <div className="nav-label">WORKSPACE</div>
-      <nav aria-label="Main navigation">{[{path:'/inbox',label:'Inbox',icon:Inbox},{path:'/reservations',label:'Reservations',icon:Building2},{path:'/payments',label:'Payments',icon:Banknote},{path:'/automation',label:'FAQ automation',icon:ShieldCheck},{path:'/knowledge',label:'Hotel knowledge',icon:BookOpen},{path:'/activity',label:'Activity',icon:ActivityIcon},{path:'/settings',label:'Settings',icon:Settings},{path:'/team',label:'Your team',icon:ShieldCheck},{path:'/setup',label:'Hotel setup',icon:CheckCheck}].filter(n=>auth.user?.role==='Owner'||!['/team','/setup'].includes(n.path)).map(n=><a key={n.path} href={n.path} aria-label={n.label} title={n.label} className={page===n.path?'nav-item active':'nav-item'} aria-current={page===n.path?'page':undefined} onClick={e=>{e.preventDefault();go(n.path);}}><n.icon size={20}/><span>{n.label}</span>{n.path==='/inbox'&&count>0&&<b>{count}</b>}</a>)}</nav>
+      <nav aria-label="Main navigation">{[{path:'/inbox',label:'Inbox',icon:Inbox},{path:'/reservations',label:'Reservations',icon:Building2},{path:'/payments',label:'Payments',icon:Banknote},{path:'/automation',label:'FAQ automation',icon:ShieldCheck},{path:'/knowledge',label:'Hotel knowledge',icon:BookOpen},{path:'/activity',label:'Activity',icon:ActivityIcon},{path:'/settings',label:'Settings',icon:Settings},{path:'/team',label:'Your team',icon:ShieldCheck},{path:'/setup',label:'Hotel setup',icon:CheckCheck},{path:'/health',label:'Workspace health',icon:ActivityIcon}].filter(n=>auth.user?.role==='Owner'||!['/team','/setup','/health'].includes(n.path)).map(n=><a key={n.path} href={n.path} aria-label={n.label} title={n.label} className={page===n.path?'nav-item active':'nav-item'} aria-current={page===n.path?'page':undefined} onClick={e=>{e.preventDefault();go(n.path);}}><n.icon size={20}/><span>{n.label}</span>{n.path==='/inbox'&&count>0&&<b>{count}</b>}</a>)}</nav>
       <div className="sidebar-bottom"><div className="mode-card"><ShieldCheck size={20}/><strong>You're in control</strong><p>Your team controls approved answers and reply automation.</p></div><div className="profile"><span className="avatar profile-avatar">{initials(auth.user.name)}</span><div><strong>{auth.user.name}</strong><small>{auth.user.role==='Owner'?'Hotel owner':'Team member'}</small></div><button className="icon-button" onClick={logout} disabled={busy} aria-label="Sign out"><LogOut size={18}/></button></div></div>
     </aside>
     <main className="main">
-      <header className="topbar"><span><span className="breadcrumb">Workspace</span><ChevronRight size={14}/>{page==='/inbox'?'Inbox':page==='/knowledge'?'Hotel knowledge':page==='/activity'?'Activity':page==='/reservations'?'Reservations':page==='/payments'?'Payments':page==='/automation'?'FAQ automation':page==='/team'?'Your team':page==='/setup'?'Hotel setup':'Settings'}</span><div className="topbar-right">{auth.preview&&<span className="preview-pill">Preview · sample data</span>}<span className="draft-mode"><span/>{hotel?.autoReplyMode==='Live'?'FAQ auto-replies enabled':hotel?.staffSendingEnabled?'Staff-approved sending':'Draft-only mode'}</span><button className="icon-button" aria-label="Refresh workspace" disabled={busy} onClick={()=>run(refresh)}><RefreshCw size={17}/></button></div></header>
+      <header className="topbar"><span><span className="breadcrumb">Workspace</span><ChevronRight size={14}/>{page==='/inbox'?'Inbox':page==='/knowledge'?'Hotel knowledge':page==='/activity'?'Activity':page==='/reservations'?'Reservations':page==='/payments'?'Payments':page==='/automation'?'FAQ automation':page==='/team'?'Your team':page==='/setup'?'Hotel setup':page==='/health'?'Workspace health':'Settings'}</span><div className="topbar-right">{auth.preview&&<span className="preview-pill">Preview · sample data</span>}<span className="draft-mode"><span/>{hotel?.autoReplyMode==='Live'?'FAQ auto-replies enabled':hotel?.staffSendingEnabled?'Staff-approved sending':'Draft-only mode'}</span><button className="icon-button" aria-label="Refresh workspace" disabled={busy} onClick={()=>run(refresh)}><RefreshCw size={17}/></button></div></header>
       {errorBox}{notice&&<div className="toast" role="status"><Check size={17}/>{notice}</div>}
-      {!loaded?<div className="loading"><p>Loading your hotel…</p></div>:page==='/team'?<TeamPage owner={auth.user.role==='Owner'}/>:page==='/setup'?<OnboardingPage owner={auth.user.role==='Owner'} go={go}/>:page==='/inbox'?<InboxPage hotel={hotel!} mailboxes={mailboxes} conversations={conversations} knowledge={knowledge} busy={busy} run={run} onUpdate={c=>setConversations(old=>old.map(x=>x.id===c.id?c:x))} notify={setNotice} go={go}/>:page==='/automation'?<AutomationPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/payments'?<PaymentsPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/reservations'?<PmsPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/knowledge'?<KnowledgePage items={knowledge} canEdit={auth.user.role==='Owner'} busy={busy} run={run} onUpdate={item=>setKnowledge(old=>old.some(x=>x.id===item.id)?old.map(x=>x.id===item.id?item:x):[...old,item])} notify={setNotice}/>:page==='/activity'?<div className="page"><PageHeading eyebrow="A clear record" title="Workspace activity" text="Changes made by your team, in one place."/><div className="activity-list">{activity.length?activity.map(a=><div className="activity-row" key={a.id}><span className="activity-icon"><Check size={18}/></span><div><strong>{a.action}</strong><p>{a.userName}</p></div><time>{date(a.at)}</time></div>):<Empty title="No activity yet" text="Changes to your workspace will appear here."/>}</div></div>:<SettingsPage onMailboxes={setMailboxes} hotel={hotel!} mailboxes={mailboxes} preview={auth.preview} owner={auth.user.role==='Owner'} busy={busy} run={run} onSave={h=>{setHotel(h);setNotice('Hotel settings saved.');}}/>}
+      {!loaded?<div className="loading"><p>Loading your hotel…</p></div>:page==='/health'?<OperationsPage owner={auth.user.role==='Owner'} go={go}/>:page==='/team'?<TeamPage owner={auth.user.role==='Owner'}/>:page==='/setup'?<OnboardingPage owner={auth.user.role==='Owner'} go={go}/>:page==='/inbox'?<InboxPage hotel={hotel!} mailboxes={mailboxes} conversations={conversations} knowledge={knowledge} busy={busy} run={run} onUpdate={c=>setConversations(old=>old.map(x=>x.id===c.id?c:x))} notify={setNotice} go={go}/>:page==='/automation'?<AutomationPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/payments'?<PaymentsPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/reservations'?<PmsPage hotel={hotel!} owner={auth.user.role==='Owner'} busy={busy} run={run} onHotel={setHotel}/>:page==='/knowledge'?<KnowledgePage items={knowledge} canEdit={auth.user.role==='Owner'} busy={busy} run={run} onUpdate={item=>setKnowledge(old=>old.some(x=>x.id===item.id)?old.map(x=>x.id===item.id?item:x):[...old,item])} notify={setNotice}/>:page==='/activity'?<div className="page"><PageHeading eyebrow="A clear record" title="Workspace activity" text="Changes made by your team, in one place."/><div className="activity-list">{activity.length?activity.map(a=><div className="activity-row" key={a.id}><span className="activity-icon"><Check size={18}/></span><div><strong>{a.action}</strong><p>{a.userName}</p></div><time>{date(a.at)}</time></div>):<Empty title="No activity yet" text="Changes to your workspace will appear here."/>}</div></div>:<SettingsPage onMailboxes={setMailboxes} hotel={hotel!} mailboxes={mailboxes} preview={auth.preview} owner={auth.user.role==='Owner'} busy={busy} run={run} onSave={h=>{setHotel(h);setNotice('Hotel settings saved.');}}/>}
     </main>
   </div>;
 }
@@ -87,3 +88,4 @@ function SettingsPage({hotel,mailboxes,preview,owner,busy,run,onSave,onMailboxes
 }
 createRoot(document.getElementById('root')!).render(<App/>);
 
+