Add Debian persistence acceptance drill

MILESTONES.md

@@ -8,6 +8,7 @@ This is the working delivery tracker for GuestOps Web. Update a milestone when i
 ## Status key
 
 - **Implemented** — present on `main` and supported by code or automated-test evidence.
+- **In progress** — repository or environment work has started but an exit condition remains open.
 - **Acceptance required** — implemented in code but still requires a real provider, Debian host, or operational exercise.
 - **Planned** — work is not yet complete.
 - **Deferred** — intentionally outside the current release gate.
@@ -33,7 +34,7 @@ This is the working delivery tracker for GuestOps Web. Update a milestone when i
 | 7 | Google connection recovery | B | Implemented / acceptance required | Connection epochs, checkpoint recovery, and revocation handling are promoted to local `main`; complete real Google acceptance and worker-restart exercises. |
 | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. |
 | 9 | Gitea and reproducible releases | A | In progress | The reviewed candidate is promoted in the local `main` history. CI now records the full commit, matched application version, archive checksum and immutable image IDs, and the rollback procedure is documented. Push the merge, retain the successful release evidence off-host, and create a new immutable approval tag; the existing `0.1.0` tag remains attached to the original foundation release. |
-| 10 | Debian deployment and persistence | A | Planned | Provision the target host, HTTPS and reverse proxy; persist MongoDB, data-protection keys, logs, and configuration; then verify restart and upgrade behaviour. |
+| 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. |
 | 11 | Backups, monitoring, and recovery | A | Planned | Schedule backups, define alerts and ownership, prove off-host retention, and perform a timed restore and recovery drill. |
 | 12 | Google mailbox and reviewed-reply acceptance | B | Planned | Complete OAuth verification, import/send acceptance, reconnect/revocation tests, identity-change handling, and duplicate/uncertain-send drills with a sandbox mailbox. |
 | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. |

RELEASE_NOTES.md

@@ -9,7 +9,7 @@ The reviewed candidate is now promoted into the local `main` history. It is not
 - AI-assisted reply suggestions and staff-reviewed Gmail sending.
 - Approval-controlled OHIP PMS and NMI payment workflows.
 - FAQ automation controls, team invitations, password recovery, and stronger Google connection recovery.
-- Backup, restore, deployment, diagnostic, release-evidence and rollback tooling.
+- Backup, restore, deployment, persistence-drill, diagnostic, release-evidence and rollback tooling.
 
 These capabilities still require their separately documented provider, host and operational acceptance. Google, PMS and payment-provider acceptance is not established by local automated tests.
 

deploy/ops.py

@@ -55,6 +55,27 @@ def provider_configured(config, target):
     return config not in ({}, {section: {"Hotels": {}}})
 
 
+def persistence_layout(config):
+    """Return the resolved named volumes required to survive recreation."""
+    services = config["services"]
+    declared = config.get("volumes", {})
+
+    def volume_for(service, target):
+        matches = [v for v in services[service].get("volumes", []) if v["target"] == target]
+        require(len(matches) == 1 and matches[0]["type"] == "volume" and matches[0].get("source"),
+                f"{service} requires one named persistent volume at {target}.")
+        source = matches[0]["source"]
+        require(source in declared, f"{service} volume {source} must be declared at the top level.")
+        return declared[source].get("name") or source
+
+    api_keys = volume_for("api", "/var/lib/guestops/keys")
+    worker_keys = volume_for("worker", "/var/lib/guestops/keys")
+    require(api_keys == worker_keys, "API and worker key volumes differ.")
+    mongo_data = volume_for("mongo", "/data/db")
+    require(api_keys != mongo_data, "Database and key data require separate named volumes.")
+    return {"keys": api_keys, "database": mongo_data}
+
+
 AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);'
 INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));'
 
@@ -78,15 +99,60 @@ def configuration():
         require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.")
     require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.")
     require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.")
-    key_sources = []
-    for name in ("api", "worker"):
-        keys = [v for v in services[name].get("volumes", []) if v["target"] == "/var/lib/guestops/keys"]
-        require(len(keys) == 1 and keys[0]["type"] == "volume", "API and worker require a shared persistent key volume.")
-        key_sources.append(keys[0]["source"])
-    require(key_sources[0] == key_sources[1], "API and worker key volumes differ.")
+    persistence_layout(config)
     return config
 
 
+def wait_for(action, message, attempts=60):
+    for attempt in range(attempts):
+        try:
+            action()
+            return
+        except Exception:
+            if attempt == attempts - 1:
+                raise RuntimeError(message)
+            time.sleep(1)
+
+
+def volume_identity(name):
+    require(not name.startswith("-"), "Invalid volume name.")
+    details = json.loads(run(["docker", "volume", "inspect", name]))
+    require(len(details) == 1 and details[0].get("Name") == name, "Named volume inspection failed.")
+    return {"name": name, "driver": details[0].get("Driver"), "scope": details[0].get("Scope")}
+
+
+def readiness():
+    with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=10) as response:
+        require(json.load(response) == {"status": "ready"}, "Loopback readiness failed.")
+
+
+def persistence_drill(args):
+    require(args.confirm_restart, "Persistence drill requires --confirm-restart: all services will be restarted and application containers recreated.")
+    config = configuration()
+    layout = persistence_layout(config)
+    before_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
+    before_inventory = json.loads(mongo(INVENTORY))
+    probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
+    require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
+
+    # Restart the database first, then its clients, so recovery is exercised in a known order.
+    compose("restart", "-t", "150", "mongo")
+    wait_for(lambda: mongo('const r=c.getDB("admin").runCommand({ping:1});if(!r.ok)quit(1);'), "MongoDB did not recover after restart.")
+    compose("restart", "-t", "150", "api", "worker")
+    wait_for(readiness, "API readiness did not recover after restart.")
+
+    # Recreate stateless containers as an image upgrade would, without rebuilding or changing data volumes.
+    compose("up", "-d", "--no-build", "--force-recreate", "api", "worker")
+    wait_for(readiness, "API readiness did not recover after container recreation.")
+    compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + probe, "api", "--verify-backup-probe")
+
+    after_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
+    after_inventory = json.loads(mongo(INVENTORY))
+    require(after_volumes == before_volumes, "A persistent volume identity changed during the drill.")
+    require(after_inventory["collections"] == before_inventory["collections"], "Database collection counts or indexes changed during the drill.")
+    print("Persistence drill passed: named volumes, database counts/indexes, data-protection keys and readiness survived restart and container recreation.")
+
+
 def preflight(args):
     config = configuration()
     env_file = ROOT / ".env"
@@ -242,10 +308,11 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__)
     subs = parser.add_subparsers(dest="command", required=True)
     check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
+    persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true")
     save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
     drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
     args = parser.parse_args()
-    {"preflight": preflight, "backup": backup, "restore-drill": restore_drill}[args.command](args)
+    {"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "restore-drill": restore_drill}[args.command](args)
 
 
 if __name__ == "__main__":

docs/deployment.md

@@ -73,3 +73,17 @@ A multi-hotel production launch using Gmail restricted scopes requires planning
 Verify separate hotels cannot read or edit each other's records; save and reload settings; restart services and confirm persistence; import test messages twice without duplicates; check the worker resumes a paginated import; confirm no mail is sent without explicit staff approval and that default-disabled sending remains blocked. Review the activity log and Google account used by the connection.
 
 Keep reviewed image IDs and release images for rollback and backups of both MongoDB and the key volume. Do not remove named volumes to fix application errors. The initial release has no automatic schema migration that destroys data. Use the [operational preflight, encrypted backup and isolated restore drill](operations.md), and establish retention and off-server copies before importing real guest data. `/health/ready` checks database reachability; the owner's Workspace health page also reports worker heartbeat and mailbox/reply exceptions.
+
+Before accepting the host, run the confirmation-gated persistence drill during an announced maintenance window:
+
+```sh
+python3 deploy/ops.py preflight
+python3 deploy/ops.py persistence-drill --confirm-restart
+python3 deploy/ops.py preflight
+```
+
+The drill restarts MongoDB, API and worker, then force-recreates the stateless application containers using the already selected images. It verifies that the resolved database and key volumes keep the same identities, MongoDB collection counts and indexes remain unchanged, a value protected before restart can still be decrypted, and loopback readiness recovers. It does not alter provider feature flags, upgrade images, validate public TLS, or replace the separate backup/restore drill.
+
+Record the host, operator, start/end time, release record checksum, resolved image IDs, preflight output and drill result in the deployment acceptance record. Also verify Docker starts at boot and perform a controlled Debian reboot before Gate A approval. After reboot, run the online preflight and inspect the Workspace health page; do not infer worker health solely from API readiness.
+
+The supplied Docker `json-file` logs are size-capped to protect the small pilot disk, but container recreation removes that container's local log history. Before host acceptance, route GuestOps and Nginx logs to the site's durable restricted logging system, or use a reviewed Docker logging override backed by persistent systemd journal storage. Prove that operators can retrieve pre-recreation logs without exposing request credentials or OAuth callback query strings. Central retention and alerting are completed under milestone 11.

tests/test_ops.py

@@ -16,6 +16,65 @@ spec.loader.exec_module(ops)
 
 
 class ArchiveTests(unittest.TestCase):
+    def test_persistence_layout_resolves_shared_keys_and_database(self):
+        config = {
+            "services": {
+                "api": {"volumes": [{"type": "volume", "source": "app-keys", "target": "/var/lib/guestops/keys"}]},
+                "worker": {"volumes": [{"type": "volume", "source": "app-keys", "target": "/var/lib/guestops/keys"}]},
+                "mongo": {"volumes": [{"type": "volume", "source": "mongo-data", "target": "/data/db"}]},
+            },
+            "volumes": {
+                "app-keys": {"name": "guestops_app-keys"},
+                "mongo-data": {"name": "guestops_mongo-data"},
+            },
+        }
+        self.assertEqual(ops.persistence_layout(config), {"keys": "guestops_app-keys", "database": "guestops_mongo-data"})
+
+    def test_persistence_layout_rejects_anonymous_or_split_keys(self):
+        config = {
+            "services": {
+                "api": {"volumes": [{"type": "volume", "source": "api-keys", "target": "/var/lib/guestops/keys"}]},
+                "worker": {"volumes": [{"type": "volume", "source": "worker-keys", "target": "/var/lib/guestops/keys"}]},
+                "mongo": {"volumes": [{"type": "volume", "source": "mongo-data", "target": "/data/db"}]},
+            },
+            "volumes": {"api-keys": {}, "worker-keys": {}, "mongo-data": {}},
+        }
+        with self.assertRaisesRegex(RuntimeError, "key volumes differ"):
+            ops.persistence_layout(config)
+
+        config["services"]["worker"]["volumes"][0] = {"type": "volume", "target": "/var/lib/guestops/keys"}
+        with self.assertRaisesRegex(RuntimeError, "named persistent volume"):
+            ops.persistence_layout(config)
+
+    def test_persistence_drill_requires_explicit_restart_confirmation(self):
+        with self.assertRaisesRegex(RuntimeError, "confirm-restart"):
+            ops.persistence_drill(type("Args", (), {"confirm_restart": False})())
+
+    def test_persistence_drill_restarts_then_recreates_stateless_services(self):
+        compose_calls = []
+        inventory = json.dumps({"bytes": 1, "collections": {"hotels": {"count": 1, "indexes": []}}}).encode()
+
+        def compose(*args, **kwargs):
+            compose_calls.append(args)
+            return b"a" * 30 if args[-1] == "--backup-probe" else b""
+
+        def mongo(script):
+            return inventory if "storageSize" in script else b""
+
+        with patch.object(ops, "configuration", return_value={}), \
+             patch.object(ops, "persistence_layout", return_value={"keys": "keys", "database": "data"}), \
+             patch.object(ops, "volume_identity", side_effect=lambda name: {"name": name, "driver": "local", "scope": "local"}), \
+             patch.object(ops, "compose", side_effect=compose), \
+             patch.object(ops, "mongo", side_effect=mongo), \
+             patch.object(ops, "wait_for", side_effect=lambda action, message: action()), \
+             patch.object(ops, "readiness"):
+            ops.persistence_drill(type("Args", (), {"confirm_restart": True})())
+
+        self.assertIn(("restart", "-t", "150", "mongo"), compose_calls)
+        self.assertIn(("restart", "-t", "150", "api", "worker"), compose_calls)
+        self.assertIn(("up", "-d", "--no-build", "--force-recreate", "api", "worker"), compose_calls)
+        self.assertTrue(any("--verify-backup-probe" in call for call in compose_calls))
+
     def test_empty_provider_templates_are_not_secret_configuration(self):
         for section, target in (("Pms", "/run/guestops/pms.json"), ("Payments", "/run/guestops/payments.json")):
             self.assertFalse(ops.provider_configured({section: {"Hotels": {}}}, target))