Add scheduled encrypted backup operations

MILESTONES.md

@@ -35,7 +35,7 @@ This is the working delivery tracker for GuestOps Web. Update a milestone when i
 | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. |
 | 9 | Gitea and reproducible releases | A | In progress | The reviewed candidate is promoted in the local `main` history. CI now records the full commit, matched application version, archive checksum and immutable image IDs, and the rollback procedure is documented. Push the merge, retain the successful release evidence off-host, and create a new immutable approval tag; the existing `0.1.0` tag remains attached to the original foundation release. |
 | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. |
-| 11 | Backups, monitoring, and recovery | A | Planned | Schedule backups, define alerts and ownership, prove off-host retention, and perform a timed restore and recovery drill. |
+| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling without command-line secrets. Install and test it on Debian, configure monitored off-host transfer and durable logs, name alert/retention owners, and retain evidence from a timed restore and recovery drill. |
 | 12 | Google mailbox and reviewed-reply acceptance | B | Planned | Complete OAuth verification, import/send acceptance, reconnect/revocation tests, identity-change handling, and duplicate/uncertain-send drills with a sandbox mailbox. |
 | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. |
 | 14 | Payment links and status | C | Planned | Select/confirm the payment-provider path, complete sandbox and webhook acceptance, and prove expiry, replay protection, reconciliation, and support recovery. |

RELEASE_NOTES.md

@@ -9,7 +9,7 @@ The reviewed candidate is now promoted into the local `main` history. It is not
 - AI-assisted reply suggestions and staff-reviewed Gmail sending.
 - Approval-controlled OHIP PMS and NMI payment workflows.
 - FAQ automation controls, team invitations, password recovery, and stronger Google connection recovery.
-- Backup, restore, deployment, persistence-drill, diagnostic, release-evidence and rollback tooling.
+- Backup, restore, opt-in systemd scheduling, deployment, persistence-drill, diagnostic, release-evidence and rollback tooling.
 
 These capabilities still require their separately documented provider, host and operational acceptance. Google, PMS and payment-provider acceptance is not established by local automated tests.
 

deploy/ops.py

@@ -244,6 +244,24 @@ def backup(args):
     print("Encrypted backup complete. Copy it off-server and perform a restore drill.")
 
 
+def scheduled_backup(args):
+    require(args.confirm_maintenance, "Scheduled backup requires --confirm-maintenance because API and workers will briefly stop.")
+    recipient = os.environ.get("BACKUP_RECIPIENT", "")
+    directory_value = os.environ.get("BACKUP_DIRECTORY", "")
+    require(re.fullmatch(r"[A-Fa-f0-9]{40}", recipient), "BACKUP_RECIPIENT must be a verified full GPG fingerprint.")
+    require(directory_value != "", "BACKUP_DIRECTORY must name the private off-checkout staging directory.")
+    requested_directory = Path(directory_value)
+    require(requested_directory.is_absolute() and not requested_directory.is_symlink(), "BACKUP_DIRECTORY must be an absolute, non-symlink path.")
+    directory = requested_directory.resolve()
+    require(directory.is_dir(), "BACKUP_DIRECTORY must be an existing real directory.")
+    require(ROOT not in directory.parents and directory != ROOT, "Scheduled backups must be staged outside the application checkout.")
+    require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0, "BACKUP_DIRECTORY must be private (mode 700).")
+    timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
+    destination = directory / f"guestops-{timestamp}.tar.gpg"
+    backup(argparse.Namespace(confirm_maintenance=True, recipient=recipient, output=str(destination)))
+    print(f"Scheduled backup staged as {destination.name}. Off-host transfer and alert verification remain required.")
+
+
 def unpack(bundle, folder):
     with tarfile.open(bundle, "r:") as archive:
         members = archive.getmembers()
@@ -310,9 +328,10 @@ def main():
     check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
     persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true")
     save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
+    scheduled = subs.add_parser("scheduled-backup"); scheduled.add_argument("--confirm-maintenance", action="store_true")
     drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
     args = parser.parse_args()
-    {"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "restore-drill": restore_drill}[args.command](args)
+    {"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "scheduled-backup": scheduled_backup, "restore-drill": restore_drill}[args.command](args)
 
 
 if __name__ == "__main__":

deploy/systemd/guestops-backup.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=GuestOps encrypted maintenance backup
+Requires=docker.service
+After=docker.service
+ConditionPathIsDirectory=/srv/guestops
+ConditionPathIsDirectory=/var/backups/guestops
+ConditionPathIsDirectory=/var/lib/guestops-backup/gnupg
+
+[Service]
+Type=oneshot
+WorkingDirectory=/srv/guestops
+EnvironmentFile=/etc/guestops/backup.env
+Environment=GNUPGHOME=/var/lib/guestops-backup/gnupg
+UMask=0077
+ExecStart=/usr/bin/python3 /srv/guestops/deploy/ops.py scheduled-backup --confirm-maintenance
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=full
+ReadWritePaths=/var/backups/guestops /var/lib/guestops-backup/gnupg
+TimeoutStartSec=45min

deploy/systemd/guestops-backup.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run the GuestOps encrypted backup each day
+
+[Timer]
+OnCalendar=*-*-* 02:17:00 UTC
+RandomizedDelaySec=30min
+Persistent=true
+Unit=guestops-backup.service
+
+[Install]
+WantedBy=timers.target

docs/operations.md

@@ -57,6 +57,40 @@ No other application or administrator may write to this database during the snap
 
 Copy the encrypted file off-server to restricted storage after every successful backup. Retain the exact API, worker and MongoDB images with the release: an image tag alone can change, and the drill requires matching image IDs. Agree the backup schedule, retention and tolerated data loss before a hotel pilot. Scheduling, off-server transfer, deletion and alerting are operator responsibilities in this milestone; none is silently installed.
 
+### Optional systemd schedule
+
+The repository includes an opt-in daily systemd service and timer. They are templates, not automatically installed. The service assumes the reviewed checkout is `/srv/guestops` and stages encrypted files in `/var/backups/guestops`; review and change both unit files together if the host uses different paths.
+
+Create the staging directory and environment file without storing a private key or passphrase on the server:
+
+```sh
+sudo install -d -m 700 /var/backups/guestops /var/lib/guestops-backup/gnupg /etc/guestops
+sudo install -m 600 /dev/null /etc/guestops/backup.env
+sudoedit /etc/guestops/backup.env
+sudo env GNUPGHOME=/var/lib/guestops-backup/gnupg gpg --import /secure/path/recovery-public.asc
+sudo env GNUPGHOME=/var/lib/guestops-backup/gnupg gpg --fingerprint
+```
+
+The file contains only these two settings. `BACKUP_RECIPIENT` is the verified 40-character public-key fingerprint already imported into the service account's GPG keyring.
+
+```text
+BACKUP_RECIPIENT=0123456789ABCDEF0123456789ABCDEF01234567
+BACKUP_DIRECTORY=/var/backups/guestops
+```
+
+Review the unit sandbox against the host, copy `deploy/systemd/guestops-backup.service` and `.timer` to `/etc/systemd/system`, then validate and test before enabling:
+
+```sh
+sudo systemd-analyze verify /etc/systemd/system/guestops-backup.service /etc/systemd/system/guestops-backup.timer
+sudo systemctl daemon-reload
+sudo systemctl start guestops-backup.service
+sudo systemctl status guestops-backup.service
+sudo systemctl enable --now guestops-backup.timer
+systemctl list-timers guestops-backup.timer
+```
+
+The timer deliberately causes the same brief maintenance interruption as a manual backup. `Persistent=true` runs a missed event after downtime, so choose and communicate the maintenance window. A successful unit only stages an encrypted file locally. Configure an independently monitored off-host transfer, verify the destination checksum, alert on both unit and transfer failure, and test the alert route. Do not add automatic deletion until retention, legal hold and recovery requirements have named owners.
+
 Temporary plaintext files are held in private directories and removed on normal completion or exceptions. Process termination or power loss can leave temporary data, stopped services or TTL expiry disabled. After an interrupted run, inspect the dedicated project and remove only its identified abandoned temporary directory after securing any recovery material. Restore the recorded TTL setting (normally true) and restart the services:
 
 ```sh

tests/test_ops.py

@@ -126,6 +126,36 @@ class ArchiveTests(unittest.TestCase):
         with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"):
             ops.backup(type("Args", (), {"confirm_maintenance": False})())
 
+    def test_scheduled_backup_requires_explicit_maintenance(self):
+        with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"):
+            ops.scheduled_backup(type("Args", (), {"confirm_maintenance": False})())
+
+    @unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
+    def test_scheduled_backup_uses_private_external_directory_and_environment(self):
+        with tempfile.TemporaryDirectory() as parent:
+            directory = Path(parent) / "backups"
+            directory.mkdir(mode=0o700)
+            captured = []
+            with patch.dict(os.environ, {"BACKUP_RECIPIENT": "A" * 40, "BACKUP_DIRECTORY": str(directory)}), \
+                 patch.object(ops, "ROOT", Path(parent) / "checkout"), \
+                 patch.object(ops, "backup", side_effect=lambda args: captured.append(args)), \
+                 patch.object(ops.time, "strftime", return_value="20260929T020000Z"):
+                ops.scheduled_backup(type("Args", (), {"confirm_maintenance": True})())
+            self.assertEqual(captured[0].recipient, "A" * 40)
+            self.assertEqual(Path(captured[0].output), directory / "guestops-20260929T020000Z.tar.gpg")
+            self.assertTrue(captured[0].confirm_maintenance)
+
+    @unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
+    def test_scheduled_backup_rejects_checkout_directory(self):
+        with tempfile.TemporaryDirectory() as parent:
+            checkout = Path(parent) / "checkout"
+            directory = checkout / "backups"
+            directory.mkdir(parents=True, mode=0o700)
+            with patch.dict(os.environ, {"BACKUP_RECIPIENT": "A" * 40, "BACKUP_DIRECTORY": str(directory)}), \
+                 patch.object(ops, "ROOT", checkout):
+                with self.assertRaisesRegex(RuntimeError, "outside the application checkout"):
+                    ops.scheduled_backup(type("Args", (), {"confirm_maintenance": True})())
+
     @unittest.skipUnless(os.name == "posix", "Linux file permission semantics")
     def test_dump_failure_restarts_services_and_ttl(self):
         with tempfile.TemporaryDirectory() as temp: