From a00fad5553cc278ecdcb5316caefa5574092683e Mon Sep 17 00:00:00 2001 From: wolf-demon Date: Wed, 30 Sep 2026 11:39:51 +0100 Subject: [PATCH 01/11] milestone 9 completed --- deploy/verify_release.py | 118 +++++++++++++++++++++++++++++++++++ docs/operations.md | 22 +++++++ tests/test_verify_release.py | 87 ++++++++++++++++++++++++++ 3 files changed, 227 insertions(+) create mode 100644 deploy/verify_release.py create mode 100644 tests/test_verify_release.py diff --git a/deploy/verify_release.py b/deploy/verify_release.py new file mode 100644 index 0000000..597b26b --- /dev/null +++ b/deploy/verify_release.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Verify a GuestOps release archive and its immutable CI release record.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + + +SHA = re.compile(r"[0-9a-f]{40}") +DIGEST = re.compile(r"sha256:[0-9a-f]{64}") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def loaded_image_id(reference: str) -> str: + result = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", reference], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def validate( + archive: Path, + record_path: Path, + expected_commit: str, + expected_version: str, + verify_loaded_images: bool = False, +) -> dict[str, object]: + require(archive.is_file(), "Release archive does not exist.") + require(record_path.is_file(), "Release record does not exist.") + require(SHA.fullmatch(expected_commit) is not None, "Expected commit must be a full lowercase Git SHA.") + require(re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", expected_version) is not None, + "Expected version must use MAJOR.MINOR.PATCH.") + + record = json.loads(record_path.read_text(encoding="utf-8")) + require(isinstance(record, dict), "Release record must be a JSON object.") + require(record.get("schemaVersion") == 1, "Unsupported release-record schema.") + require(record.get("commit") == expected_commit, "Release-record commit does not match the approved candidate.") + require(record.get("version") == expected_version, "Release-record version does not match the approved version.") + + artifact = record.get("artifact") + require(isinstance(artifact, dict), "Release record has no artifact object.") + require(artifact.get("name") == archive.name, "Release archive filename does not match the record.") + require(artifact.get("size") == archive.stat().st_size, "Release archive size does not match the record.") + archive_sha = sha256(archive) + require(artifact.get("sha256") == archive_sha, "Release archive SHA-256 does not match the record.") + + images = record.get("images") + require(isinstance(images, dict) and set(images) == {"api", "worker"}, + "Release record must contain exactly API and worker images.") + expected_references = { + "api": f"guestops-api:{expected_commit}", + "worker": f"guestops-worker:{expected_commit}", + } + verified_images: dict[str, dict[str, str]] = {} + for name, reference in expected_references.items(): + image = images.get(name) + require(isinstance(image, dict), f"Release record has no {name} image object.") + require(image.get("reference") == reference, f"{name} image reference is not bound to the full candidate SHA.") + image_id = str(image.get("id", "")) + require(DIGEST.fullmatch(image_id) is not None, f"{name} image ID is not an immutable SHA-256 digest.") + if verify_loaded_images: + require(loaded_image_id(reference) == image_id, f"Loaded {name} image ID does not match the release record.") + verified_images[name] = {"reference": reference, "id": image_id} + + return { + "schemaVersion": 1, + "verified": True, + "commit": expected_commit, + "version": expected_version, + "archive": {"name": archive.name, "size": archive.stat().st_size, "sha256": archive_sha}, + "releaseRecord": {"name": record_path.name, "sha256": sha256(record_path)}, + "images": verified_images, + "loadedImageIdsVerified": verify_loaded_images, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive", required=True, type=Path) + parser.add_argument("--record", required=True, type=Path) + parser.add_argument("--commit", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--verify-loaded-images", action="store_true") + parser.add_argument("--output", type=Path, help="Optional path for the non-sensitive verification summary.") + args = parser.parse_args() + result = validate(args.archive, args.record, args.commit, args.version, args.verify_loaded_images) + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + print(rendered, end="") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, json.JSONDecodeError, subprocess.SubprocessError) as error: + print(f"Release verification failed: {error}", file=__import__("sys").stderr) + raise SystemExit(1) diff --git a/docs/operations.md b/docs/operations.md index f15c168..9cc3f1d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -18,6 +18,28 @@ print(r['commit'], r['version'], r['images']) PY ``` +The repository verifier performs the same checks strictly, also calculates the release-record checksum used by later acceptance records, and can compare the record with images already loaded on an isolated Docker host: + +```sh +python3 deploy/verify_release.py \ + --archive guestops-images.tar.gz \ + --record release-record.json \ + --commit a3ef408de61d895e69516fa3ba014b43621032bc \ + --version 0.2.0 \ + --output release-verification.json + +gunzip -c guestops-images.tar.gz | docker load +python3 deploy/verify_release.py \ + --archive guestops-images.tar.gz \ + --record release-record.json \ + --commit a3ef408de61d895e69516fa3ba014b43621032bc \ + --version 0.2.0 \ + --verify-loaded-images \ + --output loaded-image-verification.json +``` + +Retain both verification summaries with the untouched archive, release record, its SHA-256, and the exact default-branch CI metadata. A waiting, cancelled, failed, or branch-only run is not release evidence. Do not substitute a local rebuild for the archived default-branch images. + Load the archive, verify each loaded image ID matches the record, set `GUESTOPS_API_IMAGE` and `GUESTOPS_WORKER_IMAGE` to the recorded full-SHA references, and run the deployment preflight. Record the CI run, commit, checksum and operator in the change ticket. A release tag is an approval marker; do not move or reuse an existing tag. The application and web versions must match before the record can be created. For rollback, first disable worker-driven external writes and reconcile any sending, payment or PMS operation that may have completed since the prior release. Confirm the previous release archive and record are retained, verify its checksum and image IDs, take an encrypted backup, then select the previous recorded image references in `.env` and recreate only the API and worker. Do not roll back MongoDB or the key volume merely to change application images. Run the online preflight, readiness check and read-only smoke test before re-enabling the worker or provider writes. If a release introduced an incompatible data change, follow its release-specific recovery plan rather than starting an older image against newer data. diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py new file mode 100644 index 0000000..0286f70 --- /dev/null +++ b/tests/test_verify_release.py @@ -0,0 +1,87 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("verify_release", ROOT / "deploy" / "verify_release.py") +verify_release = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verify_release) + + +COMMIT = "a" * 40 +API_ID = "sha256:" + "b" * 64 +WORKER_ID = "sha256:" + "c" * 64 + + +class VerifyReleaseTests(unittest.TestCase): + def fixture(self, directory: str): + root = Path(directory) + archive = root / "guestops-images.tar.gz" + record = root / "release-record.json" + archive.write_bytes(b"reviewed image archive") + release = { + "schemaVersion": 1, + "version": "0.2.0", + "commit": COMMIT, + "artifact": { + "name": archive.name, + "size": archive.stat().st_size, + "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + }, + "images": { + "api": {"reference": f"guestops-api:{COMMIT}", "id": API_ID}, + "worker": {"reference": f"guestops-worker:{COMMIT}", "id": WORKER_ID}, + }, + } + record.write_text(json.dumps(release), encoding="utf-8") + return archive, record, release + + def test_verifies_candidate_archive_record_and_record_checksum(self): + with tempfile.TemporaryDirectory() as directory: + archive, record, _ = self.fixture(directory) + result = verify_release.validate(archive, record, COMMIT, "0.2.0") + self.assertTrue(result["verified"]) + self.assertEqual(result["archive"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) + self.assertEqual(result["releaseRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest()) + self.assertFalse(result["loadedImageIdsVerified"]) + + def test_rejects_changed_archive(self): + with tempfile.TemporaryDirectory() as directory: + archive, record, _ = self.fixture(directory) + archive.write_bytes(b"changed") + with self.assertRaisesRegex(ValueError, "size does not match"): + verify_release.validate(archive, record, COMMIT, "0.2.0") + + def test_rejects_wrong_commit_version_reference_and_mutable_id(self): + cases = [ + (lambda value: value.update(commit="d" * 40), "commit does not match"), + (lambda value: value.update(version="0.3.0"), "version does not match"), + (lambda value: value["images"]["api"].update(reference="guestops-api:latest"), "full candidate SHA"), + (lambda value: value["images"]["worker"].update(id="worker-image"), "immutable SHA-256"), + ] + for mutate, message in cases: + with self.subTest(message=message), tempfile.TemporaryDirectory() as directory: + archive, record, release = self.fixture(directory) + mutate(release) + record.write_text(json.dumps(release), encoding="utf-8") + with self.assertRaisesRegex(ValueError, message): + verify_release.validate(archive, record, COMMIT, "0.2.0") + + def test_loaded_image_ids_must_match(self): + with tempfile.TemporaryDirectory() as directory: + archive, record, _ = self.fixture(directory) + with patch.object(verify_release, "loaded_image_id", side_effect=[API_ID, WORKER_ID]): + result = verify_release.validate(archive, record, COMMIT, "0.2.0", True) + self.assertTrue(result["loadedImageIdsVerified"]) + with patch.object(verify_release, "loaded_image_id", return_value="sha256:" + "d" * 64): + with self.assertRaisesRegex(ValueError, "Loaded api image ID"): + verify_release.validate(archive, record, COMMIT, "0.2.0", True) + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From d7e19bd9dac8eb24f3a8489cb055b6d71ebfd0ad Mon Sep 17 00:00:00 2001 From: wolf-demon Date: Wed, 30 Sep 2026 15:17:34 +0100 Subject: [PATCH 02/11] Gate B release candidate --- MILESTONES.md | 30 ++- deploy/backup-restore-acceptance.example.json | 81 ++++++ deploy/backup_restore_acceptance.py | 220 ++++++++++++++++ deploy/backup_transfer.py | 238 ++++++++++++++++++ deploy/debian-host-acceptance.example.json | 45 ++++ deploy/debian_acceptance.py | 206 +++++++++++++++ deploy/monitor_status.py | 189 ++++++++++++++ deploy/persistence-acceptance.example.json | 31 +++ deploy/systemd/backup-transfer.env.example | 6 + deploy/systemd/backup.env.example | 2 + .../systemd/guestops-backup-transfer.service | 19 ++ deploy/systemd/guestops-backup-transfer.timer | 11 + .../systemd/guestops-monitor-status.service | 21 ++ deploy/systemd/guestops-monitor-status.timer | 11 + .../zabbix-agent-guestops.conf.example | 2 + docs/deployment.md | 12 + docs/operations.md | 53 ++++ tests/test_backup_restore_acceptance.py | 129 ++++++++++ tests/test_backup_transfer.py | 81 ++++++ tests/test_debian_acceptance.py | 113 +++++++++ tests/test_monitor_status.py | 75 ++++++ 21 files changed, 1573 insertions(+), 2 deletions(-) create mode 100644 deploy/backup-restore-acceptance.example.json create mode 100644 deploy/backup_restore_acceptance.py create mode 100644 deploy/backup_transfer.py create mode 100644 deploy/debian-host-acceptance.example.json create mode 100644 deploy/debian_acceptance.py create mode 100644 deploy/monitor_status.py create mode 100644 deploy/persistence-acceptance.example.json create mode 100644 deploy/systemd/backup-transfer.env.example create mode 100644 deploy/systemd/backup.env.example create mode 100644 deploy/systemd/guestops-backup-transfer.service create mode 100644 deploy/systemd/guestops-backup-transfer.timer create mode 100644 deploy/systemd/guestops-monitor-status.service create mode 100644 deploy/systemd/guestops-monitor-status.timer create mode 100644 deploy/systemd/zabbix-agent-guestops.conf.example create mode 100644 tests/test_backup_restore_acceptance.py create mode 100644 tests/test_backup_transfer.py create mode 100644 tests/test_debian_acceptance.py create mode 100644 tests/test_monitor_status.py diff --git a/MILESTONES.md b/MILESTONES.md index 960b844..e4f5e12 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -1,10 +1,36 @@ # GuestOps Milestone Report Version: **0.2.0 release candidate** -Last updated: **29 September 2026** +Last updated: **30 September 2026** This is the working delivery tracker for GuestOps Web. Update a milestone when its state changes and link the pull request, release artifact, test run, or acceptance record that proves the change. +## Milestones at a glance + +This summary explains what each milestone delivers and where it currently stands. The release-gate and delivery tables below contain the detailed evidence and exit conditions. + +| # | Milestone | What it delivers | Current position | +| ---: | --- | --- | --- | +| 1 | Web foundation | The React application, ASP.NET Core API, tenant-isolated MongoDB storage, preview mode, and container foundation. | **Implemented.** The application foundation and automated tests are on `main`. | +| 2 | AI suggestions and reviewed Gmail sending | AI-assisted reply drafts and staff-reviewed Gmail delivery with safety and duplicate-send controls. | **Implemented; acceptance required.** Real Gmail threading, revocation, uncertain-send, and staff-review scenarios still need live evidence. | +| 3 | OHIP PMS workflow | Internal proposal, approval, and execution controls for PMS operations. | **Implemented; acceptance required.** The workflow exists, but provider-contract and sandbox acceptance remain outstanding. | +| 4 | NMI payment workflow | Internal payment proposal, approval, status, expiry, and reconciliation controls. | **Implemented; acceptance required.** The workflow exists, but real payment-provider sandbox acceptance remains outstanding. | +| 5 | FAQ automation | Knowledge-based FAQ drafting, test mode, approval controls, and guarded live automation. | **Implemented; acceptance required.** Live mode remains disabled pending quality, false-positive, monitoring, and rollback acceptance. | +| 6 | Team onboarding and account recovery | Staff invitations, password setup/reset, access disable/restore, and administrator recovery. | **Implemented; acceptance required.** Deployed link delivery, expiry, recovery, and administrator procedures still need operational evidence. | +| 7 | Google connection recovery | OAuth reconnect, checkpoint recovery, grant revocation handling, and worker restart safety. | **Implemented; acceptance required.** Dedicated Google-account and worker-restart exercises have not yet been accepted. | +| 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | +| 9 | Gitea and reproducible releases | CI-built immutable images, checksummed release records, retained artifacts, and approval tagging. | **In progress.** The `0.2.0` candidate is frozen; successful exact-commit CI evidence and the post-Gate-B tag are still required. | +| 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** Acceptance tooling is ready, but Docker, correct HTTPS/network exposure, exact artifacts, privileged installation, and the supervised drills remain open. | +| 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** Repository tooling is ready; installation and timed operational evidence are blocked until Milestones 9 and 10 pass. | +| 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | +| 13 | Rezlynx/Guestline adapter | The real PMS provider adapter, mappings, idempotency, reconciliation, and ambiguous-write handling. | **Planned.** Provider contract and sandbox access are still required before implementation and acceptance. | +| 14 | Payment links and status | The real payment-provider integration, webhooks, expiry, replay protection, and reconciliation. | **Planned.** The provider path and sandbox acceptance plan still need to be confirmed and completed. | +| 15 | Knowledge, AI, and FAQ activation | Supervised knowledge-quality, AI-draft, FAQ test-mode, staff-training, and stop-control acceptance. | **Implemented; acceptance required.** The evaluation tooling exists; the supervised evaluation and independent approval remain outstanding. | +| 16 | Identity, preferences, and privacy | Account/session controls, hotel preferences, privacy inventory, retention decisions, and audit review. | **Implemented; acceptance required.** Legal and operational decisions, identity checks, and independent review remain outstanding. | +| 17 | Inbox usability and desktop parity | Stable pagination, protected unsaved drafts, hotel-timezone display, and desktop workflow parity. | **Implemented; acceptance required.** Automated checks pass; the supervised desktop exercise and independent approval remain outstanding. | +| 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** Validators and targets exist; Gate A/B prerequisites, capacity evidence, incident rehearsal, pilot, and named approvals remain open. | +| 19 | Account security and self-service | TOTP MFA, recovery codes, transactional email, granular roles, preferences, and security notifications. | **Implemented on the development branch; acceptance required.** Keep it separate until `0.2.0` is approved and tagged, then review, merge, and version it as `0.3.0`. | + ## Status key - **Implemented** — present on `main` and supported by code or automated-test evidence. @@ -35,7 +61,7 @@ This is the working delivery tracker for GuestOps Web. Update a milestone when i | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | | 9 | Gitea and reproducible releases | A | In progress | The `0.2.0` candidate is versioned on `main`. CI records the full commit, matched application version, archive checksum and immutable image IDs, and the rollback procedure is documented. Retain the successful default-branch evidence off-host and create the immutable approval tag only after Gate B approval; the existing `0.1.0` tag remains attached to the foundation release. | | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. | -| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling without command-line secrets. Install and test it on Debian, configure monitored off-host transfer and durable logs, name alert/retention owners, and retain evidence from a timed restore and recovery drill. | +| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling, checksum-verified rsync transfer, a restricted Zabbix status boundary, guarded local retention and a release-bound acceptance validator. Install and test it on Debian, configure the restricted store and alerts, name operational/review owners, and retain independently reviewed evidence from the timed restore and rollback drill. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. | | 14 | Payment links and status | C | Planned | Select/confirm the payment-provider path, complete sandbox and webhook acceptance, and prove expiry, replay protection, reconciliation, and support recovery. | diff --git a/deploy/backup-restore-acceptance.example.json b/deploy/backup-restore-acceptance.example.json new file mode 100644 index 0000000..ff32ff1 --- /dev/null +++ b/deploy/backup-restore-acceptance.example.json @@ -0,0 +1,81 @@ +{ + "schemaVersion": 1, + "system": "guestops-backup-recovery", + "evidenceId": "backup-restore", + "dataClassification": "synthetic-only", + "releaseVersion": "0.2.0", + "releaseCommit": "0000000000000000000000000000000000000000", + "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "environment": "https://sandbox-guestops.futuresens.co.uk", + "hostIdentifier": "guestops-sandbox-01", + "images": { + "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + "mongo": {"reference": "mongo:8.0", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} + }, + "owners": { + "backup": "REPLACE", + "monitoring": "REPLACE", + "recoveryOperator": "REPLACE", + "technicalEscalation": "REPLACE", + "retention": "REPLACE", + "independentReviewer": "REPLACE" + }, + "startedAt": "2026-10-01T09:00:00Z", + "endedAt": "2026-10-01T13:00:00Z", + "reviewedAt": "2026-10-01T14:00:00Z", + "recoveryObjectives": { + "targetRpoHours": 24, + "observedRpoHours": 25, + "targetRtoMinutes": 240, + "observedRtoMinutes": 241 + }, + "retention": { + "localVerifiedDays": 7, + "offHostDaily": 35, + "offHostMonthly": 12, + "legalHoldOverrideTested": false + }, + "backup": { + "createdAt": "2026-10-01T08:00:00Z", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "transferredSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "privateKeyPresentOnHost": false + }, + "rollback": { + "previousReleaseCommit": "1111111111111111111111111111111111111111", + "previousArchiveSha256": "1111111111111111111111111111111111111111111111111111111111111111", + "previousImages": { + "api": {"reference": "guestops-api:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"}, + "worker": {"reference": "guestops-worker:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"} + }, + "persistentVolumesReplaced": false, + "restoredReleaseCommit": "0000000000000000000000000000000000000000", + "unresolvedOperations": 1, + "finalControls": { + "googleSending": "disabled", + "faqLiveMode": "disabled", + "pmsWrites": "disabled", + "paymentCreation": "disabled" + } + }, + "monitoringState": "not-configured", + "unresolvedCriticalFindings": 1, + "scenarios": [ + {"id": "alert-escalation", "status": "not-run", "evidence": []}, + {"id": "atomic-off-host-transfer", "status": "not-run", "evidence": []}, + {"id": "controlled-return-to-service", "status": "not-run", "evidence": []}, + {"id": "data-protection-recovery", "status": "not-run", "evidence": []}, + {"id": "database-inventory", "status": "not-run", "evidence": []}, + {"id": "durable-secret-free-logs", "status": "not-run", "evidence": []}, + {"id": "encrypted-manual-backup", "status": "not-run", "evidence": []}, + {"id": "image-rollback", "status": "not-run", "evidence": []}, + {"id": "isolated-restore", "status": "not-run", "evidence": []}, + {"id": "monitoring-coverage", "status": "not-run", "evidence": []}, + {"id": "off-host-checksum", "status": "not-run", "evidence": []}, + {"id": "production-service-recovery", "status": "not-run", "evidence": []}, + {"id": "retention-and-legal-hold", "status": "not-run", "evidence": []}, + {"id": "scheduled-backup", "status": "not-run", "evidence": []} + ] +} diff --git a/deploy/backup_restore_acceptance.py b/deploy/backup_restore_acceptance.py new file mode 100644 index 0000000..2f76bc3 --- /dev/null +++ b/deploy/backup_restore_acceptance.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Validate a restricted GuestOps backup, monitoring and recovery record.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +from pathlib import Path +import re +from urllib.parse import urlparse + + +VERSION = "0.2.0" +SCENARIOS = { + "encrypted-manual-backup", + "scheduled-backup", + "production-service-recovery", + "atomic-off-host-transfer", + "off-host-checksum", + "retention-and-legal-hold", + "monitoring-coverage", + "alert-escalation", + "durable-secret-free-logs", + "isolated-restore", + "data-protection-recovery", + "database-inventory", + "image-rollback", + "controlled-return-to-service", +} +SHA256 = re.compile(r"[0-9a-f]{64}") +GIT_SHA = re.compile(r"[0-9a-f]{40}") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def timestamp(value: object, field: str) -> dt.datetime: + require(isinstance(value, str) and value.endswith("Z"), + f"{field} must be a UTC timestamp ending in Z.") + try: + parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + except ValueError as error: + raise ValueError(f"{field} is not a valid timestamp.") from error + require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.") + return parsed + + +def safe_name(value: object, field: str) -> str: + name = str(value or "").strip() + require(2 <= len(name) <= 120 and "@" not in name and "/" not in name and "\\" not in name, + f"{field} requires a name without an email address or path.") + return name + + +def validate(record: object, expected_commit: str, expected_release_sha256: str) -> None: + require(isinstance(record, dict), "Acceptance record must be a JSON object.") + require(record.get("schemaVersion") == 1, "Unsupported backup/recovery schema.") + require(record.get("system") == "guestops-backup-recovery", + "system must be guestops-backup-recovery.") + require(record.get("evidenceId") == "backup-restore", + "evidenceId must be backup-restore.") + require(record.get("dataClassification") == "synthetic-only", + "Recovery acceptance must use synthetic data only.") + require(record.get("releaseVersion") == VERSION, f"releaseVersion must be {VERSION}.") + require(GIT_SHA.fullmatch(str(expected_commit)) is not None, + "Expected release commit must be a full lowercase Git SHA.") + require(SHA256.fullmatch(str(expected_release_sha256)) is not None, + "Expected release-record checksum must be a lowercase SHA-256 digest.") + require(record.get("releaseCommit") == expected_commit, + "releaseCommit does not match the approved candidate.") + require(record.get("releaseRecordSha256") == expected_release_sha256, + "releaseRecordSha256 does not match the retained release record.") + require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None, + "archiveSha256 must be a lowercase SHA-256 digest.") + + origin = urlparse(str(record.get("environment", ""))) + require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/") + and not origin.query and not origin.fragment and origin.username is None + and origin.password is None, + "environment must be an HTTPS origin without credentials, path, query or fragment.") + safe_name(record.get("hostIdentifier"), "hostIdentifier") + + images = record.get("images") + require(isinstance(images, dict) and set(images) == {"api", "worker", "mongo"}, + "images must contain exactly api, worker and mongo.") + for name in ("api", "worker"): + image = images[name] + require(isinstance(image, dict) and set(image) == {"reference", "id"}, + f"images.{name} must contain exactly reference and id.") + require(image["reference"] == f"guestops-{name}:{expected_commit}", + f"images.{name}.reference must use the full approved commit.") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, + f"images.{name}.id must be immutable.") + mongo = images["mongo"] + require(isinstance(mongo, dict) and set(mongo) == {"reference", "id"} + and mongo["reference"] == "mongo:8.0" + and re.fullmatch(r"sha256:[0-9a-f]{64}", str(mongo["id"])) is not None, + "images.mongo must identify the immutable mongo:8.0 image.") + + owners = record.get("owners") + owner_keys = {"backup", "monitoring", "recoveryOperator", "technicalEscalation", + "retention", "independentReviewer"} + require(isinstance(owners, dict) and set(owners) == owner_keys, + "owners must contain the exact operational and review roles.") + names = {key: safe_name(value, f"owners.{key}") for key, value in owners.items()} + reviewer = names["independentReviewer"].casefold() + require(reviewer not in {names[key].casefold() for key in owner_keys - {"independentReviewer"}}, + "independentReviewer must be different from every operational owner.") + + started = timestamp(record.get("startedAt"), "startedAt") + ended = timestamp(record.get("endedAt"), "endedAt") + reviewed = timestamp(record.get("reviewedAt"), "reviewedAt") + require(started <= ended <= reviewed, "Acceptance timestamps are out of order.") + + recovery = record.get("recoveryObjectives") + require(isinstance(recovery, dict) and set(recovery) == { + "targetRpoHours", "observedRpoHours", "targetRtoMinutes", "observedRtoMinutes", + }, "recoveryObjectives must contain exact target and observed RPO/RTO values.") + for field in recovery: + require(isinstance(recovery[field], (int, float)) and not isinstance(recovery[field], bool) + and recovery[field] >= 0, f"recoveryObjectives.{field} must be non-negative.") + require(recovery["targetRpoHours"] == 24 and recovery["observedRpoHours"] <= 24, + "Observed RPO must meet the approved 24-hour target.") + require(recovery["targetRtoMinutes"] == 240 and recovery["observedRtoMinutes"] <= 240, + "Observed RTO must meet the approved four-hour target.") + + retention = record.get("retention") + require(retention == { + "localVerifiedDays": 7, + "offHostDaily": 35, + "offHostMonthly": 12, + "legalHoldOverrideTested": True, + }, "Retention must record seven local days, 35 daily and 12 monthly off-host copies, and legal-hold testing.") + + backup = record.get("backup") + require(isinstance(backup, dict) and set(backup) == { + "createdAt", "sha256", "transferredSha256", "privateKeyPresentOnHost", + }, "backup must contain exact creation, checksum, transfer and private-key fields.") + created = timestamp(backup["createdAt"], "backup.createdAt") + require(created <= started, "The accepted backup must exist when the timed exercise starts.") + require(abs(recovery["observedRpoHours"] - (started - created).total_seconds() / 3600) < 0.01, + "Observed RPO must match the backup and exercise timestamps.") + require(abs(recovery["observedRtoMinutes"] - (ended - started).total_seconds() / 60) < 0.01, + "Observed RTO must match the exercise timestamps.") + require(SHA256.fullmatch(str(backup["sha256"])) is not None + and backup["transferredSha256"] == backup["sha256"], + "Local and transferred backup checksums must match.") + require(backup["privateKeyPresentOnHost"] is False, + "The recovery private key must not be present on the Debian host.") + + rollback = record.get("rollback") + require(isinstance(rollback, dict) and set(rollback) == { + "previousReleaseCommit", "previousArchiveSha256", "previousImages", + "persistentVolumesReplaced", "restoredReleaseCommit", "unresolvedOperations", "finalControls", + }, "rollback must contain the exact rehearsal and final-state fields.") + require(GIT_SHA.fullmatch(str(rollback["previousReleaseCommit"])) is not None + and rollback["previousReleaseCommit"] != expected_commit, + "Rollback must use a different retained previous release.") + require(SHA256.fullmatch(str(rollback["previousArchiveSha256"])) is not None, + "Rollback requires the previous archive checksum.") + previous_images = rollback["previousImages"] + require(isinstance(previous_images, dict) and set(previous_images) == {"api", "worker"}, + "Rollback requires exact previous API and worker images.") + for name in ("api", "worker"): + image = previous_images[name] + require(isinstance(image, dict) and set(image) == {"reference", "id"} + and image["reference"] == f"guestops-{name}:{rollback['previousReleaseCommit']}" + and re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, + f"rollback.previousImages.{name} must use the retained previous release identity.") + require(rollback["persistentVolumesReplaced"] is False, + "Image rollback must not replace persistent volumes.") + require(rollback["restoredReleaseCommit"] == expected_commit, + "The exercise must finish on the approved candidate.") + require(rollback["unresolvedOperations"] == 0, + "The exercise must finish without unresolved operations.") + require(rollback["finalControls"] == { + "googleSending": "disabled", "faqLiveMode": "disabled", + "pmsWrites": "disabled", "paymentCreation": "disabled", + }, "The exercise must finish with all unaccepted external writes disabled.") + + scenarios = record.get("scenarios") + require(isinstance(scenarios, list), "scenarios must be a list.") + ids = [item.get("id") for item in scenarios if isinstance(item, dict)] + require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == SCENARIOS, + "Acceptance record requires the exact backup/recovery scenario set.") + for item in scenarios: + scenario_id = item["id"] + require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.") + evidence = item.get("evidence") + require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all( + isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value + and "\\" not in value and not value.startswith("/") for value in evidence + ), f"Scenario {scenario_id} requires safe opaque evidence references.") + require(record.get("monitoringState") == "healthy", + "Monitoring must be healthy at acceptance completion.") + require(record.get("unresolvedCriticalFindings") == 0, + "Acceptance cannot pass with unresolved critical findings.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("record", type=Path) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--expected-release-record-sha256", required=True) + args = parser.parse_args() + validate(json.loads(args.record.read_text(encoding="utf-8")), + args.expected_commit, args.expected_release_record_sha256) + print("Backup, monitoring and recovery acceptance record is structurally complete and passed. " + "This validates the record, not its restricted evidence.") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"Backup/recovery acceptance record rejected: {error}", file=__import__("sys").stderr) + raise SystemExit(1) diff --git a/deploy/backup_transfer.py b/deploy/backup_transfer.py new file mode 100644 index 0000000..ee445fc --- /dev/null +++ b/deploy/backup_transfer.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Atomically transfer encrypted GuestOps backups to restricted storage.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import stat +import subprocess +import tempfile +import time +import uuid + + +BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg") +SAFE_HOST = re.compile(r"[A-Za-z0-9.-]{1,253}") +SAFE_USER = re.compile(r"[A-Za-z_][A-Za-z0-9_-]{0,31}") +SAFE_REMOTE_PATH = re.compile(r"/[A-Za-z0-9._/-]{1,500}") +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def private_directory(value: str) -> Path: + requested = Path(value) + require(requested.is_absolute() and not requested.is_symlink(), + "BACKUP_DIRECTORY must be an absolute, non-symlink path.") + directory = requested.resolve() + require(directory.is_dir(), "BACKUP_DIRECTORY must exist.") + require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0, + "BACKUP_DIRECTORY must not be accessible to group or other users.") + return directory + + +def regular_file(value: str, field: str, *, private: bool) -> Path: + requested = Path(value) + require(requested.is_absolute() and not requested.is_symlink(), + f"{field} must be an absolute, non-symlink path.") + path = requested.resolve() + require(path.is_file(), f"{field} must be an existing regular file.") + if private: + require(stat.S_IMODE(path.stat().st_mode) & 0o077 == 0, + f"{field} must not be accessible to group or other users.") + return path + + +def configuration(environment: dict[str, str]) -> dict[str, object]: + directory = private_directory(environment.get("BACKUP_DIRECTORY", "")) + host = environment.get("BACKUP_REMOTE_HOST", "") + user = environment.get("BACKUP_REMOTE_USER", "") + remote = environment.get("BACKUP_REMOTE_DIRECTORY", "") + require(SAFE_HOST.fullmatch(host) is not None, "BACKUP_REMOTE_HOST is invalid.") + require(SAFE_USER.fullmatch(user) is not None, "BACKUP_REMOTE_USER is invalid.") + require(SAFE_REMOTE_PATH.fullmatch(remote) is not None and "//" not in remote + and "/../" not in remote + "/" and not remote.endswith("/.."), + "BACKUP_REMOTE_DIRECTORY must be a safe absolute path.") + identity = regular_file(environment.get("BACKUP_SSH_IDENTITY", ""), + "BACKUP_SSH_IDENTITY", private=True) + known_hosts = regular_file(environment.get("BACKUP_SSH_KNOWN_HOSTS", ""), + "BACKUP_SSH_KNOWN_HOSTS", private=False) + require(shutil.which("ssh") is not None and shutil.which("rsync") is not None, + "ssh and rsync are required.") + return { + "directory": directory, + "host": host, + "user": user, + "remote": remote.rstrip("/"), + "identity": identity, + "known_hosts": known_hosts, + } + + +def ssh_base(config: dict[str, object]) -> list[str]: + return [ + "ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", + "-o", f"UserKnownHostsFile={config['known_hosts']}", + "-i", str(config["identity"]), f"{config['user']}@{config['host']}", + ] + + +def run(args: list[str], *, environment: dict[str, str] | None = None) -> bytes: + result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=900, env=environment) + require(result.returncode == 0, + f"{Path(args[0]).name} step failed; review the restricted operator logs.") + return result.stdout + + +def remote_digest(config: dict[str, object], remote_path: str) -> str | None: + command = f"if test -f {remote_path} && test ! -L {remote_path}; then sha256sum -- {remote_path}; fi" + output = run([*ssh_base(config), command]).decode("utf-8", "strict").strip() + if not output: + return None + value = output.split()[0] + require(SHA256.fullmatch(value) is not None, "Remote checksum response was invalid.") + return value + + +def marker_path(backup: Path) -> Path: + return backup.with_name(backup.name + ".transferred.json") + + +def write_marker(backup: Path, checksum: str) -> None: + marker = marker_path(backup) + payload = json.dumps({ + "schemaVersion": 1, + "backup": backup.name, + "sha256": checksum, + "verifiedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + }, sort_keys=True) + "\n" + fd, temporary = tempfile.mkstemp(prefix=marker.name + ".", dir=marker.parent) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, marker) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def transfer_one(config: dict[str, object], backup: Path) -> None: + require(backup.is_file() and not backup.is_symlink() + and BACKUP_NAME.fullmatch(backup.name) is not None, + "Refusing to transfer an unexpected backup path.") + checksum = digest(backup) + remote_final = f"{config['remote']}/{backup.name}" + existing = remote_digest(config, remote_final) + if existing is not None: + require(existing == checksum, "A remote backup with this name has a different checksum.") + write_marker(backup, checksum) + return + + remote_partial = f"{config['remote']}/.{backup.name}.partial-{uuid.uuid4().hex}" + rsh = shlex.join([ + "ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", + "-o", f"UserKnownHostsFile={config['known_hosts']}", + "-i", str(config["identity"]), + ]) + rsync_environment = os.environ.copy() + rsync_environment["RSYNC_RSH"] = rsh + try: + run(["rsync", "--archive", "--chmod=F600", "--protect-args", "--", + str(backup), f"{config['user']}@{config['host']}:{remote_partial}"], + environment=rsync_environment) + require(remote_digest(config, remote_partial) == checksum, + "Transferred backup checksum does not match the local file.") + command = (f"test ! -e {remote_final} && mv -T -- {remote_partial} {remote_final} " + f"&& chmod 600 -- {remote_final}") + run([*ssh_base(config), command]) + require(remote_digest(config, remote_final) == checksum, + "Final remote backup checksum does not match the local file.") + write_marker(backup, checksum) + except Exception: + # The name contains a fresh random suffix and is the only remote object this run may remove. + subprocess.run([*ssh_base(config), f"rm -f -- {remote_partial}"], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) + raise + + +def transfer_pending(config: dict[str, object]) -> int: + directory = config["directory"] + backups = sorted(path for path in directory.iterdir() + if path.is_file() and not path.is_symlink() + and BACKUP_NAME.fullmatch(path.name) is not None) + for backup in backups: + marker = marker_path(backup) + if marker.is_file() and not marker.is_symlink(): + try: + recorded = json.loads(marker.read_text(encoding="utf-8")) + if recorded.get("sha256") == digest(backup): + continue + except (OSError, ValueError, json.JSONDecodeError): + pass + transfer_one(config, backup) + return len(backups) + + +def prune_verified(config: dict[str, object], retention_days: int, now: float | None = None) -> int: + require(1 <= retention_days <= 365, "Local retention must be between 1 and 365 days.") + threshold = (time.time() if now is None else now) - retention_days * 86400 + removed = 0 + for backup in config["directory"].iterdir(): + if not backup.is_file() or backup.is_symlink() or BACKUP_NAME.fullmatch(backup.name) is None: + continue + marker = marker_path(backup) + if backup.stat().st_mtime >= threshold or not marker.is_file() or marker.is_symlink(): + continue + try: + recorded = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): + continue + if recorded.get("backup") != backup.name or recorded.get("sha256") != digest(backup): + continue + backup.unlink() + marker.unlink() + removed += 1 + return removed + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prune-verified", action="store_true") + parser.add_argument("--retention-days", type=int, default=7) + args = parser.parse_args() + config = configuration(dict(os.environ)) + observed = transfer_pending(config) + removed = prune_verified(config, args.retention_days) if args.prune_verified else 0 + print(f"Backup transfer completed: {observed} encrypted backup(s) inspected; " + f"{removed} verified local backup(s) expired.") + + +if __name__ == "__main__": + try: + main() + except (OSError, RuntimeError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"Backup transfer failed: {error}", file=__import__("sys").stderr) + raise SystemExit(1) diff --git a/deploy/debian-host-acceptance.example.json b/deploy/debian-host-acceptance.example.json new file mode 100644 index 0000000..544b96b --- /dev/null +++ b/deploy/debian-host-acceptance.example.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "system": "guestops-debian-host", + "evidenceId": "debian-host", + "releaseVersion": "0.2.0", + "releaseCommit": "0000000000000000000000000000000000000000", + "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "environment": "https://sandbox-guestops.futuresens.co.uk", + "hostIdentifier": "guestops-sandbox-01", + "images": { + "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} + }, + "hostFacts": { + "debianMajor": 12, + "cpuCores": 4, + "memoryBytes": 8140382208, + "freeDiskBytes": 14275686400, + "publicTcpPorts": [] + }, + "featureControls": { + "googleSending": "disabled", + "faqLiveMode": "disabled", + "pmsWrites": "disabled", + "paymentCreation": "disabled" + }, + "operator": "REPLACE", + "reviewedBy": "REPLACE", + "startedAt": "2026-09-30T09:00:00Z", + "endedAt": "2026-09-30T10:00:00Z", + "reviewedAt": "2026-09-30T11:00:00Z", + "unresolvedCriticalFindings": 1, + "scenarios": [ + {"id": "boot-services", "status": "not-run", "evidence": []}, + {"id": "controlled-reboot", "status": "not-run", "evidence": []}, + {"id": "durable-log-retrieval", "status": "not-run", "evidence": []}, + {"id": "host-baseline", "status": "not-run", "evidence": []}, + {"id": "https-and-redirect", "status": "not-run", "evidence": []}, + {"id": "network-exposure", "status": "not-run", "evidence": []}, + {"id": "proxy-trust", "status": "not-run", "evidence": []}, + {"id": "secret-free-logs", "status": "not-run", "evidence": []}, + {"id": "workspace-health", "status": "not-run", "evidence": []} + ] +} diff --git a/deploy/debian_acceptance.py b/deploy/debian_acceptance.py new file mode 100644 index 0000000..d0ca02e --- /dev/null +++ b/deploy/debian_acceptance.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Validate restricted GuestOps Debian-host and persistence acceptance records.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +from pathlib import Path +import re +from urllib.parse import urlparse + + +VERSION = "0.2.0" +SYSTEMS = { + "guestops-debian-host": { + "evidenceId": "debian-host", + "scenarios": { + "host-baseline", + "network-exposure", + "https-and-redirect", + "proxy-trust", + "boot-services", + "controlled-reboot", + "workspace-health", + "durable-log-retrieval", + "secret-free-logs", + }, + }, + "guestops-persistence": { + "evidenceId": "persistence", + "scenarios": { + "separate-volume-layout", + "service-restart", + "container-recreation", + "database-inventory", + "data-protection-key", + "image-identity", + "post-reboot-persistence", + }, + }, +} +SHA256 = re.compile(r"[0-9a-f]{64}") +GIT_SHA = re.compile(r"[0-9a-f]{40}") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def timestamp(value: object, field: str) -> dt.datetime: + require(isinstance(value, str) and value.endswith("Z"), + f"{field} must be a UTC timestamp ending in Z.") + try: + parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00") + except ValueError as error: + raise ValueError(f"{field} is not a valid timestamp.") from error + require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.") + return parsed + + +def safe_text(value: object, field: str, minimum: int = 2, maximum: int = 160) -> str: + text = str(value or "").strip() + require(minimum <= len(text) <= maximum and "@" not in text and "\\" not in text, + f"{field} must be safe text without an email address or local path.") + return text + + +def validate_common(record: object, expected_commit: str, expected_release_sha256: str) -> str: + require(isinstance(record, dict), "Acceptance record must be a JSON object.") + require(record.get("schemaVersion") == 1, "Unsupported Debian acceptance schema.") + system = record.get("system") + require(system in SYSTEMS, "Unknown Debian acceptance record system.") + require(record.get("evidenceId") == SYSTEMS[system]["evidenceId"], + f"{system} has the wrong evidenceId.") + require(record.get("releaseVersion") == VERSION, + f"releaseVersion must be {VERSION}.") + require(GIT_SHA.fullmatch(str(expected_commit)) is not None, + "Expected release commit must be a full lowercase Git SHA.") + require(SHA256.fullmatch(str(expected_release_sha256)) is not None, + "Expected release-record checksum must be a lowercase SHA-256 digest.") + require(record.get("releaseCommit") == expected_commit, + "releaseCommit does not match the approved candidate.") + require(record.get("releaseRecordSha256") == expected_release_sha256, + "releaseRecordSha256 does not match the retained release record.") + require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None, + "archiveSha256 must be a lowercase SHA-256 digest.") + + origin = urlparse(str(record.get("environment", ""))) + require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/") + and not origin.query and not origin.fragment and origin.username is None + and origin.password is None, + "environment must be an HTTPS origin without credentials, path, query or fragment.") + host = safe_text(record.get("hostIdentifier"), "hostIdentifier") + + images = record.get("images") + require(isinstance(images, dict) and set(images) == {"api", "worker"}, + "images must contain exactly api and worker.") + for name in ("api", "worker"): + image = images[name] + require(isinstance(image, dict) and set(image) == {"reference", "id"}, + f"images.{name} must contain exactly reference and id.") + require(image["reference"] == f"guestops-{name}:{expected_commit}", + f"images.{name}.reference must use the full approved commit.") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, + f"images.{name}.id must be an immutable image ID.") + + operator = safe_text(record.get("operator"), "operator") + reviewer = safe_text(record.get("reviewedBy"), "reviewedBy") + require(operator.casefold() != reviewer.casefold(), + "operator and reviewedBy must be different people.") + started = timestamp(record.get("startedAt"), "startedAt") + ended = timestamp(record.get("endedAt"), "endedAt") + reviewed = timestamp(record.get("reviewedAt"), "reviewedAt") + require(started <= ended <= reviewed, "Acceptance timestamps are out of order.") + + scenarios = record.get("scenarios") + require(isinstance(scenarios, list), "scenarios must be a list.") + ids = [item.get("id") for item in scenarios if isinstance(item, dict)] + required = SYSTEMS[system]["scenarios"] + require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == required, + f"{system} requires its exact acceptance scenario set.") + for item in scenarios: + scenario_id = item["id"] + require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.") + evidence = item.get("evidence") + require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all( + isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value + and "\\" not in value and not value.startswith("/") for value in evidence + ), f"Scenario {scenario_id} requires safe opaque evidence references.") + + require(record.get("unresolvedCriticalFindings") == 0, + "Acceptance cannot pass with unresolved critical findings.") + return host + + +def validate_host(record: object, expected_commit: str, expected_release_sha256: str) -> str: + host = validate_common(record, expected_commit, expected_release_sha256) + require(record.get("system") == "guestops-debian-host", + "First record must be guestops-debian-host.") + facts = record.get("hostFacts") + require(isinstance(facts, dict) and set(facts) == { + "debianMajor", "cpuCores", "memoryBytes", "freeDiskBytes", "publicTcpPorts", + }, "hostFacts must contain the exact reviewed host facts.") + require(isinstance(facts["debianMajor"], int) and facts["debianMajor"] >= 12, + "Debian 12 or newer is required.") + require(isinstance(facts["cpuCores"], int) and facts["cpuCores"] >= 4, + "At least four CPU cores are required.") + require(isinstance(facts["memoryBytes"], int) and facts["memoryBytes"] >= 7_500_000_000, + "At least 7.5 GB of memory is required.") + require(isinstance(facts["freeDiskBytes"], int) and facts["freeDiskBytes"] >= 8 * 1024**3, + "At least 8 GiB of free disk space is required.") + require(facts["publicTcpPorts"] == [80, 443], + "Only TCP ports 80 and 443 may be public.") + require(record.get("featureControls") == { + "googleSending": "disabled", + "faqLiveMode": "disabled", + "pmsWrites": "disabled", + "paymentCreation": "disabled", + }, "Unaccepted external writes and FAQ live mode must remain disabled.") + return host + + +def validate_persistence(record: object, expected_commit: str, expected_release_sha256: str) -> str: + host = validate_common(record, expected_commit, expected_release_sha256) + require(record.get("system") == "guestops-persistence", + "Second record must be guestops-persistence.") + require(record.get("drillCommand") == "python3 deploy/ops.py persistence-drill --confirm-restart", + "drillCommand must identify the confirmation-gated persistence drill.") + return host + + +def validate_pair(host_record: object, persistence_record: object, + expected_commit: str, expected_release_sha256: str) -> None: + host = validate_host(host_record, expected_commit, expected_release_sha256) + persistence_host = validate_persistence( + persistence_record, expected_commit, expected_release_sha256) + require(host == persistence_host, "Both records must identify the same host.") + for field in ("environment", "releaseVersion", "releaseCommit", "releaseRecordSha256", + "archiveSha256", "images"): + require(host_record.get(field) == persistence_record.get(field), + f"Both records must use the same {field}.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("host_record", type=Path) + parser.add_argument("persistence_record", type=Path) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--expected-release-record-sha256", required=True) + args = parser.parse_args() + host_record = json.loads(args.host_record.read_text(encoding="utf-8")) + persistence_record = json.loads(args.persistence_record.read_text(encoding="utf-8")) + validate_pair(host_record, persistence_record, + args.expected_commit, args.expected_release_record_sha256) + print("Debian-host and persistence acceptance records are structurally complete and passed. " + "This validates the records, not their restricted evidence.") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"Debian acceptance records rejected: {error}", file=__import__("sys").stderr) + raise SystemExit(1) diff --git a/deploy/monitor_status.py b/deploy/monitor_status.py new file mode 100644 index 0000000..368d393 --- /dev/null +++ b/deploy/monitor_status.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Write non-sensitive GuestOps host status for a read-only monitoring agent.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +from pathlib import Path +import re +import shutil +import socket +import ssl +import stat +import subprocess +import tempfile +import urllib.request +from urllib.parse import urlparse + + +BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg") +MARKER_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg\.transferred\.json") +AUTH = ('const c=new Mongo("mongodb://127.0.0.1");' + 'c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,' + 'process.env.MONGO_INITDB_ROOT_PASSWORD);') +HEARTBEAT = ('const x=c.getDB("guestops").workerheartbeat.findOne({_id:"worker"});' + 'print(JSON.stringify(x&&x.At?x.At:null));') + + +def require(condition: bool, message: str) -> None: + if not condition: + raise RuntimeError(message) + + +def run(args: list[str], root: Path, timeout: int = 30) -> bytes: + result = subprocess.run(args, cwd=root, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + require(result.returncode == 0, f"{Path(args[0]).name} probe failed.") + return result.stdout + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def age_seconds(path: Path, pattern: re.Pattern[str], now: dt.datetime) -> int | None: + if not path.is_dir() or path.is_symlink(): + return None + times = [item.stat().st_mtime for item in path.iterdir() + if item.is_file() and not item.is_symlink() and pattern.fullmatch(item.name)] + return max(0, int(now.timestamp() - max(times))) if times else None + + +def https_status(origin: str, now: dt.datetime) -> dict[str, object]: + parsed = urlparse(origin) + require(parsed.scheme == "https" and parsed.hostname and parsed.port in (None, 443) + and parsed.path in ("", "/") and not parsed.query and not parsed.fragment + and parsed.username is None and parsed.password is None, + "GUESTOPS_ORIGIN must be an HTTPS origin without credentials or a path.") + context = ssl.create_default_context() + with socket.create_connection((parsed.hostname, 443), timeout=10) as connection: + with context.wrap_socket(connection, server_hostname=parsed.hostname) as secured: + certificate = secured.getpeercert() + expires = dt.datetime.strptime(certificate["notAfter"], "%b %d %H:%M:%S %Y %Z").replace( + tzinfo=dt.timezone.utc) + with urllib.request.urlopen(origin.rstrip("/") + "/health/ready", timeout=15, + context=context) as response: + ready = response.status == 200 and json.load(response) == {"status": "ready"} + return {"ready": ready, "certificateDaysRemaining": max(0, int((expires - now).total_seconds() // 86400))} + + +def parse_compose(value: bytes) -> dict[str, dict[str, str]]: + text = value.decode("utf-8", "strict").strip() + if not text: + return {} + try: + parsed = json.loads(text) + rows = parsed if isinstance(parsed, list) else [parsed] + except json.JSONDecodeError: + rows = [json.loads(line) for line in text.splitlines() if line.strip()] + result = {} + for row in rows: + if not isinstance(row, dict): + continue + service = str(row.get("Service", "")) + if service in {"api", "worker", "mongo"}: + result[service] = { + "state": str(row.get("State", "unknown")).lower(), + "health": str(row.get("Health", "none") or "none").lower(), + } + return result + + +def worker_heartbeat_age(root: Path, now: dt.datetime) -> int | None: + output = run(["docker", "compose", "exec", "-T", "mongo", "mongosh", "--quiet", + "--nodb", "--eval", AUTH + HEARTBEAT], root).decode("utf-8", "strict").strip() + value = json.loads(output) + if value is None: + return None + require(isinstance(value, str), "Worker heartbeat response was invalid.") + parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + require(parsed.tzinfo is not None, "Worker heartbeat must contain a timezone.") + return max(0, int((now - parsed.astimezone(dt.timezone.utc)).total_seconds())) + + +def build_status(root: Path, backup_directory: Path, origin: str, + now: dt.datetime | None = None) -> tuple[dict[str, object], list[str]]: + moment = now or utc_now() + errors: list[str] = [] + try: + https = https_status(origin, moment) + except Exception: + https = {"ready": False, "certificateDaysRemaining": None} + errors.append("https-probe-failed") + try: + containers = parse_compose(run(["docker", "compose", "ps", "--format", "json"], root)) + if set(containers) != {"api", "worker", "mongo"}: + errors.append("container-set-incomplete") + except Exception: + containers = {} + errors.append("container-probe-failed") + try: + heartbeat_age = worker_heartbeat_age(root, moment) + if heartbeat_age is None: + errors.append("worker-heartbeat-missing") + except Exception: + heartbeat_age = None + errors.append("worker-heartbeat-probe-failed") + disk = shutil.disk_usage(root) + status = { + "schemaVersion": 1, + "generatedAt": moment.isoformat().replace("+00:00", "Z"), + "https": https, + "containers": containers, + "workerHeartbeatAgeSeconds": heartbeat_age, + "backupAgeSeconds": age_seconds(backup_directory, BACKUP_NAME, moment), + "verifiedTransferAgeSeconds": age_seconds(backup_directory, MARKER_NAME, moment), + "diskFreePercent": round(disk.free * 100 / disk.total, 2), + "persistentJournal": Path("/var/log/journal").is_dir(), + "errors": sorted(set(errors)), + } + return status, errors + + +def write_status(path: Path, status: dict[str, object]) -> None: + require(path.is_absolute() and not path.is_symlink(), + "Status output must be an absolute, non-symlink path.") + parent = path.parent.resolve() + require(parent.is_dir() and not path.parent.is_symlink(), "Status output directory must exist.") + fd, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=parent) + try: + os.fchmod(fd, 0o644) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(status, stream, sort_keys=True, separators=(",", ":")) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--backup-directory", type=Path, default=Path("/var/backups/guestops")) + parser.add_argument("--origin", default="https://sandbox-guestops.futuresens.co.uk") + parser.add_argument("--output", type=Path, default=Path("/run/guestops-monitor/status.json")) + args = parser.parse_args() + root = args.root.resolve() + require(root.is_dir(), "GuestOps root must exist.") + backup_directory = args.backup_directory.resolve() + status, errors = build_status(root, backup_directory, args.origin) + write_status(args.output, status) + print("GuestOps monitoring status updated." if not errors + else "GuestOps monitoring status updated with failed probes.") + raise SystemExit(1 if errors else 0) + + +if __name__ == "__main__": + try: + main() + except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"GuestOps monitoring probe failed: {error}", file=__import__("sys").stderr) + raise SystemExit(1) diff --git a/deploy/persistence-acceptance.example.json b/deploy/persistence-acceptance.example.json new file mode 100644 index 0000000..bc34f89 --- /dev/null +++ b/deploy/persistence-acceptance.example.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "system": "guestops-persistence", + "evidenceId": "persistence", + "releaseVersion": "0.2.0", + "releaseCommit": "0000000000000000000000000000000000000000", + "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", + "environment": "https://sandbox-guestops.futuresens.co.uk", + "hostIdentifier": "guestops-sandbox-01", + "images": { + "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, + "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} + }, + "drillCommand": "python3 deploy/ops.py persistence-drill --confirm-restart", + "operator": "REPLACE", + "reviewedBy": "REPLACE", + "startedAt": "2026-09-30T09:00:00Z", + "endedAt": "2026-09-30T10:00:00Z", + "reviewedAt": "2026-09-30T11:00:00Z", + "unresolvedCriticalFindings": 1, + "scenarios": [ + {"id": "container-recreation", "status": "not-run", "evidence": []}, + {"id": "data-protection-key", "status": "not-run", "evidence": []}, + {"id": "database-inventory", "status": "not-run", "evidence": []}, + {"id": "image-identity", "status": "not-run", "evidence": []}, + {"id": "post-reboot-persistence", "status": "not-run", "evidence": []}, + {"id": "separate-volume-layout", "status": "not-run", "evidence": []}, + {"id": "service-restart", "status": "not-run", "evidence": []} + ] +} diff --git a/deploy/systemd/backup-transfer.env.example b/deploy/systemd/backup-transfer.env.example new file mode 100644 index 0000000..35eb787 --- /dev/null +++ b/deploy/systemd/backup-transfer.env.example @@ -0,0 +1,6 @@ +BACKUP_DIRECTORY=/var/backups/guestops +BACKUP_REMOTE_HOST=restricted-store.example.invalid +BACKUP_REMOTE_USER=guestops_upload +BACKUP_REMOTE_DIRECTORY=/restricted/guestops/backups +BACKUP_SSH_IDENTITY=/etc/guestops/backup-transfer.key +BACKUP_SSH_KNOWN_HOSTS=/etc/guestops/backup-known-hosts diff --git a/deploy/systemd/backup.env.example b/deploy/systemd/backup.env.example new file mode 100644 index 0000000..cabeb3b --- /dev/null +++ b/deploy/systemd/backup.env.example @@ -0,0 +1,2 @@ +BACKUP_RECIPIENT=0123456789ABCDEF0123456789ABCDEF01234567 +BACKUP_DIRECTORY=/var/backups/guestops diff --git a/deploy/systemd/guestops-backup-transfer.service b/deploy/systemd/guestops-backup-transfer.service new file mode 100644 index 0000000..44eac9e --- /dev/null +++ b/deploy/systemd/guestops-backup-transfer.service @@ -0,0 +1,19 @@ +[Unit] +Description=Transfer GuestOps encrypted backups to restricted storage +Wants=network-online.target +After=network-online.target guestops-backup.service +ConditionPathIsDirectory=/srv/guestops +ConditionPathIsDirectory=/var/backups/guestops + +[Service] +Type=oneshot +WorkingDirectory=/srv/guestops +EnvironmentFile=/etc/guestops/backup-transfer.env +UMask=0077 +ExecStart=/usr/bin/python3 /srv/guestops/deploy/backup_transfer.py --prune-verified --retention-days 7 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=full +ReadWritePaths=/var/backups/guestops +TimeoutStartSec=30min diff --git a/deploy/systemd/guestops-backup-transfer.timer b/deploy/systemd/guestops-backup-transfer.timer new file mode 100644 index 0000000..1b7dd29 --- /dev/null +++ b/deploy/systemd/guestops-backup-transfer.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Retry GuestOps off-host backup transfer + +[Timer] +OnBootSec=10min +OnUnitActiveSec=15min +Persistent=true +Unit=guestops-backup-transfer.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/guestops-monitor-status.service b/deploy/systemd/guestops-monitor-status.service new file mode 100644 index 0000000..c369c6e --- /dev/null +++ b/deploy/systemd/guestops-monitor-status.service @@ -0,0 +1,21 @@ +[Unit] +Description=Write non-sensitive GuestOps monitoring status +Requires=docker.service +After=docker.service network-online.target +ConditionPathIsDirectory=/srv/guestops +ConditionPathIsDirectory=/var/backups/guestops + +[Service] +Type=oneshot +WorkingDirectory=/srv/guestops +UMask=0022 +ExecStart=/usr/bin/python3 /srv/guestops/deploy/monitor_status.py +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +RuntimeDirectory=guestops-monitor +RuntimeDirectoryMode=0755 +RuntimeDirectoryPreserve=yes +ReadWritePaths=/run/guestops-monitor +TimeoutStartSec=2min diff --git a/deploy/systemd/guestops-monitor-status.timer b/deploy/systemd/guestops-monitor-status.timer new file mode 100644 index 0000000..24e4b95 --- /dev/null +++ b/deploy/systemd/guestops-monitor-status.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Refresh GuestOps monitoring status each minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +Persistent=true +Unit=guestops-monitor-status.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/zabbix-agent-guestops.conf.example b/deploy/systemd/zabbix-agent-guestops.conf.example new file mode 100644 index 0000000..265e409 --- /dev/null +++ b/deploy/systemd/zabbix-agent-guestops.conf.example @@ -0,0 +1,2 @@ +# The privileged systemd probe owns Docker/database access. Zabbix reads only this aggregate JSON. +UserParameter=guestops.status,cat /run/guestops-monitor/status.json diff --git a/docs/deployment.md b/docs/deployment.md index 014c559..65f9068 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -86,4 +86,16 @@ The drill restarts MongoDB, API and worker, then force-recreates the stateless a Record the host, operator, start/end time, release record checksum, resolved image IDs, preflight output and drill result in the deployment acceptance record. Also verify Docker starts at boot and perform a controlled Debian reboot before Gate A approval. After reboot, run the online preflight and inspect the Workspace health page; do not infer worker health solely from API readiness. +Keep the `debian-host` and `persistence` records in the restricted evidence store. Start from `deploy/debian-host-acceptance.example.json` and `deploy/persistence-acceptance.example.json`; the examples deliberately fail until every supervised scenario has passed. Bind both records to the expected release identifiers and validate them together: + +```sh +python3 deploy/debian_acceptance.py \ + /secure/acceptance/debian-host.json \ + /secure/acceptance/persistence.json \ + --expected-commit FULL_40_CHARACTER_SHA \ + --expected-release-record-sha256 RELEASE_RECORD_SHA256 +``` + +The validator requires matching archive and image identities, separate operator and reviewer names, the approved host capacity, only ports 80 and 443 recorded as publicly reachable, disabled unaccepted external writes, exact passed scenario sets and no unresolved critical findings. It validates record structure, not the restricted evidence itself. Retain the records, validator output and their checksums outside Git. + The supplied Docker `json-file` logs are size-capped to protect the small pilot disk, but container recreation removes that container's local log history. Before host acceptance, route GuestOps and Nginx logs to the site's durable restricted logging system, or use a reviewed Docker logging override backed by persistent systemd journal storage. Prove that operators can retrieve pre-recreation logs without exposing request credentials or OAuth callback query strings. Central retention and alerting are completed under milestone 11. diff --git a/docs/operations.md b/docs/operations.md index 9cc3f1d..dcda4e1 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -113,6 +113,43 @@ systemctl list-timers guestops-backup.timer The timer deliberately causes the same brief maintenance interruption as a manual backup. `Persistent=true` runs a missed event after downtime, so choose and communicate the maintenance window. A successful unit only stages an encrypted file locally. Configure an independently monitored off-host transfer, verify the destination checksum, alert on both unit and transfer failure, and test the alert route. Do not add automatic deletion until retention, legal hold and recovery requirements have named owners. +### Restricted off-host transfer + +The optional transfer service uses rsync over pinned-host SSH. Create a dedicated upload-only account at the restricted store, disable interactive login, agent/port forwarding and access outside the GuestOps backup directory, and keep its private key only in `/etc/guestops` with mode 600. Pin the reviewed server host key; never use `StrictHostKeyChecking=no`. + +Start from `deploy/systemd/backup-transfer.env.example` and store the completed file as `/etc/guestops/backup-transfer.env` with mode 600. The local and remote directories must already exist and remain private. Install and verify the service and its 15-minute retry timer: + +```sh +sudo install -m 600 deploy/systemd/backup-transfer.env.example /etc/guestops/backup-transfer.env +sudoedit /etc/guestops/backup-transfer.env +sudo install -m 644 deploy/systemd/guestops-backup-transfer.service /etc/systemd/system/ +sudo install -m 644 deploy/systemd/guestops-backup-transfer.timer /etc/systemd/system/ +sudo systemd-analyze verify /etc/systemd/system/guestops-backup-transfer.service /etc/systemd/system/guestops-backup-transfer.timer +sudo systemctl daemon-reload +sudo systemctl start guestops-backup-transfer.service +sudo systemctl enable --now guestops-backup-transfer.timer +``` + +Only files named `guestops-YYYYMMDDTHHMMSSZ.tar.gpg` are eligible. The transfer writes a unique remote partial file, compares the remote and local SHA-256 values, atomically publishes a previously unused final name, verifies it again, and then writes a non-sensitive local marker. An existing remote name is accepted only when its checksum matches. Failed or mismatched transfers are never marked. The service removes local backups older than seven days only when the marker still matches the local checksum; untransferred or changed files are never pruned. + +The restricted store is the durable copy. Configure its independently reviewed retention policy for 35 daily and 12 monthly recovery points. Legal hold must override expiry. The host tool does not delete remote data or enforce remote retention. + +### Zabbix status boundary + +Do not give the Zabbix agent access to Docker, MongoDB credentials or the application owner session. A root-owned systemd probe reads those local sources and atomically publishes aggregate status under `/run/guestops-monitor/status.json`; the agent reads that file only. + +```sh +sudo install -m 644 deploy/systemd/guestops-monitor-status.service /etc/systemd/system/ +sudo install -m 644 deploy/systemd/guestops-monitor-status.timer /etc/systemd/system/ +sudo install -m 644 deploy/systemd/zabbix-agent-guestops.conf.example /etc/zabbix/zabbix_agentd.d/guestops.conf +sudo systemd-analyze verify /etc/systemd/system/guestops-monitor-status.service /etc/systemd/system/guestops-monitor-status.timer +sudo systemctl daemon-reload +sudo systemctl enable --now guestops-monitor-status.timer +sudo systemctl restart zabbix-agent +``` + +Create dependent Zabbix items from `guestops.status` for HTTPS readiness, certificate days remaining, container state/health, worker heartbeat age, backup age, verified-transfer age, free-disk percentage, persistent journal availability and probe error categories. Alert when the heartbeat is older than three minutes; backup or transfer is older than 30 hours; free disk falls below 25% (warning) or 15% (critical); the certificate has fewer than 30 days (warning) or 14 days (critical); any required container is absent/unhealthy; or the probe/timers fail. Route alerts to the monitoring owner and escalate unacknowledged critical events to the technical owner after 15 minutes. Exercise every trigger and its recovery notification with synthetic conditions. + Temporary plaintext files are held in private directories and removed on normal completion or exceptions. Process termination or power loss can leave temporary data, stopped services or TTL expiry disabled. After an interrupted run, inspect the dedicated project and remove only its identified abandoned temporary directory after securing any recovery material. Restore the recorded TTL setting (normally true) and restart the services: ```sh @@ -146,4 +183,20 @@ Restore into new isolated MongoDB and key volumes; preserve the damaged original **A restored database can predate emails, invoices and PMS changes that providers already completed.** Review pending, sending and uncertain records against provider evidence before enabling any worker, including automatic FAQ rules. Do not replay an older approval merely because the restored record says it is pending. Reconcile external effects, validate account sessions and mailbox authorization, and explicitly approve the cutover only after these checks. Rotate credentials if compromise prompted the recovery. Keep the old deployment stopped when enabling the replacement. CI exercises a synthetic encrypted backup and isolated restore drill, including actual key decryption and database comparison. A successful CI drill is separate from the required rehearsal on the Debian server with its actual deployment configuration. + +## Milestone 11 acceptance record + +Keep raw backups, restored data, remote paths, host keys, monitoring recipients, screenshots and logs outside Git. Copy `deploy/backup-restore-acceptance.example.json` to the restricted evidence store and replace every placeholder only after completing the supervised exercises. The example deliberately fails. + +The accepted record uses a 24-hour RPO, four-hour RTO, seven-day verified local staging window, 35 daily and 12 monthly off-host recovery points, and the opaque evidence ID `backup-restore`. Bind it to the same release identifiers as the Debian-host and persistence records: + +```sh +python3 deploy/backup_restore_acceptance.py \ + /secure/acceptance/backup-restore.json \ + --expected-commit FULL_40_CHARACTER_SHA \ + --expected-release-record-sha256 RELEASE_RECORD_SHA256 +sha256sum /secure/acceptance/backup-restore.json +``` + +The validator requires matching local/remote backup checksums, exact immutable image identities, a timed isolated restore, tested retention/legal hold and alert escalation, an image rollback that preserves persistent volumes, return to the approved candidate with external writes disabled, separate independent review, healthy monitoring and no unresolved critical findings. Structural validation does not inspect the restricted evidence or authorize a release by itself. diff --git a/tests/test_backup_restore_acceptance.py b/tests/test_backup_restore_acceptance.py new file mode 100644 index 0000000..a3a33f8 --- /dev/null +++ b/tests/test_backup_restore_acceptance.py @@ -0,0 +1,129 @@ +import importlib.util +from pathlib import Path +import unittest + + +spec = importlib.util.spec_from_file_location( + "backup_restore_acceptance", + Path(__file__).resolve().parents[1] / "deploy" / "backup_restore_acceptance.py") +acceptance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(acceptance) + +COMMIT = "a" * 40 +RELEASE_SHA = "b" * 64 + + +def valid_record(): + return { + "schemaVersion": 1, + "system": "guestops-backup-recovery", + "evidenceId": "backup-restore", + "dataClassification": "synthetic-only", + "releaseVersion": "0.2.0", + "releaseCommit": COMMIT, + "releaseRecordSha256": RELEASE_SHA, + "archiveSha256": "c" * 64, + "environment": "https://sandbox-guestops.futuresens.co.uk", + "hostIdentifier": "guestops-sandbox-01", + "images": { + "api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64}, + "worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64}, + "mongo": {"reference": "mongo:8.0", "id": "sha256:" + "f" * 64}, + }, + "owners": { + "backup": "Backup owner", + "monitoring": "Monitoring owner", + "recoveryOperator": "Recovery operator", + "technicalEscalation": "Technical owner", + "retention": "Retention owner", + "independentReviewer": "Independent reviewer", + }, + "startedAt": "2026-10-01T09:00:00Z", + "endedAt": "2026-10-01T12:00:00Z", + "reviewedAt": "2026-10-01T13:00:00Z", + "recoveryObjectives": { + "targetRpoHours": 24, "observedRpoHours": 1, + "targetRtoMinutes": 240, "observedRtoMinutes": 180, + }, + "retention": { + "localVerifiedDays": 7, "offHostDaily": 35, + "offHostMonthly": 12, "legalHoldOverrideTested": True, + }, + "backup": { + "createdAt": "2026-10-01T08:00:00Z", + "sha256": "1" * 64, + "transferredSha256": "1" * 64, + "privateKeyPresentOnHost": False, + }, + "rollback": { + "previousReleaseCommit": "2" * 40, + "previousArchiveSha256": "3" * 64, + "previousImages": { + "api": {"reference": "guestops-api:" + "2" * 40, "id": "sha256:" + "4" * 64}, + "worker": {"reference": "guestops-worker:" + "2" * 40, "id": "sha256:" + "5" * 64}, + }, + "persistentVolumesReplaced": False, + "restoredReleaseCommit": COMMIT, + "unresolvedOperations": 0, + "finalControls": { + "googleSending": "disabled", "faqLiveMode": "disabled", + "pmsWrites": "disabled", "paymentCreation": "disabled", + }, + }, + "monitoringState": "healthy", + "unresolvedCriticalFindings": 0, + "scenarios": [ + {"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]} + for index, scenario in enumerate(sorted(acceptance.SCENARIOS), 1) + ], + } + + +class BackupRestoreAcceptanceTests(unittest.TestCase): + def test_complete_record_passes(self): + acceptance.validate(valid_record(), COMMIT, RELEASE_SHA) + + def test_release_identity_and_images_are_bound(self): + record = valid_record(); record["releaseCommit"] = "9" * 40 + with self.assertRaisesRegex(ValueError, "approved candidate"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["images"]["api"]["reference"] = "guestops-api:latest" + with self.assertRaisesRegex(ValueError, "full approved commit"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["rollback"]["previousImages"]["worker"]["id"] = "mutable" + with self.assertRaisesRegex(ValueError, "retained previous release identity"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + + def test_recovery_objectives_and_checksums_must_pass(self): + record = valid_record(); record["recoveryObjectives"]["observedRtoMinutes"] = 241 + with self.assertRaisesRegex(ValueError, "four-hour"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["backup"]["transferredSha256"] = "9" * 64 + with self.assertRaisesRegex(ValueError, "checksums must match"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["recoveryObjectives"]["observedRpoHours"] = 2 + with self.assertRaisesRegex(ValueError, "match the backup"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + + def test_independent_review_retention_and_final_state_are_required(self): + record = valid_record(); record["owners"]["independentReviewer"] = record["owners"]["backup"] + with self.assertRaisesRegex(ValueError, "different"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["retention"]["legalHoldOverrideTested"] = False + with self.assertRaisesRegex(ValueError, "Retention"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["rollback"]["persistentVolumesReplaced"] = True + with self.assertRaisesRegex(ValueError, "must not replace"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + + def test_exact_passed_scenarios_and_monitoring_are_required(self): + record = valid_record(); record["scenarios"].pop() + with self.assertRaisesRegex(ValueError, "exact backup/recovery scenario"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + record = valid_record(); record["monitoringState"] = "degraded" + with self.assertRaisesRegex(ValueError, "Monitoring must be healthy"): + acceptance.validate(record, COMMIT, RELEASE_SHA) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_backup_transfer.py b/tests/test_backup_transfer.py new file mode 100644 index 0000000..60c8ca3 --- /dev/null +++ b/tests/test_backup_transfer.py @@ -0,0 +1,81 @@ +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import time +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + "backup_transfer", Path(__file__).resolve().parents[1] / "deploy" / "backup_transfer.py") +transfer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(transfer) + + +class BackupTransferTests(unittest.TestCase): + def backup(self, directory: Path, name="guestops-20261001T021700Z.tar.gpg") -> Path: + path = directory / name + path.write_bytes(b"encrypted-backup") + return path + + def config(self, directory: Path): + return { + "directory": directory, "host": "store.example.invalid", "user": "guestops_upload", + "remote": "/restricted/guestops", "identity": Path("/safe/key"), + "known_hosts": Path("/safe/known_hosts"), + } + + def test_existing_matching_remote_is_marked_without_upload(self): + with tempfile.TemporaryDirectory() as folder: + backup = self.backup(Path(folder)); checksum = transfer.digest(backup) + with patch.object(transfer, "remote_digest", return_value=checksum), \ + patch.object(transfer, "run") as run: + transfer.transfer_one(self.config(Path(folder)), backup) + run.assert_not_called() + marker = json.loads(transfer.marker_path(backup).read_text(encoding="utf-8")) + self.assertEqual(marker["sha256"], checksum) + + def test_existing_mismatched_remote_is_never_overwritten(self): + with tempfile.TemporaryDirectory() as folder: + backup = self.backup(Path(folder)) + with patch.object(transfer, "remote_digest", return_value="9" * 64), \ + patch.object(transfer, "run") as run: + with self.assertRaisesRegex(RuntimeError, "different checksum"): + transfer.transfer_one(self.config(Path(folder)), backup) + run.assert_not_called() + self.assertFalse(transfer.marker_path(backup).exists()) + + def test_new_remote_is_uploaded_verified_and_marked(self): + with tempfile.TemporaryDirectory() as folder: + backup = self.backup(Path(folder)); checksum = transfer.digest(backup) + with patch.object(transfer, "remote_digest", side_effect=[None, checksum, checksum]), \ + patch.object(transfer, "run", return_value=b"") as run, \ + patch.object(transfer.subprocess, "run"): + transfer.transfer_one(self.config(Path(folder)), backup) + self.assertTrue(any(call.args[0][0] == "rsync" for call in run.call_args_list)) + self.assertTrue(transfer.marker_path(backup).exists()) + + def test_prune_removes_only_old_checksum_verified_backups(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + verified = self.backup(directory) + checksum = transfer.digest(verified) + transfer.write_marker(verified, checksum) + unverified = self.backup(directory, "guestops-20261002T021700Z.tar.gpg") + old = time.time() - 8 * 86400 + os.utime(verified, (old, old)); os.utime(unverified, (old, old)) + removed = transfer.prune_verified(self.config(directory), 7, now=time.time()) + self.assertEqual(removed, 1) + self.assertFalse(verified.exists()) + self.assertTrue(unverified.exists()) + + def test_retention_bounds_are_enforced(self): + with tempfile.TemporaryDirectory() as folder: + with self.assertRaisesRegex(RuntimeError, "between 1 and 365"): + transfer.prune_verified(self.config(Path(folder)), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_debian_acceptance.py b/tests/test_debian_acceptance.py new file mode 100644 index 0000000..742538d --- /dev/null +++ b/tests/test_debian_acceptance.py @@ -0,0 +1,113 @@ +import importlib.util +from pathlib import Path +import unittest + + +spec = importlib.util.spec_from_file_location( + "debian_acceptance", Path(__file__).resolve().parents[1] / "deploy" / "debian_acceptance.py") +acceptance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(acceptance) + +COMMIT = "a" * 40 +RELEASE_SHA = "b" * 64 + + +def common(system): + return { + "schemaVersion": 1, + "system": system, + "evidenceId": acceptance.SYSTEMS[system]["evidenceId"], + "releaseVersion": "0.2.0", + "releaseCommit": COMMIT, + "releaseRecordSha256": RELEASE_SHA, + "archiveSha256": "c" * 64, + "environment": "https://sandbox-guestops.futuresens.co.uk", + "hostIdentifier": "guestops-sandbox-01", + "images": { + "api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64}, + "worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64}, + }, + "operator": "Deployment operator", + "reviewedBy": "Independent reviewer", + "startedAt": "2026-09-30T09:00:00Z", + "endedAt": "2026-09-30T10:00:00Z", + "reviewedAt": "2026-09-30T11:00:00Z", + "unresolvedCriticalFindings": 0, + "scenarios": [ + {"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]} + for index, scenario in enumerate(sorted(acceptance.SYSTEMS[system]["scenarios"]), 1) + ], + } + + +def valid_records(): + host = common("guestops-debian-host") + host["hostFacts"] = { + "debianMajor": 12, + "cpuCores": 4, + "memoryBytes": 8_140_382_208, + "freeDiskBytes": 14_275_686_400, + "publicTcpPorts": [80, 443], + } + host["featureControls"] = { + "googleSending": "disabled", + "faqLiveMode": "disabled", + "pmsWrites": "disabled", + "paymentCreation": "disabled", + } + persistence = common("guestops-persistence") + persistence["drillCommand"] = "python3 deploy/ops.py persistence-drill --confirm-restart" + return host, persistence + + +class DebianAcceptanceTests(unittest.TestCase): + def test_complete_matching_records_pass(self): + acceptance.validate_pair(*valid_records(), COMMIT, RELEASE_SHA) + + def test_expected_release_identity_is_required(self): + host, persistence = valid_records() + host["releaseCommit"] = "f" * 40 + with self.assertRaisesRegex(ValueError, "approved candidate"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + host, persistence = valid_records() + persistence["releaseRecordSha256"] = "f" * 64 + with self.assertRaisesRegex(ValueError, "retained release record"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + + def test_exact_images_and_cross_record_identity_are_required(self): + host, persistence = valid_records() + host["images"]["api"]["reference"] = "guestops-api:latest" + with self.assertRaisesRegex(ValueError, "full approved commit"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + host, persistence = valid_records() + persistence["archiveSha256"] = "f" * 64 + with self.assertRaisesRegex(ValueError, "same archiveSha256"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + + def test_host_capacity_ports_and_disabled_controls_are_required(self): + host, persistence = valid_records() + host["hostFacts"]["publicTcpPorts"] = [80, 443, 8080] + with self.assertRaisesRegex(ValueError, "Only TCP ports"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + host, persistence = valid_records() + host["featureControls"]["googleSending"] = "enabled" + with self.assertRaisesRegex(ValueError, "must remain disabled"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + + def test_independent_review_scenarios_and_findings_are_required(self): + host, persistence = valid_records() + host["reviewedBy"] = host["operator"] + with self.assertRaisesRegex(ValueError, "different people"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + host, persistence = valid_records() + persistence["scenarios"][0]["status"] = "not-run" + with self.assertRaisesRegex(ValueError, "has not passed"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + host, persistence = valid_records() + host["unresolvedCriticalFindings"] = 1 + with self.assertRaisesRegex(ValueError, "critical findings"): + acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_monitor_status.py b/tests/test_monitor_status.py new file mode 100644 index 0000000..3324c9b --- /dev/null +++ b/tests/test_monitor_status.py @@ -0,0 +1,75 @@ +import datetime as dt +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +spec = importlib.util.spec_from_file_location( + "monitor_status", Path(__file__).resolve().parents[1] / "deploy" / "monitor_status.py") +monitor = importlib.util.module_from_spec(spec) +spec.loader.exec_module(monitor) + + +class MonitorStatusTests(unittest.TestCase): + def test_compose_json_is_reduced_to_safe_state(self): + value = json.dumps([ + {"Service": "api", "State": "running", "Health": "healthy", "Publishers": "secret"}, + {"Service": "worker", "State": "running", "Health": ""}, + {"Service": "mongo", "State": "running", "Health": "healthy"}, + ]).encode() + self.assertEqual(monitor.parse_compose(value)["api"], {"state": "running", "health": "healthy"}) + self.assertNotIn("Publishers", monitor.parse_compose(value)["api"]) + + def test_age_ignores_symlinks_and_unexpected_files(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + backup = directory / "guestops-20261001T021700Z.tar.gpg" + backup.write_bytes(b"fixture") + moment = dt.datetime(2026, 10, 1, 3, 17, tzinfo=dt.timezone.utc) + os.utime(backup, (moment.timestamp() - 3600, moment.timestamp() - 3600)) + (directory / "secret.txt").write_text("ignored", encoding="utf-8") + self.assertEqual(monitor.age_seconds(directory, monitor.BACKUP_NAME, moment), 3600) + + def test_build_status_contains_only_aggregate_health(self): + now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc) + compose = json.dumps([ + {"Service": name, "State": "running", "Health": "healthy"} + for name in ("api", "worker", "mongo") + ]).encode() + usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})() + with tempfile.TemporaryDirectory() as folder, \ + patch.object(monitor, "https_status", return_value={"ready": True, "certificateDaysRemaining": 60}), \ + patch.object(monitor, "run", return_value=compose), \ + patch.object(monitor, "worker_heartbeat_age", return_value=30), \ + patch.object(monitor.shutil, "disk_usage", return_value=usage): + status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now) + self.assertEqual(errors, []) + self.assertEqual(status["workerHeartbeatAgeSeconds"], 30) + self.assertEqual(status["diskFreePercent"], 50) + self.assertNotIn("credentials", json.dumps(status).lower()) + + def test_failed_probes_write_categories_not_exception_details(self): + now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc) + usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})() + with tempfile.TemporaryDirectory() as folder, \ + patch.object(monitor, "https_status", side_effect=RuntimeError("secret value")), \ + patch.object(monitor, "run", side_effect=RuntimeError("secret value")), \ + patch.object(monitor, "worker_heartbeat_age", side_effect=RuntimeError("secret value")), \ + patch.object(monitor.shutil, "disk_usage", return_value=usage): + status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now) + self.assertGreaterEqual(len(errors), 3) + self.assertNotIn("secret value", json.dumps(status)) + + def test_status_output_is_atomic_json(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "status.json" + monitor.write_status(path, {"schemaVersion": 1, "errors": []}) + self.assertEqual(json.loads(path.read_text(encoding="utf-8"))["schemaVersion"], 1) + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From 8588b058c5f86dd89120f1438edbf3285e93f545 Mon Sep 17 00:00:00 2001 From: mathew Date: Wed, 30 Sep 2026 19:59:28 +0100 Subject: [PATCH 03/11] removed git runner --- .env.example | 2 +- .github/workflows/web.yml | 98 ------------------------------------ MILESTONES.md | 11 ++-- README.md | 2 +- RELEASE_NOTES.md | 4 +- deploy/pilot_approval.py | 2 +- deploy/release_record.py | 2 +- deploy/verify_release.py | 2 +- docs/deployment.md | 23 ++++++--- docs/operations.md | 19 ++++--- docs/payments.md | 2 +- docs/pilot-release.md | 2 +- docs/replies.md | 2 +- tests/production_smoke.py | 6 +-- tests/test_release_record.py | 2 +- tests/test_verify_release.py | 2 +- 16 files changed, 47 insertions(+), 134 deletions(-) delete mode 100644 .github/workflows/web.yml diff --git a/.env.example b/.env.example index d385972..2fc0dee 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ AI_MODEL= # Optional private host-side JSON file binding internal hotel IDs to OHIP credentials. # Default example has no connections. Never commit the real configuration. PMS_CONFIG_FILE_HOST=./deploy/pms.example.json -# CI produces image archives. Set these to the loaded, reviewed commit tags. +# Ansible builds the reviewed source package. Set these to its full-commit image tags. GUESTOPS_API_IMAGE=guestops-api:local GUESTOPS_WORKER_IMAGE=guestops-worker:local diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml deleted file mode 100644 index 54064e4..0000000 --- a/.github/workflows/web.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Build and verify web migration -on: - push: - branches: [main, 'codex/**'] - tags: ['[0-9]+.[0-9]+.[0-9]+'] - pull_request: - workflow_dispatch: -permissions: - contents: read -jobs: - verify: - runs-on: ubuntu-latest - services: - mongo: - image: mongo:8.0 - ports: ['27017:27017'] - options: >- - --health-cmd "mongosh --quiet --eval 'db.adminCommand({ping:1}).ok'" - --health-interval 10s --health-timeout 5s --health-retries 10 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v4 - with: { dotnet-version: '10.0.x' } - - uses: actions/setup-node@v4 - with: { node-version: '22', cache: npm, cache-dependency-path: web/package-lock.json } - - name: Build services - run: dotnet build src/GuestOps.Worker/GuestOps.Worker.csproj -c Release - - name: Verify backup validation and failure recovery - run: python3 -m unittest discover -s tests -p 'test_*.py' - - name: Build interface - working-directory: web - run: npm ci && npm run build - - name: Start isolated preview API - run: | - ASPNETCORE_ENVIRONMENT=Development Preview=true dotnet src/GuestOps.Api/bin/Release/net10.0/GuestOps.Api.dll --urls http://127.0.0.1:5180 > /tmp/guestops-api.log 2>&1 & - for i in $(seq 1 30); do curl -fsS http://127.0.0.1:5180/health && exit 0; sleep 1; done - cat /tmp/guestops-api.log - exit 1 - - name: Verify MongoDB and HTTP boundaries - env: - MONGO_TEST_URI: mongodb://127.0.0.1:27017 - TEST_API_URL: http://127.0.0.1:5180 - run: dotnet run --project tests/GuestOps.Tests/GuestOps.Tests.csproj -c Release - - name: Build Linux images - run: | - docker build --target api -t guestops-api:${{ github.sha }} . - docker build --target worker -t guestops-worker:${{ github.sha }} . - - name: Package reviewed images - if: github.event_name != 'pull_request' - run: | - docker save guestops-api:${{ github.sha }} guestops-worker:${{ github.sha }} | gzip -n > guestops-images.tar.gz - python3 deploy/release_record.py \ - --artifact guestops-images.tar.gz \ - --commit '${{ github.sha }}' \ - --api-image 'guestops-api:${{ github.sha }}' \ - --api-id "$(docker image inspect --format '{{.Id}}' 'guestops-api:${{ github.sha }}')" \ - --worker-image 'guestops-worker:${{ github.sha }}' \ - --worker-id "$(docker image inspect --format '{{.Id}}' 'guestops-worker:${{ github.sha }}')" \ - --output release-record.json - sha256sum --check <(python3 -c "import json; r=json.load(open('release-record.json')); print(r['artifact']['sha256'] + ' ' + r['artifact']['name'])") - - name: Smoke test production containers and restart persistence - env: - GUESTOPS_API_IMAGE: guestops-api:${{ github.sha }} - GUESTOPS_WORKER_IMAGE: guestops-worker:${{ github.sha }} - BOOTSTRAP_EMAIL: ci-owner@example.invalid - BOOTSTRAP_HOTEL: CI test hotel - run: | - export MONGO_ROOT_PASSWORD=$(openssl rand -hex 32) - export MONGO_APP_PASSWORD=$(openssl rand -hex 32) - export BOOTSTRAP_PASSWORD=$(openssl rand -hex 24) - trap 'docker compose down --volumes' EXIT - docker compose config --quiet - docker compose up -d --no-build - curl --retry 30 --retry-delay 2 --retry-all-errors --fail http://127.0.0.1:8080/health - docker compose run --rm --no-deps -e BOOTSTRAP_EMAIL -e BOOTSTRAP_HOTEL -e BOOTSTRAP_PASSWORD api --bootstrap - python3 tests/production_smoke.py - docker compose restart api worker - curl --retry 30 --retry-delay 2 --retry-all-errors --fail http://127.0.0.1:8080/health - python3 tests/production_smoke.py --read - install -m 600 /dev/null .env - python3 deploy/ops.py preflight --offline - mkdir -m 700 .guestops-test-keyring - export GNUPGHOME="$PWD/.guestops-test-keyring" - gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key 'GuestOps CI ' rsa2048 encr 1d - BACKUP_RECIPIENT=$(gpg --batch --with-colons --list-keys | awk -F: '$1=="fpr" {print $10; exit}') - backup_dir=$(mktemp -d) - python3 deploy/ops.py backup --recipient "$BACKUP_RECIPIENT" --output "$backup_dir/fixture.tar.gpg" --confirm-maintenance - python3 deploy/ops.py restore-drill "$backup_dir/fixture.tar.gpg" --api-image "$GUESTOPS_API_IMAGE" - curl --retry 30 --retry-delay 2 --retry-all-errors --fail http://127.0.0.1:8080/health/ready - python3 tests/production_smoke.py --read - - uses: actions/upload-artifact@v4 - if: github.event_name != 'pull_request' - with: - name: guestops-linux-${{ github.run_number }} - path: | - guestops-images.tar.gz - release-record.json - retention-days: 90 diff --git a/MILESTONES.md b/MILESTONES.md index e4f5e12..03c346d 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -19,7 +19,7 @@ This summary explains what each milestone delivers and where it currently stands | 6 | Team onboarding and account recovery | Staff invitations, password setup/reset, access disable/restore, and administrator recovery. | **Implemented; acceptance required.** Deployed link delivery, expiry, recovery, and administrator procedures still need operational evidence. | | 7 | Google connection recovery | OAuth reconnect, checkpoint recovery, grant revocation handling, and worker restart safety. | **Implemented; acceptance required.** Dedicated Google-account and worker-restart exercises have not yet been accepted. | | 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | -| 9 | Gitea and reproducible releases | CI-built immutable images, checksummed release records, retained artifacts, and approval tagging. | **In progress.** The `0.2.0` candidate is frozen; successful exact-commit CI evidence and the post-Gate-B tag are still required. | +| 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The Gitea Action has been removed to match the CMS/CMSFront deployment model; the exact-commit package, Ansible playbook evidence, and post-Gate-B tag are still required. | | 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** Acceptance tooling is ready, but Docker, correct HTTPS/network exposure, exact artifacts, privileged installation, and the supervised drills remain open. | | 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** Repository tooling is ready; installation and timed operational evidence are blocked until Milestones 9 and 10 pass. | | 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | @@ -59,7 +59,7 @@ This summary explains what each milestone delivers and where it currently stands | 6 | Team onboarding and account recovery | B | Implemented / acceptance required | Invitation, password reset, and recovery flows are promoted to local `main`; verify deployed links, mail delivery, token expiry, and administrator recovery procedures. | | 7 | Google connection recovery | B | Implemented / acceptance required | Connection epochs, checkpoint recovery, and revocation handling are promoted to local `main`; complete real Google acceptance and worker-restart exercises. | | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | -| 9 | Gitea and reproducible releases | A | In progress | The `0.2.0` candidate is versioned on `main`. CI records the full commit, matched application version, archive checksum and immutable image IDs, and the rollback procedure is documented. Retain the successful default-branch evidence off-host and create the immutable approval tag only after Gate B approval; the existing `0.1.0` tag remains attached to the foundation release. | +| 9 | Gitea and reproducible releases | A | In progress | Package the exact versioned `main` commit as a checksummed source archive and hand it to a version-selected Ansible playbook, following the CMS/CMSFront pattern. Ansible installs the archive, builds commit-tagged images, records their immutable IDs, and deploys without using a Gitea runner. Retain the package/install evidence off-host and create the immutable approval tag only after Gate B approval; the existing `0.1.0` tag remains attached to the foundation release. | | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. | | 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling, checksum-verified rsync transfer, a restricted Zabbix status boundary, guarded local retention and a release-bound acceptance validator. Install and test it on Debian, configure the restricted store and alerts, name operational/review owners, and retain independently reviewed evidence from the timed restore and rollback drill. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | @@ -68,7 +68,7 @@ This summary explains what each milestone delivers and where it currently stands | 15 | Knowledge, AI, and FAQ activation | B | Implemented / acceptance required | Owners can run a bounded no-send batch evaluation, and a release-bound acceptance record enforces positive/negative coverage, zero FAQ errors, separate AI review, staff training, stop-control evidence and named monitoring/rollback owners. Complete the supervised evaluation and retain independent approval. | | 16 | Identity, preferences, and privacy | B/C | Implemented / acceptance required | Login throttling trusts the client address only after one-hop processing by the configured proxy. A release-bound review now covers owner-controlled preferences, account/session controls, data inventory, retention/deletion/legal-hold ownership, provider decisions, audit evidence and known identity limitations. Complete the legal/operational decisions and independently approve the record. | | 17 | Inbox usability and desktop parity | B | Implemented / acceptance required | The inbox uses tenant-scoped stable cursor pagination in pages of 50 and protects unsaved drafts during route/history navigation, reload, conversation selection, filtering and search. Operational timestamps use the saved hotel timezone, and a release-bound desktop-parity acceptance record is implemented. The implementation and preview HTTP suite pass; run the supervised exercise against the approved release and retain independent approval. | -| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.0` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision record validators with agreed targets. Push and retain CI evidence, complete Gate A and Gate B prerequisites, run the probe and supervised exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | +| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.0` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision record validators with agreed targets. Retain the exact source-package and Ansible-install evidence, complete Gate A and Gate B prerequisites, run the probe and supervised exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | ## Delivery sequence @@ -80,8 +80,9 @@ Milestones 13 (Guestline/Rezlynx) and 14 (payments) can progress as parallel pro ## Next actions -- [ ] Push the local release-candidate promotion to the intended default branch and retain its successful CI evidence. -- [ ] Retain successful `0.2.0` default-branch CI evidence, then create and archive the immutable approval tag only after Gate B approval. +- [ ] Merge the Action removal and release-process update to the intended default branch; that resulting commit becomes the new package candidate. +- [ ] Create and checksum the `0.2.0` source archive, add/select it in the GuestOps Ansible playbook, and retain the package and installation evidence. +- [ ] Create and archive the immutable `0.2.0` approval tag only after Gate B approval. - [ ] Deploy to the target Debian environment with persistent MongoDB and data-protection keys. - [ ] Run and record backup, restore, restart, monitoring, and rollback exercises. - [ ] Complete real Google mailbox acceptance without using production guest data. diff --git a/README.md b/README.md index 850406d..eb9761b 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ dotnet run --project tests/GuestOps.Tests cd web && npm ci && npm run build ``` -Set `MONGO_TEST_URI` to an isolated MongoDB server and `TEST_API_URL=http://127.0.0.1:5180` with a preview API running to enable database and HTTP integration checks. The suite creates and drops only its own randomly named `guestops_test_*` database. CI runs both integrations and builds both Linux images. +Set `MONGO_TEST_URI` to an isolated MongoDB server and `TEST_API_URL=http://127.0.0.1:5180` with a preview API running to enable database and HTTP integration checks. The suite creates and drops only its own randomly named `guestops_test_*` database. Run these checks before creating a versioned source package. GuestOps does not require a Gitea Actions runner; deployment follows the existing Futuresens source-package and Ansible process described in the [deployment guide](docs/deployment.md). See [controlled FAQ automation](docs/auto-replies.md), [NMI payment setup and recovery](docs/payments.md), [OHIP reservation setup and recovery](docs/pms.md), [AI drafts and reply delivery setup](docs/replies.md), [migration status](docs/migration.md) and [deployment guide](docs/deployment.md). diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e45796d..b87ba0c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ ## 0.2.0 — Gate B release candidate -This candidate freezes the implemented Gate B scope for controlled acceptance. It is not yet approved for live hotel operations and does not become a release until the exact commit is pushed, CI and operational evidence are retained, the supervised pilot is approved and the immutable `0.2.0` tag is created. +This candidate freezes the implemented Gate B scope for controlled acceptance. It is not yet approved for live hotel operations and does not become a release until the exact commit is pushed, a checksummed source package and operational evidence are retained, the supervised pilot is approved and the immutable `0.2.0` tag is created. ### Promoted scope @@ -19,7 +19,7 @@ These capabilities still require their separately documented provider, host and ### Known limitations and launch conditions -- Gate A still requires a successful default-branch CI run, durable off-host release archive, target-Debian deployment, persistent storage/key validation, monitoring, and a successful restore/rollback exercise. +- Gate A still requires a checksummed source package from the exact default-branch commit, Ansible installation on the target Debian host, persistent storage/key validation, monitoring, and a successful restore/rollback exercise. - Gate B still requires real Google acceptance and supervised staff testing, including desktop-parity acceptance of pagination, draft protection, proxy-aware login throttling and saved-hotel-timezone rendering. - Gate C still requires the Rezlynx/Guestline adapter and independently accepted PMS/payment workflows, plus privacy, identity, capacity, and release approvals. - FAQ live mode and all external write actions must remain disabled until their corresponding acceptance gate has passed. diff --git a/deploy/pilot_approval.py b/deploy/pilot_approval.py index d4a2fc8..047bdc8 100644 --- a/deploy/pilot_approval.py +++ b/deploy/pilot_approval.py @@ -11,7 +11,7 @@ import re GATE_B = { - "release-ci", "debian-host", "persistence", "backup-restore", "google-mailbox", + "release-package", "debian-host", "persistence", "backup-restore", "google-mailbox", "automation", "identity-privacy", "inbox-usability", "capacity", "incident-support", "pilot-findings", } diff --git a/deploy/release_record.py b/deploy/release_record.py index b6b86ff..54ad727 100644 --- a/deploy/release_record.py +++ b/deploy/release_record.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Create deterministic evidence for a reviewed GuestOps image archive.""" +"""Bind a reviewed GuestOps source package to its installed image identities.""" from __future__ import annotations diff --git a/deploy/verify_release.py b/deploy/verify_release.py index 597b26b..b356680 100644 --- a/deploy/verify_release.py +++ b/deploy/verify_release.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Verify a GuestOps release archive and its immutable CI release record.""" +"""Verify a GuestOps source package and its immutable release record.""" from __future__ import annotations diff --git a/docs/deployment.md b/docs/deployment.md index 65f9068..60f8a83 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -8,19 +8,30 @@ Check the existing Nginx, Docker, MongoDB and firewall configuration before inst Install Docker Engine/Compose and Nginx using their official Debian instructions. Keep SSH access unchanged. Only HTTPS/HTTP for this hostname need public access; port 8080 is loopback-only and MongoDB has no published port. -Clone the private repository using an authorized GitHub account. Place the checkout in a dedicated application directory. Copy `.env.example` to `.env`, set permissions to 600, and fill two different MongoDB passwords generated with `openssl rand -hex 32`. Use hex values so they are safe in the MongoDB URI. Store real values only on the server or in its secret-management system. +Install the versioned source package through the Futuresens Ansible repository. Do not clone GuestOps from the target server and do not place Gitea credentials on it. The playbook should extract the package into a dedicated application directory. Copy `.env.example` to `.env`, set permissions to 600, and fill two different MongoDB passwords generated with `openssl rand -hex 32`. Use hex values so they are safe in the MongoDB URI. Store real values only on the server or in its secret-management system. The MongoDB initialization script runs only on a new volume. Changing `.env` later does not rotate existing database users. Rotate those credentials through MongoDB administration and update application configuration together. -## 2. Load reviewed application images +## 2. Package and install with Ansible -CI builds API and worker images and packages them in a `guestops-linux-*` artifact. Download the successful artifact for the desired commit and transfer it to the sandbox through your normal authorized deployment process. +GuestOps follows the CMS/CMSFront deployment pattern: Gitea stores the application source, a specific committed version is compressed, and a version-selected Ansible playbook installs it. No Gitea Actions runner is required. + +On the trusted packaging machine, run the automated checks, select the full commit SHA, and create the archive from that committed tree rather than from a working directory: ```sh -docker load -i guestops-images.tar.gz +git archive --format=tar.gz --prefix=GuestOps-0.2.0/ \ + --output GuestOps-0.2.0.tar.gz FULL_40_CHARACTER_SHA +sha256sum GuestOps-0.2.0.tar.gz > GuestOps-0.2.0.tar.gz.sha256 ``` -Set `GUESTOPS_API_IMAGE=guestops-api:` and `GUESTOPS_WORKER_IMAGE=guestops-worker:` in `.env` using that exact build's SHA. Then run: +Store the archive and checksum under a versioned GuestOps files directory in the private Ansible repository. The GuestOps playbook and environment variables should select that version, copy and verify the archive, extract it into the application directory, preserve the private `.env` and provider configuration, and build images tagged with the full source commit: + +```sh +docker build --target api -t guestops-api:FULL_40_CHARACTER_SHA . +docker build --target worker -t guestops-worker:FULL_40_CHARACTER_SHA . +``` + +Record the resulting immutable image IDs and bind them to the source archive with `deploy/release_record.py`. Retain the source archive, checksum, release record, its checksum, build output, commit and Ansible run result in the restricted release store. Set `GUESTOPS_API_IMAGE=guestops-api:FULL_40_CHARACTER_SHA` and `GUESTOPS_WORKER_IMAGE=guestops-worker:FULL_40_CHARACTER_SHA` in `.env`. Then the playbook runs: ```sh docker compose config --quiet @@ -28,7 +39,7 @@ docker compose up -d --no-build curl --fail http://127.0.0.1:8080/health ``` -Do not run `docker compose config` without `--quiet` in shared logs: expanded configuration contains secrets. Local image builds are available with Compose for development, but CI builds avoid consuming sandbox resources. +Do not run `docker compose config` without `--quiet` in shared logs: expanded configuration contains secrets. Do not store `.env`, provider credentials, host inventory secrets, or restricted evidence in either application repository. Ansible must stop on a checksum, commit, version, build, or health-check mismatch. ## 3. Provision the first hotel owner diff --git a/docs/operations.md b/docs/operations.md index dcda4e1..c726d15 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -4,7 +4,7 @@ The owner-only **Workspace health** page reports database reachability, the work ## Release evidence and rollback -Every non-pull-request CI build packages the API and worker images under the full Git commit SHA. The accompanying `release-record.json` binds the archive checksum, application version, commit, image references and immutable Docker image IDs. Retain both files together in restricted off-host release storage; the CI artifact is a transfer mechanism, not the permanent archive. +Create the release source archive from an exact committed Gitea tree with `git archive`, then verify its SHA-256 before Ansible installs it. Ansible builds the API and worker images under the full Git commit SHA. The accompanying `release-record.json` binds the source-archive checksum, application version, commit, image references and immutable Docker image IDs. Retain the archive, checksum, record and Ansible result together in restricted off-host release storage. Before deployment, verify the archive against its record without loading it: @@ -18,29 +18,28 @@ print(r['commit'], r['version'], r['images']) PY ``` -The repository verifier performs the same checks strictly, also calculates the release-record checksum used by later acceptance records, and can compare the record with images already loaded on an isolated Docker host: +After Ansible has built the commit-tagged images, the repository verifier performs the same checks strictly, calculates the release-record checksum used by later acceptance records, and compares the record with the installed Docker images: ```sh python3 deploy/verify_release.py \ - --archive guestops-images.tar.gz \ + --archive GuestOps-0.2.0.tar.gz \ --record release-record.json \ - --commit a3ef408de61d895e69516fa3ba014b43621032bc \ + --commit FULL_40_CHARACTER_SHA \ --version 0.2.0 \ --output release-verification.json -gunzip -c guestops-images.tar.gz | docker load python3 deploy/verify_release.py \ - --archive guestops-images.tar.gz \ + --archive GuestOps-0.2.0.tar.gz \ --record release-record.json \ - --commit a3ef408de61d895e69516fa3ba014b43621032bc \ + --commit FULL_40_CHARACTER_SHA \ --version 0.2.0 \ --verify-loaded-images \ --output loaded-image-verification.json ``` -Retain both verification summaries with the untouched archive, release record, its SHA-256, and the exact default-branch CI metadata. A waiting, cancelled, failed, or branch-only run is not release evidence. Do not substitute a local rebuild for the archived default-branch images. +Retain both verification summaries with the untouched source archive, its checksum, the release record, its SHA-256, and the exact Ansible run metadata. A package from an uncommitted working tree, a mismatched checksum, or a failed Ansible run is not release evidence. Do not silently replace an approved package or rebuild under the same release identity. -Load the archive, verify each loaded image ID matches the record, set `GUESTOPS_API_IMAGE` and `GUESTOPS_WORKER_IMAGE` to the recorded full-SHA references, and run the deployment preflight. Record the CI run, commit, checksum and operator in the change ticket. A release tag is an approval marker; do not move or reuse an existing tag. The application and web versions must match before the record can be created. +Verify each installed image ID matches the record, set `GUESTOPS_API_IMAGE` and `GUESTOPS_WORKER_IMAGE` to the recorded full-SHA references, and run the deployment preflight. Record the Ansible run, commit, archive checksum and operator in the change ticket. A release tag is an approval marker; do not move or reuse an existing tag. The application and web versions must match before the package is accepted. For rollback, first disable worker-driven external writes and reconcile any sending, payment or PMS operation that may have completed since the prior release. Confirm the previous release archive and record are retained, verify its checksum and image IDs, take an encrypted backup, then select the previous recorded image references in `.env` and recreate only the API and worker. Do not roll back MongoDB or the key volume merely to change application images. Run the online preflight, readiness check and read-only smoke test before re-enabling the worker or provider writes. If a release introduced an incompatible data change, follow its release-specific recovery plan rather than starting an older image against newer data. @@ -182,7 +181,7 @@ Restore into new isolated MongoDB and key volumes; preserve the damaged original **A restored database can predate emails, invoices and PMS changes that providers already completed.** Review pending, sending and uncertain records against provider evidence before enabling any worker, including automatic FAQ rules. Do not replay an older approval merely because the restored record says it is pending. Reconcile external effects, validate account sessions and mailbox authorization, and explicitly approve the cutover only after these checks. Rotate credentials if compromise prompted the recovery. Keep the old deployment stopped when enabling the replacement. -CI exercises a synthetic encrypted backup and isolated restore drill, including actual key decryption and database comparison. A successful CI drill is separate from the required rehearsal on the Debian server with its actual deployment configuration. +Run the synthetic encrypted backup and isolated restore drill in a controlled test environment, including actual key decryption and database comparison. This automated check is separate from the required rehearsal on the Debian server with its actual deployment configuration. ## Milestone 11 acceptance record diff --git a/docs/payments.md b/docs/payments.md index 1356b52..0bcef45 100644 --- a/docs/payments.md +++ b/docs/payments.md @@ -46,6 +46,6 @@ Only unsubmitted proposals can be cancelled in GuestOps. Invoice closure, refund ## Acceptance before live use -Automated tests use an in-process fake HTTP handler and never contact NMI. They cover tenant isolation, amount/currency/identity mismatches, partial status, concurrent approvals, lost create responses, pagination and merchant changes. CI checks the production configuration mount and disabled defaults. +Automated tests use an in-process fake HTTP handler and never contact NMI. They cover tenant isolation, amount/currency/identity mismatches, partial status, concurrent approvals, lost create responses, pagination and merchant changes. Run the production-configuration and disabled-default checks before packaging and again during the Ansible deployment verification. Use a dedicated sandbox merchant and recipient to validate authentication, supported currency, exact request/response fields, preservation of `order_details.order_id`, customer invoice email and hosted checkout, partial/full payments and interrupted-request recovery. Live acceptance remains pending. Planet, direct payment links in GuestOps replies, automatic booking after payment, automated expiry/closure and background polling are follow-on work. diff --git a/docs/pilot-release.md b/docs/pilot-release.md index 6e274ab..9de0939 100644 --- a/docs/pilot-release.md +++ b/docs/pilot-release.md @@ -41,7 +41,7 @@ The example deliberately fails while daily reviews are `not-run`. A structurally The go/no-go record must bind all evidence to the same release commit and release-record checksum. Record named owners, dates, evidence locations, findings and explicit dispositions for: -- default-branch CI and immutable release archive; +- exact-commit source package, checksum, recorded image IDs, and successful Ansible installation; - Debian preflight, HTTPS, persistence and controlled reboot; - encrypted off-host backup and timed isolated restore; - Google mailbox and reviewed-send acceptance; diff --git a/docs/replies.md b/docs/replies.md index ba448e1..b5a3280 100644 --- a/docs/replies.md +++ b/docs/replies.md @@ -34,6 +34,6 @@ Current limitations: messages imported before reply headers were stored must be ## Verification -The test suite covers competing workers, send identity/thread headers, invalid recipients, header injection, uncertain outcomes, restart recovery, pre-send token failure, disabling hotel sending, cross-hotel access, Gmail reconciliation mismatches, AI source isolation, invalid citations, escalations and incomplete responses. CI also runs production container login/restart-persistence smoke checks. Live acceptance must additionally verify Google consent, actual threading, grant revocation, a representative AI draft evaluation set and provider error behaviour with the configured accounts. +The test suite covers competing workers, send identity/thread headers, invalid recipients, header injection, uncertain outcomes, restart recovery, pre-send token failure, disabling hotel sending, cross-hotel access, Gmail reconciliation mismatches, AI source isolation, invalid citations, escalations and incomplete responses. Run the production-container login and restart-persistence smoke checks before packaging and during Ansible deployment verification. Live acceptance must additionally verify Google consent, actual threading, grant revocation, a representative AI draft evaluation set and provider error behaviour with the configured accounts. Implementation references: [OpenAI Structured Outputs](https://developers.openai.com/api/docs/guides/structured-outputs), [Gmail sending](https://developers.google.com/workspace/gmail/api/guides/sending), [Gmail threads](https://developers.google.com/workspace/gmail/api/guides/threads). diff --git a/tests/production_smoke.py b/tests/production_smoke.py index 7b59084..39666a0 100644 --- a/tests/production_smoke.py +++ b/tests/production_smoke.py @@ -1,4 +1,4 @@ -"""Exercise disposable CI containers through the host's trusted proxy address. +"""Exercise disposable deployment containers through the host's trusted proxy address. The forwarded HTTPS header simulates Nginx TLS termination; this is never run against an existing hotel database. Cookie values and credentials are not logged. @@ -66,12 +66,12 @@ team = request("/api/team") assert all("passwordHash" not in member and "securityStamp" not in member and "accountLinkHash" not in member for member in team) if "--read" in sys.argv: assert hotel["signature"] == "Persisted across container restart" - invited = next(member for member in team if member["email"] == "ci-staff@example.invalid") + invited = next(member for member in team if member["email"] == "deployment-staff@example.invalid") assert invited["pending"] and not invited["active"] and invited["linkPurpose"] == "Invite" else: hotel["signature"] = "Persisted across container restart" request("/api/hotel", "PUT", hotel) - invite = request("/api/team/invite", "POST", {"name": "CI Staff", "email": "ci-staff@example.invalid"}) + invite = request("/api/team/invite", "POST", {"name": "Deployment Staff", "email": "deployment-staff@example.invalid"}) assert invite["link"].startswith("https://sandbox-guestops.futuresens.co.uk/account#token=") request("/api/auth/logout", "POST") request("/api/hotel", expected=401) diff --git a/tests/test_release_record.py b/tests/test_release_record.py index 000a387..072eb30 100644 --- a/tests/test_release_record.py +++ b/tests/test_release_record.py @@ -13,7 +13,7 @@ ROOT = Path(__file__).resolve().parents[1] class ReleaseRecordTests(unittest.TestCase): def test_writes_versions_checksum_and_immutable_image_ids(self): with tempfile.TemporaryDirectory() as directory: - artifact = Path(directory) / "guestops-images.tar.gz" + artifact = Path(directory) / "GuestOps-0.2.0.tar.gz" output = Path(directory) / "release-record.json" artifact.write_bytes(b"reviewed image archive") diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py index 0286f70..a0c6a13 100644 --- a/tests/test_verify_release.py +++ b/tests/test_verify_release.py @@ -21,7 +21,7 @@ WORKER_ID = "sha256:" + "c" * 64 class VerifyReleaseTests(unittest.TestCase): def fixture(self, directory: str): root = Path(directory) - archive = root / "guestops-images.tar.gz" + archive = root / "GuestOps-0.2.0.tar.gz" record = root / "release-record.json" archive.write_bytes(b"reviewed image archive") release = { -- 2.20.1 From 9c29bfcd862ed6ff2e1d2d8727780022a2088ab9 Mon Sep 17 00:00:00 2001 From: mathew Date: Wed, 30 Sep 2026 21:03:16 +0100 Subject: [PATCH 04/11] Add deterministic source release packaging --- MILESTONES.md | 7 +- README.md | 2 + deploy/package_source.py | 124 +++++++++++++++++++++++++++++++++++ docs/deployment.md | 14 ++-- docs/operations.md | 15 +++-- tests/test_package_source.py | 88 +++++++++++++++++++++++++ 6 files changed, 238 insertions(+), 12 deletions(-) create mode 100644 deploy/package_source.py create mode 100644 tests/test_package_source.py diff --git a/MILESTONES.md b/MILESTONES.md index 03c346d..02a4e3e 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -19,7 +19,7 @@ This summary explains what each milestone delivers and where it currently stands | 6 | Team onboarding and account recovery | Staff invitations, password setup/reset, access disable/restore, and administrator recovery. | **Implemented; acceptance required.** Deployed link delivery, expiry, recovery, and administrator procedures still need operational evidence. | | 7 | Google connection recovery | OAuth reconnect, checkpoint recovery, grant revocation handling, and worker restart safety. | **Implemented; acceptance required.** Dedicated Google-account and worker-restart exercises have not yet been accepted. | | 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | -| 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The Gitea Action has been removed to match the CMS/CMSFront deployment model; the exact-commit package, Ansible playbook evidence, and post-Gate-B tag are still required. | +| 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The runner-free deterministic packager is implemented and the Gitea Action is removed; the release-line identity must be confirmed, then the package, Ansible installation evidence, and approval record must be retained. | | 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** Acceptance tooling is ready, but Docker, correct HTTPS/network exposure, exact artifacts, privileged installation, and the supervised drills remain open. | | 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** Repository tooling is ready; installation and timed operational evidence are blocked until Milestones 9 and 10 pass. | | 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | @@ -59,7 +59,7 @@ This summary explains what each milestone delivers and where it currently stands | 6 | Team onboarding and account recovery | B | Implemented / acceptance required | Invitation, password reset, and recovery flows are promoted to local `main`; verify deployed links, mail delivery, token expiry, and administrator recovery procedures. | | 7 | Google connection recovery | B | Implemented / acceptance required | Connection epochs, checkpoint recovery, and revocation handling are promoted to local `main`; complete real Google acceptance and worker-restart exercises. | | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | -| 9 | Gitea and reproducible releases | A | In progress | Package the exact versioned `main` commit as a checksummed source archive and hand it to a version-selected Ansible playbook, following the CMS/CMSFront pattern. Ansible installs the archive, builds commit-tagged images, records their immutable IDs, and deploys without using a Gitea runner. Retain the package/install evidence off-host and create the immutable approval tag only after Gate B approval; the existing `0.1.0` tag remains attached to the foundation release. | +| 9 | Gitea and reproducible releases | A | In progress | `deploy/package_source.py` now packages only an explicit committed ref, verifies matched application versions, produces deterministic gzip output and a SHA-256 source record, and refuses overwrite. Hand that package to a version-selected Ansible playbook following the CMS/CMSFront pattern. Ansible must verify and install it, build commit-tagged images, record their immutable IDs, and deploy without a Gitea runner. Retain the package/install evidence off-host and resolve the release-line/tag identity before approval; the existing `0.1.0` tag remains attached to the foundation release. | | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. | | 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling, checksum-verified rsync transfer, a restricted Zabbix status boundary, guarded local retention and a release-bound acceptance validator. Install and test it on Debian, configure the restricted store and alerts, name operational/review owners, and retain independently reviewed evidence from the timed restore and rollback drill. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | @@ -81,7 +81,8 @@ Milestones 13 (Guestline/Rezlynx) and 14 (payments) can progress as parallel pro ## Next actions - [ ] Merge the Action removal and release-process update to the intended default branch; that resulting commit becomes the new package candidate. -- [ ] Create and checksum the `0.2.0` source archive, add/select it in the GuestOps Ansible playbook, and retain the package and installation evidence. +- [ ] Confirm the intended release commit/version because remote `main` and the published `0.2.0` tag currently identify different histories; do not move or reuse the published tag. +- [ ] Create and checksum the approved source archive with `deploy/package_source.py`, add/select it in the GuestOps Ansible playbook, and retain the package and installation evidence. - [ ] Create and archive the immutable `0.2.0` approval tag only after Gate B approval. - [ ] Deploy to the target Debian environment with persistent MongoDB and data-protection keys. - [ ] Run and record backup, restore, restart, monitoring, and rollback exercises. diff --git a/README.md b/README.md index eb9761b..70507ec 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ For MongoDB-backed operation, disable Preview and set `Mongo__ConnectionString`, Operational tooling includes an owner-only Workspace health page and Linux deployment preflight, encrypted backup and isolated restore drill. See [operations and recovery](docs/operations.md) before the hotel pilot. +Release packaging is runner-free. `deploy/package_source.py` creates a deterministic archive and checksum record from one explicit committed Git ref for handoff to the Futuresens Ansible deployment. It never packages uncommitted working-tree files. + ## Verification ```sh diff --git a/deploy/package_source.py b/deploy/package_source.py new file mode 100644 index 0000000..b3b48f3 --- /dev/null +++ b/deploy/package_source.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Create a deterministic GuestOps source package from one committed Git tree.""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +from pathlib import Path +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + + +SEMVER = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +FULL_SHA = re.compile(r"[0-9a-f]{40}") + + +def git(repository: Path, *arguments: str, binary: bool = False) -> bytes | str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + text=not binary, + ) + return result.stdout if binary else result.stdout.strip() + + +def committed_text(repository: Path, commit: str, path: str) -> str: + return str(git(repository, "show", f"{commit}:{path}")) + + +def versions(repository: Path, commit: str) -> tuple[str, str]: + props = ET.fromstring(committed_text(repository, commit, "Directory.Build.props")) + version_node = props.find(".//Version") + if version_node is None or not version_node.text: + raise ValueError("The committed Directory.Build.props has no Version element.") + dotnet_version = version_node.text.strip() + package = json.loads(committed_text(repository, commit, "web/package.json")) + web_version = str(package.get("version", "")) + return dotnet_version, web_version + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def build_package(repository: Path, ref: str, output_directory: Path) -> tuple[Path, Path, dict[str, object]]: + repository = repository.resolve() + output_directory = output_directory.resolve() + if not (repository / ".git").exists(): + raise ValueError("--repository must be a Git working tree.") + + commit = str(git(repository, "rev-parse", "--verify", f"{ref}^{{commit}}")).lower() + if FULL_SHA.fullmatch(commit) is None: + raise ValueError("The selected ref did not resolve to a full Git commit SHA.") + + dotnet_version, web_version = versions(repository, commit) + if dotnet_version != web_version: + raise ValueError( + f"Committed release versions differ: .NET={dotnet_version}, web={web_version}." + ) + if SEMVER.fullmatch(dotnet_version) is None: + raise ValueError("The committed version must use MAJOR.MINOR.PATCH.") + + prefix = f"GuestOps-{dotnet_version}/" + tar_bytes = bytes( + git( + repository, + "archive", + "--format=tar", + f"--prefix={prefix}", + commit, + binary=True, + ) + ) + compressed = io.BytesIO() + with gzip.GzipFile(fileobj=compressed, mode="wb", filename="", mtime=0) as stream: + stream.write(tar_bytes) + archive_bytes = compressed.getvalue() + + stem = f"GuestOps-{dotnet_version}-{commit[:12]}" + archive = output_directory / f"{stem}.tar.gz" + record_path = output_directory / f"{stem}.source.json" + if archive.exists() or record_path.exists(): + raise ValueError("The output package or source record already exists; releases are not overwritten.") + + output_directory.mkdir(parents=True, exist_ok=True) + archive.write_bytes(archive_bytes) + record: dict[str, object] = { + "artifact": { + "name": archive.name, + "sha256": sha256_bytes(archive_bytes), + "size": len(archive_bytes), + }, + "commit": commit, + "schemaVersion": 1, + "version": dotnet_version, + } + record_path.write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return archive, record_path, record + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", required=True, help="Exact tag, branch, or full commit to package.") + parser.add_argument("--repository", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output-directory", required=True, type=Path) + args = parser.parse_args() + try: + archive, record, details = build_package(args.repository, args.ref, args.output_directory) + except (OSError, ValueError, ET.ParseError, json.JSONDecodeError, subprocess.SubprocessError) as error: + print(f"Source packaging failed: {error}", file=sys.stderr) + raise SystemExit(1) + print(json.dumps({"archive": str(archive), "record": str(record), **details}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/deployment.md b/docs/deployment.md index 60f8a83..9b96b42 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -16,15 +16,19 @@ The MongoDB initialization script runs only on a new volume. Changing `.env` lat GuestOps follows the CMS/CMSFront deployment pattern: Gitea stores the application source, a specific committed version is compressed, and a version-selected Ansible playbook installs it. No Gitea Actions runner is required. -On the trusted packaging machine, run the automated checks, select the full commit SHA, and create the archive from that committed tree rather than from a working directory: +On the trusted packaging machine, fetch the current remote refs, run the automated checks, select the full commit SHA, and create the archive from that committed tree rather than from a working directory: ```sh -git archive --format=tar.gz --prefix=GuestOps-0.2.0/ \ - --output GuestOps-0.2.0.tar.gz FULL_40_CHARACTER_SHA -sha256sum GuestOps-0.2.0.tar.gz > GuestOps-0.2.0.tar.gz.sha256 +git fetch origin --prune --tags +git rev-parse origin/main +python3 deploy/package_source.py \ + --ref FULL_40_CHARACTER_SHA \ + --output-directory /secure/release-staging ``` -Store the archive and checksum under a versioned GuestOps files directory in the private Ansible repository. The GuestOps playbook and environment variables should select that version, copy and verify the archive, extract it into the application directory, preserve the private `.env` and provider configuration, and build images tagged with the full source commit: +The packaging command resolves the ref to a full commit, reads both application versions from that committed tree, requires them to match, creates deterministic gzip output, and writes a `.source.json` record containing the version, commit, filename, size and SHA-256. Existing package files are never overwritten. Independently confirm that the selected commit is the intended release line before transferring it. + +Store the archive and source record under a versioned GuestOps files directory in the private Ansible repository. The GuestOps playbook and environment variables should select that version, verify the archive against the source record, extract it into the application directory, preserve the private `.env` and provider configuration, and build images tagged with the full source commit: ```sh docker build --target api -t guestops-api:FULL_40_CHARACTER_SHA . diff --git a/docs/operations.md b/docs/operations.md index c726d15..54516c6 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -4,17 +4,24 @@ The owner-only **Workspace health** page reports database reachability, the work ## Release evidence and rollback -Create the release source archive from an exact committed Gitea tree with `git archive`, then verify its SHA-256 before Ansible installs it. Ansible builds the API and worker images under the full Git commit SHA. The accompanying `release-record.json` binds the source-archive checksum, application version, commit, image references and immutable Docker image IDs. Retain the archive, checksum, record and Ansible result together in restricted off-host release storage. +Create the release source archive from an exact committed Gitea tree with `deploy/package_source.py`, then verify its generated source record before Ansible installs it. The tool packages only the selected commit, produces deterministic gzip output, rejects version disagreement, and refuses to overwrite an existing release. Ansible builds the API and worker images under the full Git commit SHA. The post-build `release-record.json` binds the source-archive checksum, application version, commit, image references and immutable Docker image IDs. Retain the archive, source record, release record and Ansible result together in restricted off-host release storage. -Before deployment, verify the archive against its record without loading it: +```sh +git fetch origin --prune --tags +python3 deploy/package_source.py \ + --ref FULL_40_CHARACTER_SHA \ + --output-directory /secure/release-staging +``` + +Before deployment, verify the archive against its source record: ```sh python3 - <<'PY' import hashlib, json, pathlib -r = json.load(open('release-record.json', encoding='utf-8')) +r = json.load(open('GuestOps-0.2.0-COMMIT.source.json', encoding='utf-8')) p = pathlib.Path(r['artifact']['name']) assert hashlib.sha256(p.read_bytes()).hexdigest() == r['artifact']['sha256'] -print(r['commit'], r['version'], r['images']) +print(r['commit'], r['version'], r['artifact']) PY ``` diff --git a/tests/test_package_source.py b/tests/test_package_source.py new file mode 100644 index 0000000..72d3d6e --- /dev/null +++ b/tests/test_package_source.py @@ -0,0 +1,88 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("package_source", ROOT / "deploy" / "package_source.py") +package_source = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package_source) + + +def run(repository: Path, *arguments: str) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +class PackageSourceTests(unittest.TestCase): + def repository(self, root: Path, dotnet_version: str = "1.2.3", web_version: str = "1.2.3") -> tuple[Path, str]: + repository = root / "repository" + (repository / "web").mkdir(parents=True) + run(repository, "init") + run(repository, "config", "user.name", "GuestOps Test") + run(repository, "config", "user.email", "guestops@example.invalid") + run(repository, "config", "core.autocrlf", "false") + (repository / "Directory.Build.props").write_text( + f"{dotnet_version}\n", + encoding="utf-8", + ) + (repository / "web" / "package.json").write_text( + json.dumps({"version": web_version}) + "\n", encoding="utf-8" + ) + (repository / "application.txt").write_text("committed application\n", encoding="utf-8") + run(repository, "add", ".") + run(repository, "commit", "-m", "fixture") + return repository, run(repository, "rev-parse", "HEAD") + + def test_packages_only_the_selected_commit_and_records_identity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository, commit = self.repository(root) + (repository / "application.txt").write_text("uncommitted change\n", encoding="utf-8") + + archive, record_path, record = package_source.build_package(repository, commit, root / "output") + + self.assertEqual(record["commit"], commit) + self.assertEqual(record["version"], "1.2.3") + self.assertEqual(record["artifact"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) + self.assertEqual(json.loads(record_path.read_text(encoding="utf-8")), record) + with tarfile.open(archive, "r:gz") as package: + member = package.extractfile("GuestOps-1.2.3/application.txt") + self.assertIsNotNone(member) + self.assertEqual(member.read().decode("utf-8").strip(), "committed application") + + def test_same_commit_produces_identical_package(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository, commit = self.repository(root) + first, _, _ = package_source.build_package(repository, commit, root / "first") + second, _, _ = package_source.build_package(repository, commit, root / "second") + self.assertEqual(first.read_bytes(), second.read_bytes()) + + def test_rejects_mismatched_versions_and_existing_output(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository, commit = self.repository(root, web_version="1.2.4") + with self.assertRaisesRegex(ValueError, "versions differ"): + package_source.build_package(repository, commit, root / "output") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + repository, commit = self.repository(root) + package_source.build_package(repository, commit, root / "output") + with self.assertRaisesRegex(ValueError, "not overwritten"): + package_source.build_package(repository, commit, root / "output") + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From 9971a6363526ed863378a67bf6f0b3d7e4bb46ff Mon Sep 17 00:00:00 2001 From: mathew Date: Wed, 30 Sep 2026 21:06:15 +0100 Subject: [PATCH 05/11] Prepare clean 0.2.1 Gate B candidate --- Directory.Build.props | 2 +- MILESTONES.md | 10 +++++----- README.md | 2 +- RELEASE_NOTES.md | 10 +++++++--- deploy/backup-restore-acceptance.example.json | 2 +- deploy/backup_restore_acceptance.py | 2 +- deploy/debian-host-acceptance.example.json | 2 +- deploy/debian_acceptance.py | 2 +- deploy/persistence-acceptance.example.json | 2 +- deploy/pilot_approval.py | 2 +- docs/gate-b-prerequisites.md | 2 +- docs/operations.md | 10 +++++----- docs/pilot-release.md | 2 +- tests/test_backup_restore_acceptance.py | 2 +- tests/test_debian_acceptance.py | 2 +- tests/test_release_record.py | 4 ++-- tests/test_verify_release.py | 14 +++++++------- web/package-lock.json | 4 ++-- web/package.json | 2 +- 19 files changed, 41 insertions(+), 37 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 4ad83ea..8ccdb6b 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ net10.0 - 0.2.0 + 0.2.1 enable enable true diff --git a/MILESTONES.md b/MILESTONES.md index 02a4e3e..69435b6 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -1,6 +1,6 @@ # GuestOps Milestone Report -Version: **0.2.0 release candidate** +Version: **0.2.1 release candidate** Last updated: **30 September 2026** This is the working delivery tracker for GuestOps Web. Update a milestone when its state changes and link the pull request, release artifact, test run, or acceptance record that proves the change. @@ -29,7 +29,7 @@ This summary explains what each milestone delivers and where it currently stands | 16 | Identity, preferences, and privacy | Account/session controls, hotel preferences, privacy inventory, retention decisions, and audit review. | **Implemented; acceptance required.** Legal and operational decisions, identity checks, and independent review remain outstanding. | | 17 | Inbox usability and desktop parity | Stable pagination, protected unsaved drafts, hotel-timezone display, and desktop workflow parity. | **Implemented; acceptance required.** Automated checks pass; the supervised desktop exercise and independent approval remain outstanding. | | 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** Validators and targets exist; Gate A/B prerequisites, capacity evidence, incident rehearsal, pilot, and named approvals remain open. | -| 19 | Account security and self-service | TOTP MFA, recovery codes, transactional email, granular roles, preferences, and security notifications. | **Implemented on the development branch; acceptance required.** Keep it separate until `0.2.0` is approved and tagged, then review, merge, and version it as `0.3.0`. | +| 19 | Account security and self-service | TOTP MFA, recovery codes, transactional email, granular roles, preferences, and security notifications. | **Implemented on the development branch; acceptance required.** Keep it separate until `0.2.1` is approved and tagged, then review, merge, and version it as `0.3.0`. | ## Status key @@ -68,7 +68,7 @@ This summary explains what each milestone delivers and where it currently stands | 15 | Knowledge, AI, and FAQ activation | B | Implemented / acceptance required | Owners can run a bounded no-send batch evaluation, and a release-bound acceptance record enforces positive/negative coverage, zero FAQ errors, separate AI review, staff training, stop-control evidence and named monitoring/rollback owners. Complete the supervised evaluation and retain independent approval. | | 16 | Identity, preferences, and privacy | B/C | Implemented / acceptance required | Login throttling trusts the client address only after one-hop processing by the configured proxy. A release-bound review now covers owner-controlled preferences, account/session controls, data inventory, retention/deletion/legal-hold ownership, provider decisions, audit evidence and known identity limitations. Complete the legal/operational decisions and independently approve the record. | | 17 | Inbox usability and desktop parity | B | Implemented / acceptance required | The inbox uses tenant-scoped stable cursor pagination in pages of 50 and protects unsaved drafts during route/history navigation, reload, conversation selection, filtering and search. Operational timestamps use the saved hotel timezone, and a release-bound desktop-parity acceptance record is implemented. The implementation and preview HTTP suite pass; run the supervised exercise against the approved release and retain independent approval. | -| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.0` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision record validators with agreed targets. Retain the exact source-package and Ansible-install evidence, complete Gate A and Gate B prerequisites, run the probe and supervised exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | +| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.1` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision record validators with agreed targets. Retain the exact source-package and Ansible-install evidence, complete Gate A and Gate B prerequisites, run the probe and supervised exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | ## Delivery sequence @@ -81,9 +81,9 @@ Milestones 13 (Guestline/Rezlynx) and 14 (payments) can progress as parallel pro ## Next actions - [ ] Merge the Action removal and release-process update to the intended default branch; that resulting commit becomes the new package candidate. -- [ ] Confirm the intended release commit/version because remote `main` and the published `0.2.0` tag currently identify different histories; do not move or reuse the published tag. +- [x] Preserve the published `0.2.0` tag unchanged and use the clean `0.2.1` candidate line from `main`, excluding Milestone 19 application code. - [ ] Create and checksum the approved source archive with `deploy/package_source.py`, add/select it in the GuestOps Ansible playbook, and retain the package and installation evidence. -- [ ] Create and archive the immutable `0.2.0` approval tag only after Gate B approval. +- [ ] Create and archive the immutable `0.2.1` approval tag only after Gate B approval. - [ ] Deploy to the target Debian environment with persistent MongoDB and data-protection keys. - [ ] Run and record backup, restore, restart, monitoring, and rollback exercises. - [ ] Complete real Google mailbox acceptance without using production guest data. diff --git a/README.md b/README.md index 70507ec..c04c7fd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A Linux-hosted hotel email workspace, developed separately from the Windows GuestOps application. **This migration now includes AI draft generation, staff-approved Gmail sending, reviewed OHIP reservation updates, NMI hosted invoices, controlled FAQ auto-replies and team onboarding. It is not yet a production-complete replacement.** -Current release-candidate version: **0.2.0** +Current release-candidate version: **0.2.1** Project progress is tracked in the [milestone report](MILESTONES.md). User-visible changes and release limitations are recorded in the [release notes](RELEASE_NOTES.md). diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index b87ba0c..9d6e5cf 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,8 +1,8 @@ # GuestOps Release Notes -## 0.2.0 — Gate B release candidate +## 0.2.1 — Gate B release candidate -This candidate freezes the implemented Gate B scope for controlled acceptance. It is not yet approved for live hotel operations and does not become a release until the exact commit is pushed, a checksummed source package and operational evidence are retained, the supervised pilot is approved and the immutable `0.2.0` tag is created. +This candidate freezes the implemented Gate B scope for controlled acceptance. It is not yet approved for live hotel operations and does not become a release until the exact commit is pushed, a checksummed source package and operational evidence are retained, the supervised pilot is approved and the immutable `0.2.1` tag is created. ### Promoted scope @@ -26,6 +26,10 @@ These capabilities still require their separately documented provider, host and See [MILESTONES.md](MILESTONES.md) for the gate assessment, delivery sequence, and remaining work. +## 0.2.0 — Superseded candidate identifier + +The published `0.2.0` tag is retained unchanged for auditability but is not the approved deployment candidate. It identifies a development-line commit that is not on the clean Gate B release branch. Do not package or deploy it; `0.2.1` supersedes it. + ## 0.1.0 — 29 September 2026 The `0.1.0` tag identifies the initial GuestOps Web foundation. It is not approved for live hotel operations. @@ -41,4 +45,4 @@ The `0.1.0` tag identifies the initial GuestOps Web foundation. It is not approv ### Versioning -The foundation remains tagged `0.1.0`. The .NET projects and frontend package now share candidate version `0.2.0`; create that immutable tag only after the exact commit, checksummed artifacts and Gate B acceptance evidence have been approved. +The foundation remains tagged `0.1.0`. The .NET projects and frontend package now share candidate version `0.2.1`; create that immutable tag only after the exact commit, checksummed artifacts and Gate B acceptance evidence have been approved. diff --git a/deploy/backup-restore-acceptance.example.json b/deploy/backup-restore-acceptance.example.json index ff32ff1..5a51045 100644 --- a/deploy/backup-restore-acceptance.example.json +++ b/deploy/backup-restore-acceptance.example.json @@ -3,7 +3,7 @@ "system": "guestops-backup-recovery", "evidenceId": "backup-restore", "dataClassification": "synthetic-only", - "releaseVersion": "0.2.0", + "releaseVersion": "0.2.1", "releaseCommit": "0000000000000000000000000000000000000000", "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", diff --git a/deploy/backup_restore_acceptance.py b/deploy/backup_restore_acceptance.py index 2f76bc3..ea74e55 100644 --- a/deploy/backup_restore_acceptance.py +++ b/deploy/backup_restore_acceptance.py @@ -11,7 +11,7 @@ import re from urllib.parse import urlparse -VERSION = "0.2.0" +VERSION = "0.2.1" SCENARIOS = { "encrypted-manual-backup", "scheduled-backup", diff --git a/deploy/debian-host-acceptance.example.json b/deploy/debian-host-acceptance.example.json index 544b96b..48e226f 100644 --- a/deploy/debian-host-acceptance.example.json +++ b/deploy/debian-host-acceptance.example.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "system": "guestops-debian-host", "evidenceId": "debian-host", - "releaseVersion": "0.2.0", + "releaseVersion": "0.2.1", "releaseCommit": "0000000000000000000000000000000000000000", "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", diff --git a/deploy/debian_acceptance.py b/deploy/debian_acceptance.py index d0ca02e..250d8fa 100644 --- a/deploy/debian_acceptance.py +++ b/deploy/debian_acceptance.py @@ -11,7 +11,7 @@ import re from urllib.parse import urlparse -VERSION = "0.2.0" +VERSION = "0.2.1" SYSTEMS = { "guestops-debian-host": { "evidenceId": "debian-host", diff --git a/deploy/persistence-acceptance.example.json b/deploy/persistence-acceptance.example.json index bc34f89..8dc4ff7 100644 --- a/deploy/persistence-acceptance.example.json +++ b/deploy/persistence-acceptance.example.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "system": "guestops-persistence", "evidenceId": "persistence", - "releaseVersion": "0.2.0", + "releaseVersion": "0.2.1", "releaseCommit": "0000000000000000000000000000000000000000", "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", diff --git a/deploy/pilot_approval.py b/deploy/pilot_approval.py index 047bdc8..f5681aa 100644 --- a/deploy/pilot_approval.py +++ b/deploy/pilot_approval.py @@ -34,7 +34,7 @@ def timestamp(value: object, field: str) -> dt.datetime: def validate(record: object) -> None: require(isinstance(record, dict), "Approval record must be a JSON object.") - require(record.get("schemaVersion") == 2, "Unsupported approval schema; Gate B 0.2.0 requires schemaVersion 2.") + require(record.get("schemaVersion") == 2, "Unsupported approval schema; Gate B 0.2.1 requires schemaVersion 2.") gate = record.get("targetGate") require(gate in ("B", "C"), "targetGate must be B or C.") require(record.get("decision") == "approved", "Only an explicit approved decision passes validation.") diff --git a/docs/gate-b-prerequisites.md b/docs/gate-b-prerequisites.md index fe4a394..cfe6182 100644 --- a/docs/gate-b-prerequisites.md +++ b/docs/gate-b-prerequisites.md @@ -1,6 +1,6 @@ # Gate B automation, identity and privacy acceptance -Run these reviews against the exact `0.2.0` candidate on the accepted HTTPS sandbox. Keep guest data, staff addresses, provider agreements, screenshots and raw reports in the restricted evidence store. Repository records contain opaque references only. +Run these reviews against the exact `0.2.1` candidate on the accepted HTTPS sandbox. Keep guest data, staff addresses, provider agreements, screenshots and raw reports in the restricted evidence store. Repository records contain opaque references only. ## Knowledge, AI and FAQ automation diff --git a/docs/operations.md b/docs/operations.md index 54516c6..f753440 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -18,7 +18,7 @@ Before deployment, verify the archive against its source record: ```sh python3 - <<'PY' import hashlib, json, pathlib -r = json.load(open('GuestOps-0.2.0-COMMIT.source.json', encoding='utf-8')) +r = json.load(open('GuestOps-0.2.1-COMMIT.source.json', encoding='utf-8')) p = pathlib.Path(r['artifact']['name']) assert hashlib.sha256(p.read_bytes()).hexdigest() == r['artifact']['sha256'] print(r['commit'], r['version'], r['artifact']) @@ -29,17 +29,17 @@ After Ansible has built the commit-tagged images, the repository verifier perfor ```sh python3 deploy/verify_release.py \ - --archive GuestOps-0.2.0.tar.gz \ + --archive GuestOps-0.2.1.tar.gz \ --record release-record.json \ --commit FULL_40_CHARACTER_SHA \ - --version 0.2.0 \ + --version 0.2.1 \ --output release-verification.json python3 deploy/verify_release.py \ - --archive GuestOps-0.2.0.tar.gz \ + --archive GuestOps-0.2.1.tar.gz \ --record release-record.json \ --commit FULL_40_CHARACTER_SHA \ - --version 0.2.0 \ + --version 0.2.1 \ --verify-loaded-images \ --output loaded-image-verification.json ``` diff --git a/docs/pilot-release.md b/docs/pilot-release.md index 9de0939..ce67efb 100644 --- a/docs/pilot-release.md +++ b/docs/pilot-release.md @@ -22,7 +22,7 @@ python3 deploy/capacity_probe.py \ unset CAPACITY_EMAIL CAPACITY_PASSWORD ``` -For the `0.2.0` Gate B candidate, the approved targets are concurrency 10, p95 latency at or below 500 ms, error rate at or below 1%, and at least 25% CPU and memory headroom on the documented four-core, 7.6 GiB host. The generated result reports HTTP observations, not a pass/fail claim. Repeat after a warm-up, investigate every error, and retain independently captured host metrics with the report. Hash both retained files for the approval record. Do not point the probe at a live hotel or increase its built-in bounds to simulate a denial of service. +For the `0.2.1` Gate B candidate, the approved targets are concurrency 10, p95 latency at or below 500 ms, error rate at or below 1%, and at least 25% CPU and memory headroom on the documented four-core, 7.6 GiB host. The generated result reports HTTP observations, not a pass/fail claim. Repeat after a warm-up, investigate every error, and retain independently captured host metrics with the report. Hash both retained files for the approval record. Do not point the probe at a live hotel or increase its built-in bounds to simulate a denial of service. ## Five-business-day supervised pilot diff --git a/tests/test_backup_restore_acceptance.py b/tests/test_backup_restore_acceptance.py index a3a33f8..0b1be9d 100644 --- a/tests/test_backup_restore_acceptance.py +++ b/tests/test_backup_restore_acceptance.py @@ -19,7 +19,7 @@ def valid_record(): "system": "guestops-backup-recovery", "evidenceId": "backup-restore", "dataClassification": "synthetic-only", - "releaseVersion": "0.2.0", + "releaseVersion": "0.2.1", "releaseCommit": COMMIT, "releaseRecordSha256": RELEASE_SHA, "archiveSha256": "c" * 64, diff --git a/tests/test_debian_acceptance.py b/tests/test_debian_acceptance.py index 742538d..f59e842 100644 --- a/tests/test_debian_acceptance.py +++ b/tests/test_debian_acceptance.py @@ -17,7 +17,7 @@ def common(system): "schemaVersion": 1, "system": system, "evidenceId": acceptance.SYSTEMS[system]["evidenceId"], - "releaseVersion": "0.2.0", + "releaseVersion": "0.2.1", "releaseCommit": COMMIT, "releaseRecordSha256": RELEASE_SHA, "archiveSha256": "c" * 64, diff --git a/tests/test_release_record.py b/tests/test_release_record.py index 072eb30..378d7b7 100644 --- a/tests/test_release_record.py +++ b/tests/test_release_record.py @@ -13,7 +13,7 @@ ROOT = Path(__file__).resolve().parents[1] class ReleaseRecordTests(unittest.TestCase): def test_writes_versions_checksum_and_immutable_image_ids(self): with tempfile.TemporaryDirectory() as directory: - artifact = Path(directory) / "GuestOps-0.2.0.tar.gz" + artifact = Path(directory) / "GuestOps-0.2.1.tar.gz" output = Path(directory) / "release-record.json" artifact.write_bytes(b"reviewed image archive") @@ -40,7 +40,7 @@ class ReleaseRecordTests(unittest.TestCase): ) record = json.loads(output.read_text(encoding="utf-8")) - self.assertEqual(record["version"], "0.2.0") + self.assertEqual(record["version"], "0.2.1") self.assertEqual(record["commit"], "a" * 40) self.assertEqual(record["images"]["api"]["id"], "sha256:api") self.assertEqual( diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py index a0c6a13..4b286e9 100644 --- a/tests/test_verify_release.py +++ b/tests/test_verify_release.py @@ -21,12 +21,12 @@ WORKER_ID = "sha256:" + "c" * 64 class VerifyReleaseTests(unittest.TestCase): def fixture(self, directory: str): root = Path(directory) - archive = root / "GuestOps-0.2.0.tar.gz" + archive = root / "GuestOps-0.2.1.tar.gz" record = root / "release-record.json" archive.write_bytes(b"reviewed image archive") release = { "schemaVersion": 1, - "version": "0.2.0", + "version": "0.2.1", "commit": COMMIT, "artifact": { "name": archive.name, @@ -44,7 +44,7 @@ class VerifyReleaseTests(unittest.TestCase): def test_verifies_candidate_archive_record_and_record_checksum(self): with tempfile.TemporaryDirectory() as directory: archive, record, _ = self.fixture(directory) - result = verify_release.validate(archive, record, COMMIT, "0.2.0") + result = verify_release.validate(archive, record, COMMIT, "0.2.1") self.assertTrue(result["verified"]) self.assertEqual(result["archive"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) self.assertEqual(result["releaseRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest()) @@ -55,7 +55,7 @@ class VerifyReleaseTests(unittest.TestCase): archive, record, _ = self.fixture(directory) archive.write_bytes(b"changed") with self.assertRaisesRegex(ValueError, "size does not match"): - verify_release.validate(archive, record, COMMIT, "0.2.0") + verify_release.validate(archive, record, COMMIT, "0.2.1") def test_rejects_wrong_commit_version_reference_and_mutable_id(self): cases = [ @@ -70,17 +70,17 @@ class VerifyReleaseTests(unittest.TestCase): mutate(release) record.write_text(json.dumps(release), encoding="utf-8") with self.assertRaisesRegex(ValueError, message): - verify_release.validate(archive, record, COMMIT, "0.2.0") + verify_release.validate(archive, record, COMMIT, "0.2.1") def test_loaded_image_ids_must_match(self): with tempfile.TemporaryDirectory() as directory: archive, record, _ = self.fixture(directory) with patch.object(verify_release, "loaded_image_id", side_effect=[API_ID, WORKER_ID]): - result = verify_release.validate(archive, record, COMMIT, "0.2.0", True) + result = verify_release.validate(archive, record, COMMIT, "0.2.1", True) self.assertTrue(result["loadedImageIdsVerified"]) with patch.object(verify_release, "loaded_image_id", return_value="sha256:" + "d" * 64): with self.assertRaisesRegex(ValueError, "Loaded api image ID"): - verify_release.validate(archive, record, COMMIT, "0.2.0", True) + verify_release.validate(archive, record, COMMIT, "0.2.1", True) if __name__ == "__main__": diff --git a/web/package-lock.json b/web/package-lock.json index 8ec7573..31deacc 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "guestops-web", - "version": "0.2.0", + "version": "0.2.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "guestops-web", - "version": "0.2.0", + "version": "0.2.1", "dependencies": { "lucide-react": "^0.577.0", "react": "19.2.8", diff --git a/web/package.json b/web/package.json index 9b7063b..ccea170 100644 --- a/web/package.json +++ b/web/package.json @@ -1 +1 @@ -{"name":"guestops-web","private":true,"version":"0.2.0","type":"module","scripts":{"dev":"vite --host 127.0.0.1","build":"tsc -b && vite build","check":"tsc -b"},"dependencies":{"react":"19.2.8","react-dom":"19.2.8","lucide-react":"^0.577.0"},"devDependencies":{"@types/react":"^19.2.0","@types/react-dom":"^19.2.0","@vitejs/plugin-react":"6.1.1","typescript":"~5.9.3","vite":"8.2.2"}} +{"name":"guestops-web","private":true,"version":"0.2.1","type":"module","scripts":{"dev":"vite --host 127.0.0.1","build":"tsc -b && vite build","check":"tsc -b"},"dependencies":{"react":"19.2.8","react-dom":"19.2.8","lucide-react":"^0.577.0"},"devDependencies":{"@types/react":"^19.2.0","@types/react-dom":"^19.2.0","@vitejs/plugin-react":"6.1.1","typescript":"~5.9.3","vite":"8.2.2"}} -- 2.20.1 From 864100b8130926b3396d688c1f45b99744987978 Mon Sep 17 00:00:00 2001 From: mathew Date: Wed, 30 Sep 2026 21:06:30 +0100 Subject: [PATCH 06/11] Ignore local release package output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3b36fb8..4fde547 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ **/payments.local.json .guestops-maintenance.lock .guestops-test-keyring/ +.guestops-release/ guestops-backup-*/ guestops-restore-*/ *.tar.gpg -- 2.20.1 From 3293472cadecb16f4aa09fed1716b91f27ddc771 Mon Sep 17 00:00:00 2001 From: mathew Date: Thu, 1 Oct 2026 07:32:43 +0100 Subject: [PATCH 07/11] Add verified Ansible release handoff --- deploy/ansible/README.md | 31 +++ deploy/ansible/guestops.yml | 292 ++++++++++++++++++++++++++++ deploy/verify_source_package.py | 104 ++++++++++ docs/deployment.md | 2 + tests/test_ansible_handoff.py | 47 +++++ tests/test_verify_source_package.py | 72 +++++++ 6 files changed, 548 insertions(+) create mode 100644 deploy/ansible/README.md create mode 100644 deploy/ansible/guestops.yml create mode 100644 deploy/verify_source_package.py create mode 100644 tests/test_ansible_handoff.py create mode 100644 tests/test_verify_source_package.py diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md new file mode 100644 index 0000000..535f654 --- /dev/null +++ b/deploy/ansible/README.md @@ -0,0 +1,31 @@ +# GuestOps Ansible handoff + +This playbook is the application-owned deployment contract. Copy or import it into the private Futuresens Ansible repository and bind the `guestops` inventory group there. Do not commit inventory secrets, `.env` contents, provider credentials, hostnames other than the public service name, or restricted evidence paths to GuestOps. + +Required extra variables: + +- `guestops_release_version`: matched application version, for example `0.2.1`; +- `guestops_release_commit`: full 40-character packaged commit; +- `guestops_archive`: controller path to the generated `.tar.gz`; +- `guestops_archive_sha256`: approved lowercase SHA-256; +- `guestops_source_record`: controller path to the matching `.source.json`; +- `guestops_evidence_directory`: existing restricted controller directory for the fetched release record. + +The target must already have Docker Engine with Compose, Python 3, and `/etc/guestops/guestops.env` owned by root with mode `0600`. The private environment file supplies MongoDB passwords, image references, provider-file paths and disabled-by-default feature flags. Provision it through Ansible Vault or the existing Futuresens secret process, never through this repository. + +Example invocation from the central Ansible checkout: + +```sh +ansible-playbook guestops.yml \ + -l guestops_sandbox \ + -e guestops_release_version=0.2.1 \ + -e guestops_release_commit=FULL_40_CHARACTER_SHA \ + -e guestops_archive=/secure/releases/GuestOps-0.2.1-COMMIT.tar.gz \ + -e guestops_archive_sha256=LOWERCASE_SHA256 \ + -e guestops_source_record=/secure/releases/GuestOps-0.2.1-COMMIT.source.json \ + -e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 +``` + +The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, waits for readiness, and only then changes the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success. + +This playbook does not provision DNS, TLS, Nginx, firewall rules, backup keys, monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities. diff --git a/deploy/ansible/guestops.yml b/deploy/ansible/guestops.yml new file mode 100644 index 0000000..1bb319e --- /dev/null +++ b/deploy/ansible/guestops.yml @@ -0,0 +1,292 @@ +--- +- name: Install a verified GuestOps source release + hosts: guestops + become: true + gather_facts: true + + vars: + guestops_root: /opt/guestops + guestops_config_root: /etc/guestops + guestops_release_root: "{{ guestops_root }}/releases/{{ guestops_release_commit }}" + guestops_staging_archive: "/var/tmp/{{ guestops_archive | basename }}" + guestops_staging_record: "/var/tmp/{{ guestops_source_record | basename }}" + guestops_api_image: "guestops-api:{{ guestops_release_commit }}" + guestops_worker_image: "guestops-worker:{{ guestops_release_commit }}" + + pre_tasks: + - name: Validate required release variables + ansible.builtin.assert: + that: + - guestops_release_version is match('^[0-9]+\\.[0-9]+\\.[0-9]+$') + - guestops_release_commit is match('^[0-9a-f]{40}$') + - guestops_archive_sha256 is match('^[0-9a-f]{64}$') + - guestops_archive | length > 0 + - guestops_source_record | length > 0 + - guestops_evidence_directory | length > 0 + fail_msg: Release version, full commit, archive, SHA-256, source record and evidence directory are required. + + - name: Verify source package on the Ansible controller + ansible.builtin.command: + argv: + - python3 + - "{{ playbook_dir }}/../verify_source_package.py" + - --archive + - "{{ guestops_archive }}" + - --record + - "{{ guestops_source_record }}" + - --expected-commit + - "{{ guestops_release_commit }}" + - --expected-version + - "{{ guestops_release_version }}" + delegate_to: localhost + become: false + changed_when: false + + - name: Check private GuestOps environment exists + ansible.builtin.stat: + path: "{{ guestops_config_root }}/guestops.env" + register: guestops_environment + + - name: Require a private pre-provisioned environment file + ansible.builtin.assert: + that: + - guestops_environment.stat.exists + - guestops_environment.stat.isreg + - guestops_environment.stat.mode == '0600' + fail_msg: /etc/guestops/guestops.env must already exist with mode 0600. + + tasks: + - name: Create release and evidence directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - { path: "{{ guestops_root }}/releases", mode: "0755" } + - { path: "{{ guestops_release_root }}", mode: "0755" } + - { path: "{{ guestops_config_root }}/release-records", mode: "0700" } + + - name: Copy source archive to the host + ansible.builtin.copy: + src: "{{ guestops_archive }}" + dest: "{{ guestops_staging_archive }}" + owner: root + group: root + mode: "0600" + + - name: Copy source record to the host + ansible.builtin.copy: + src: "{{ guestops_source_record }}" + dest: "{{ guestops_staging_record }}" + owner: root + group: root + mode: "0600" + + - name: Calculate transferred archive checksum + ansible.builtin.stat: + path: "{{ guestops_staging_archive }}" + checksum_algorithm: sha256 + register: guestops_transferred_archive + + - name: Reject a changed transferred archive + ansible.builtin.assert: + that: + - guestops_transferred_archive.stat.checksum == guestops_archive_sha256 + fail_msg: Transferred GuestOps archive does not match the approved SHA-256. + + - name: Extract committed application files + ansible.builtin.unarchive: + src: "{{ guestops_staging_archive }}" + dest: "{{ guestops_release_root }}" + remote_src: true + extra_opts: + - --strip-components=1 + + - name: Verify transferred package and record on the target + ansible.builtin.command: + argv: + - python3 + - deploy/verify_source_package.py + - --archive + - "{{ guestops_staging_archive }}" + - --record + - "{{ guestops_staging_record }}" + - --expected-commit + - "{{ guestops_release_commit }}" + - --expected-version + - "{{ guestops_release_version }}" + - --output + - "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source-verification.json" + chdir: "{{ guestops_release_root }}" + changed_when: false + + - name: Retain approved source record on the target + ansible.builtin.copy: + src: "{{ guestops_staging_record }}" + dest: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source.json" + remote_src: true + owner: root + group: root + mode: "0600" + + - name: Select release images and safe pilot defaults in the private environment + ansible.builtin.lineinfile: + path: "{{ guestops_config_root }}/guestops.env" + regexp: "^{{ item.name }}=" + line: "{{ item.name }}={{ item.value }}" + create: false + mode: "0600" + loop: + - { name: GUESTOPS_API_IMAGE, value: "{{ guestops_api_image }}" } + - { name: GUESTOPS_WORKER_IMAGE, value: "{{ guestops_worker_image }}" } + - { name: GOOGLE_ENABLE_SENDING, value: "false" } + - { name: AUTO_REPLY_ENABLE_LIVE, value: "false" } + no_log: true + + - name: Copy private deployment environment into the release + ansible.builtin.copy: + src: "{{ guestops_config_root }}/guestops.env" + dest: "{{ guestops_release_root }}/.env" + remote_src: true + owner: root + group: root + mode: "0600" + + - name: Build API image from the verified source + ansible.builtin.command: + argv: + - docker + - build + - --target + - api + - --tag + - "{{ guestops_api_image }}" + - . + chdir: "{{ guestops_release_root }}" + register: guestops_api_build + changed_when: true + + - name: Build worker image from the verified source + ansible.builtin.command: + argv: + - docker + - build + - --target + - worker + - --tag + - "{{ guestops_worker_image }}" + - . + chdir: "{{ guestops_release_root }}" + register: guestops_worker_build + changed_when: true + + - name: Inspect API image identity + ansible.builtin.command: + argv: [docker, image, inspect, --format, "{{ '{{.Id}}' }}", "{{ guestops_api_image }}"] + register: guestops_api_identity + changed_when: false + + - name: Inspect worker image identity + ansible.builtin.command: + argv: [docker, image, inspect, --format, "{{ '{{.Id}}' }}", "{{ guestops_worker_image }}"] + register: guestops_worker_identity + changed_when: false + + - name: Create post-build release record + ansible.builtin.command: + argv: + - python3 + - deploy/release_record.py + - --artifact + - "{{ guestops_staging_archive }}" + - --commit + - "{{ guestops_release_commit }}" + - --api-image + - "{{ guestops_api_image }}" + - --api-id + - "{{ guestops_api_identity.stdout }}" + - --worker-image + - "{{ guestops_worker_image }}" + - --worker-id + - "{{ guestops_worker_identity.stdout }}" + - --output + - "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.json" + chdir: "{{ guestops_release_root }}" + + - name: Run offline deployment preflight + ansible.builtin.command: + argv: + - python3 + - deploy/ops.py + - preflight + - --offline + chdir: "{{ guestops_release_root }}" + changed_when: false + + - name: Validate Compose without exposing expanded configuration + ansible.builtin.command: + argv: + - docker + - compose + - --env-file + - "{{ guestops_config_root }}/guestops.env" + - config + - --quiet + chdir: "{{ guestops_release_root }}" + changed_when: false + + - name: Start the verified release without rebuilding + ansible.builtin.command: + argv: + - docker + - compose + - --env-file + - "{{ guestops_config_root }}/guestops.env" + - up + - --detach + - --no-build + chdir: "{{ guestops_release_root }}" + environment: + GUESTOPS_API_IMAGE: "{{ guestops_api_image }}" + GUESTOPS_WORKER_IMAGE: "{{ guestops_worker_image }}" + changed_when: true + + - name: Wait for loopback readiness + ansible.builtin.uri: + url: http://127.0.0.1:8080/health/ready + method: GET + status_code: 200 + return_content: false + register: guestops_readiness + retries: 30 + delay: 2 + until: guestops_readiness.status == 200 + + - name: Select the current successful release + ansible.builtin.file: + src: "{{ guestops_release_root }}" + dest: "{{ guestops_root }}/current" + state: link + force: true + + - name: Retain release record on the Ansible controller + ansible.builtin.fetch: + src: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.json" + dest: "{{ guestops_evidence_directory }}/release-record.json" + flat: true + + - name: Retain target source verification on the Ansible controller + ansible.builtin.fetch: + src: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source-verification.json" + dest: "{{ guestops_evidence_directory }}/source-verification.json" + flat: true + + - name: Remove transferred staging files after successful deployment + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ guestops_staging_archive }}" + - "{{ guestops_staging_record }}" diff --git a/deploy/verify_source_package.py b/deploy/verify_source_package.py new file mode 100644 index 0000000..08b406f --- /dev/null +++ b/deploy/verify_source_package.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Verify a GuestOps source archive against its deterministic source record.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys + + +FULL_SHA = re.compile(r"[0-9a-f]{40}") +SHA256 = re.compile(r"[0-9a-f]{64}") +SEMVER = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate( + archive: Path, + record_path: Path, + expected_commit: str | None = None, + expected_version: str | None = None, +) -> dict[str, object]: + require(archive.is_file(), "Source archive does not exist.") + require(record_path.is_file(), "Source record does not exist.") + record = json.loads(record_path.read_text(encoding="utf-8")) + require(isinstance(record, dict), "Source record must be a JSON object.") + require(record.get("schemaVersion") == 1, "Unsupported source-record schema.") + + commit = record.get("commit") + version = record.get("version") + require(isinstance(commit, str) and FULL_SHA.fullmatch(commit) is not None, + "Source record commit must be a full lowercase Git SHA.") + require(isinstance(version, str) and SEMVER.fullmatch(version) is not None, + "Source record version must use MAJOR.MINOR.PATCH.") + if expected_commit is not None: + require(FULL_SHA.fullmatch(expected_commit) is not None, + "Expected commit must be a full lowercase Git SHA.") + require(commit == expected_commit, "Source-record commit does not match the selected release.") + if expected_version is not None: + require(SEMVER.fullmatch(expected_version) is not None, + "Expected version must use MAJOR.MINOR.PATCH.") + require(version == expected_version, "Source-record version does not match the selected release.") + + artifact = record.get("artifact") + require(isinstance(artifact, dict), "Source record has no artifact object.") + require(artifact.get("name") == archive.name, "Source archive filename does not match the record.") + require(artifact.get("size") == archive.stat().st_size, "Source archive size does not match the record.") + recorded_sha = artifact.get("sha256") + require(isinstance(recorded_sha, str) and SHA256.fullmatch(recorded_sha) is not None, + "Source archive record must contain a lowercase SHA-256 digest.") + actual_sha = sha256(archive) + require(recorded_sha == actual_sha, "Source archive SHA-256 does not match the record.") + + return { + "artifact": {"name": archive.name, "sha256": actual_sha, "size": archive.stat().st_size}, + "commit": commit, + "schemaVersion": 1, + "sourceRecord": {"name": record_path.name, "sha256": sha256(record_path)}, + "verified": True, + "version": version, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--archive", required=True, type=Path) + parser.add_argument("--record", required=True, type=Path) + parser.add_argument("--expected-commit") + parser.add_argument("--expected-version") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + try: + result = validate( + args.archive, + args.record, + args.expected_commit, + args.expected_version, + ) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"Source package verification failed: {error}", file=sys.stderr) + raise SystemExit(1) + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + print(rendered, end="") + + +if __name__ == "__main__": + main() diff --git a/docs/deployment.md b/docs/deployment.md index 9b96b42..0b7156b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -30,6 +30,8 @@ The packaging command resolves the ref to a full commit, reads both application Store the archive and source record under a versioned GuestOps files directory in the private Ansible repository. The GuestOps playbook and environment variables should select that version, verify the archive against the source record, extract it into the application directory, preserve the private `.env` and provider configuration, and build images tagged with the full source commit: +The application-owned handoff playbook and required variables are documented in [`deploy/ansible/README.md`](../deploy/ansible/README.md). Import or copy that playbook into the central private Ansible repository, bind its `guestops` inventory group, and keep inventory and secret values there. + ```sh docker build --target api -t guestops-api:FULL_40_CHARACTER_SHA . docker build --target worker -t guestops-worker:FULL_40_CHARACTER_SHA . diff --git a/tests/test_ansible_handoff.py b/tests/test_ansible_handoff.py new file mode 100644 index 0000000..5bdf5af --- /dev/null +++ b/tests/test_ansible_handoff.py @@ -0,0 +1,47 @@ +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +PLAYBOOK = ROOT / "deploy" / "ansible" / "guestops.yml" + + +class AnsibleHandoffTests(unittest.TestCase): + def test_release_is_verified_before_build_and_start(self): + text = PLAYBOOK.read_text(encoding="utf-8") + controller_verify = text.index("Verify source package on the Ansible controller") + target_verify = text.index("Verify transferred package and record on the target") + api_build = text.index("Build API image from the verified source") + preflight = text.index("Run offline deployment preflight") + compose_start = text.index("Start the verified release without rebuilding") + readiness = text.index("Wait for loopback readiness") + current = text.index("Select the current successful release") + self.assertLess(controller_verify, target_verify) + self.assertLess(target_verify, api_build) + self.assertLess(api_build, preflight) + self.assertLess(preflight, compose_start) + self.assertLess(compose_start, readiness) + self.assertLess(readiness, current) + + def test_playbook_uses_safe_release_controls(self): + text = PLAYBOOK.read_text(encoding="utf-8") + for required in ( + "guestops_release_commit is match('^[0-9a-f]{40}$')", + "checksum_algorithm: sha256", + "deploy/verify_source_package.py", + "deploy/release_record.py", + "deploy/ops.py", + "--offline", + "--no-build", + "GOOGLE_ENABLE_SENDING", + "AUTO_REPLY_ENABLE_LIVE", + "no_log: true", + "http://127.0.0.1:8080/health/ready", + ): + self.assertIn(required, text) + self.assertNotIn("ansible.builtin.shell", text) + self.assertNotIn("password=", text.lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_source_package.py b/tests/test_verify_source_package.py new file mode 100644 index 0000000..5168418 --- /dev/null +++ b/tests/test_verify_source_package.py @@ -0,0 +1,72 @@ +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "verify_source_package", ROOT / "deploy" / "verify_source_package.py" +) +verify_source_package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verify_source_package) + + +COMMIT = "a" * 40 +VERSION = "0.2.1" + + +class VerifySourcePackageTests(unittest.TestCase): + def fixture(self, directory: str): + root = Path(directory) + archive = root / "GuestOps-0.2.1-aaaaaaaaaaaa.tar.gz" + record = root / "GuestOps-0.2.1-aaaaaaaaaaaa.source.json" + archive.write_bytes(b"committed source package") + payload = { + "artifact": { + "name": archive.name, + "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "size": archive.stat().st_size, + }, + "commit": COMMIT, + "schemaVersion": 1, + "version": VERSION, + } + record.write_text(json.dumps(payload), encoding="utf-8") + return archive, record, payload + + def test_verifies_archive_record_and_expected_identity(self): + with tempfile.TemporaryDirectory() as directory: + archive, record, _ = self.fixture(directory) + result = verify_source_package.validate(archive, record, COMMIT, VERSION) + self.assertTrue(result["verified"]) + self.assertEqual(result["commit"], COMMIT) + self.assertEqual(result["artifact"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) + self.assertEqual(result["sourceRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest()) + + def test_rejects_changed_archive(self): + with tempfile.TemporaryDirectory() as directory: + archive, record, _ = self.fixture(directory) + archive.write_bytes(b"changed source package") + with self.assertRaisesRegex(ValueError, "size does not match"): + verify_source_package.validate(archive, record, COMMIT, VERSION) + + def test_rejects_wrong_commit_version_and_filename(self): + cases = [ + (lambda value: value.update(commit="b" * 40), "commit does not match"), + (lambda value: value.update(version="0.3.0"), "version does not match"), + (lambda value: value["artifact"].update(name="other.tar.gz"), "filename does not match"), + ] + for mutate, message in cases: + with self.subTest(message=message), tempfile.TemporaryDirectory() as directory: + archive, record, payload = self.fixture(directory) + mutate(payload) + record.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex(ValueError, message): + verify_source_package.validate(archive, record, COMMIT, VERSION) + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From 86e548f2fc80fcd05717fef047e049d6efd5067d Mon Sep 17 00:00:00 2001 From: mathew Date: Thu, 1 Oct 2026 08:59:03 +0100 Subject: [PATCH 08/11] Integrate Debian host readiness checks --- MILESTONES.md | 4 +- deploy/ansible/README.md | 4 +- deploy/ansible/guestops.yml | 110 ++++++++++++++++++++++++++++++++++ docs/deployment.md | 2 + tests/test_ansible_handoff.py | 8 ++- 5 files changed, 123 insertions(+), 5 deletions(-) diff --git a/MILESTONES.md b/MILESTONES.md index 69435b6..1c8aa8d 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -20,7 +20,7 @@ This summary explains what each milestone delivers and where it currently stands | 7 | Google connection recovery | OAuth reconnect, checkpoint recovery, grant revocation handling, and worker restart safety. | **Implemented; acceptance required.** Dedicated Google-account and worker-restart exercises have not yet been accepted. | | 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | | 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The runner-free deterministic packager is implemented and the Gitea Action is removed; the release-line identity must be confirmed, then the package, Ansible installation evidence, and approval record must be retained. | -| 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** Acceptance tooling is ready, but Docker, correct HTTPS/network exposure, exact artifacts, privileged installation, and the supervised drills remain open. | +| 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** The verified Ansible handoff now covers commit-bound installation, boot services, Nginx validation, listener restrictions and public HTTPS; privileged installation, firewall review, controlled reboot and supervised persistence evidence remain open. | | 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** Repository tooling is ready; installation and timed operational evidence are blocked until Milestones 9 and 10 pass. | | 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | | 13 | Rezlynx/Guestline adapter | The real PMS provider adapter, mappings, idempotency, reconciliation, and ambiguous-write handling. | **Planned.** Provider contract and sandbox access are still required before implementation and acceptance. | @@ -60,7 +60,7 @@ This summary explains what each milestone delivers and where it currently stands | 7 | Google connection recovery | B | Implemented / acceptance required | Connection epochs, checkpoint recovery, and revocation handling are promoted to local `main`; complete real Google acceptance and worker-restart exercises. | | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | | 9 | Gitea and reproducible releases | A | In progress | `deploy/package_source.py` now packages only an explicit committed ref, verifies matched application versions, produces deterministic gzip output and a SHA-256 source record, and refuses overwrite. Hand that package to a version-selected Ansible playbook following the CMS/CMSFront pattern. Ansible must verify and install it, build commit-tagged images, record their immutable IDs, and deploy without a Gitea runner. Retain the package/install evidence off-host and resolve the release-line/tag identity before approval; the existing `0.1.0` tag remains attached to the foundation release. | -| 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The confirmation-gated persistence drill verifies restart and container-recreation behaviour. Run it on the provisioned Debian host, complete HTTPS and controlled-reboot acceptance, and retain the evidence. | +| 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The Ansible handoff verifies the source on both controller and host, enables Docker/Nginx at boot, installs and validates the reviewed proxy, rejects exposed API/MongoDB listeners, and requires trusted public HTTPS before selecting the release. Run it on the provisioned Debian host, review the firewall, complete the confirmation-gated persistence drill and controlled reboot, and retain independent evidence. | | 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling, checksum-verified rsync transfer, a restricted Zabbix status boundary, guarded local retention and a release-bound acceptance validator. Install and test it on Debian, configure the restricted store and alerts, name operational/review owners, and retain independently reviewed evidence from the timed restore and rollback drill. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. | diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index 535f654..fbbf5e5 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -11,7 +11,7 @@ Required extra variables: - `guestops_source_record`: controller path to the matching `.source.json`; - `guestops_evidence_directory`: existing restricted controller directory for the fetched release record. -The target must already have Docker Engine with Compose, Python 3, and `/etc/guestops/guestops.env` owned by root with mode `0600`. The private environment file supplies MongoDB passwords, image references, provider-file paths and disabled-by-default feature flags. Provision it through Ansible Vault or the existing Futuresens secret process, never through this repository. +The target must already have Docker Engine with Compose, Nginx, Python 3, `ss`, a trusted certificate under `/etc/letsencrypt/live/sandbox-guestops.futuresens.co.uk`, and `/etc/guestops/guestops.env` owned by root with mode `0600`. The private environment file supplies MongoDB passwords, image references, provider-file paths and disabled-by-default feature flags. Provision it through Ansible Vault or the existing Futuresens secret process, never through this repository. Example invocation from the central Ansible checkout: @@ -26,6 +26,6 @@ ansible-playbook guestops.yml \ -e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 ``` -The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, waits for readiness, and only then changes the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success. +The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, and waits for loopback readiness. It then enables Docker and Nginx at boot, validates and installs the reviewed proxy site, rejects public API or MongoDB listeners, and checks the public redirect and certificate-backed HTTPS readiness before changing the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success. This playbook does not provision DNS, TLS, Nginx, firewall rules, backup keys, monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities. diff --git a/deploy/ansible/guestops.yml b/deploy/ansible/guestops.yml index 1bb319e..a924bd3 100644 --- a/deploy/ansible/guestops.yml +++ b/deploy/ansible/guestops.yml @@ -7,6 +7,8 @@ vars: guestops_root: /opt/guestops guestops_config_root: /etc/guestops + guestops_public_hostname: sandbox-guestops.futuresens.co.uk + guestops_public_origin: "https://{{ guestops_public_hostname }}" guestops_release_root: "{{ guestops_root }}/releases/{{ guestops_release_commit }}" guestops_staging_archive: "/var/tmp/{{ guestops_archive | basename }}" guestops_staging_record: "/var/tmp/{{ guestops_source_record | basename }}" @@ -23,6 +25,8 @@ - guestops_archive | length > 0 - guestops_source_record | length > 0 - guestops_evidence_directory | length > 0 + - guestops_public_hostname is match('^[A-Za-z0-9.-]+$') + - guestops_public_hostname == 'sandbox-guestops.futuresens.co.uk' fail_msg: Release version, full commit, archive, SHA-256, source record and evidence directory are required. - name: Verify source package on the Ansible controller @@ -56,6 +60,28 @@ fail_msg: /etc/guestops/guestops.env must already exist with mode 0600. tasks: + - name: Require supported Debian host + ansible.builtin.assert: + that: + - ansible_facts.system == 'Linux' + - ansible_facts.distribution == 'Debian' + - ansible_facts.distribution_major_version | int >= 12 + - ansible_facts.processor_vcpus | default(0) | int >= 4 + - ansible_facts.memtotal_mb | default(0) | int >= 7300 + fail_msg: GuestOps requires Debian 12 or newer with at least 4 CPUs and 7.3 GiB RAM. + + - name: Enable Docker at boot + ansible.builtin.service: + name: docker + enabled: true + state: started + + - name: Enable Nginx at boot + ansible.builtin.service: + name: nginx + enabled: true + state: started + - name: Create release and evidence directories ansible.builtin.file: path: "{{ item.path }}" @@ -264,6 +290,90 @@ delay: 2 until: guestops_readiness.status == 200 + - name: Confirm TLS certificate files exist + ansible.builtin.stat: + path: "{{ item }}" + follow: true + loop: + - "/etc/letsencrypt/live/{{ guestops_public_hostname }}/fullchain.pem" + - "/etc/letsencrypt/live/{{ guestops_public_hostname }}/privkey.pem" + register: guestops_certificates + + - name: Require the pre-provisioned TLS certificate + ansible.builtin.assert: + that: + - guestops_certificates.results | map(attribute='stat.exists') | min + - guestops_certificates.results | map(attribute='stat.isreg') | min + fail_msg: A valid pre-provisioned Let's Encrypt certificate is required before enabling Nginx. + + - name: Install reviewed GuestOps Nginx site + ansible.builtin.copy: + src: "{{ playbook_dir }}/../nginx.conf" + dest: /etc/nginx/sites-available/guestops.conf + owner: root + group: root + mode: "0644" + register: guestops_nginx_site + + - name: Enable GuestOps Nginx site + ansible.builtin.file: + src: /etc/nginx/sites-available/guestops.conf + dest: /etc/nginx/sites-enabled/guestops.conf + state: link + register: guestops_nginx_enabled + + - name: Validate Nginx configuration + ansible.builtin.command: + argv: [nginx, -t] + changed_when: false + + - name: Reload Nginx after reviewed configuration change + ansible.builtin.service: + name: nginx + enabled: true + state: reloaded + when: guestops_nginx_site.changed or guestops_nginx_enabled.changed + + - name: Inspect host TCP listeners + ansible.builtin.command: + argv: [ss, -ltnH] + register: guestops_tcp_listeners + changed_when: false + + - name: Reject public API or MongoDB listeners + ansible.builtin.assert: + that: + - guestops_tcp_listeners.stdout_lines | select('search', ':8080(\\s|$)') | reject('search', '127\\.0\\.0\\.1:8080(\\s|$)') | list | length == 0 + - guestops_tcp_listeners.stdout_lines | select('search', ':27017(\\s|$)') | list | length == 0 + fail_msg: API port 8080 must be loopback-only and MongoDB must have no host listener. + + - name: Verify public HTTP redirects to HTTPS + ansible.builtin.uri: + url: "http://{{ guestops_public_hostname }}/health/ready" + method: GET + follow_redirects: none + status_code: [301, 302, 307, 308] + return_content: false + delegate_to: localhost + become: false + + - name: Verify public HTTPS readiness and certificate trust + ansible.builtin.uri: + url: "{{ guestops_public_origin }}/health/ready" + method: GET + status_code: 200 + return_content: true + validate_certs: true + register: guestops_public_readiness + retries: 10 + delay: 3 + until: + - guestops_public_readiness.status == 200 + - guestops_public_readiness.json is defined + - guestops_public_readiness.json.status == 'ready' + delegate_to: localhost + become: false + - name: Select the current successful release ansible.builtin.file: src: "{{ guestops_release_root }}" diff --git a/docs/deployment.md b/docs/deployment.md index 0b7156b..8bfa73e 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -32,6 +32,8 @@ Store the archive and source record under a versioned GuestOps files directory i The application-owned handoff playbook and required variables are documented in [`deploy/ansible/README.md`](../deploy/ansible/README.md). Import or copy that playbook into the central private Ansible repository, bind its `guestops` inventory group, and keep inventory and secret values there. +The playbook requires Debian 12 or newer with the approved CPU/RAM baseline, enables Docker and Nginx at boot, installs the reviewed site only after loopback readiness, runs `nginx -t`, rejects a host MongoDB listener or non-loopback API listener, and verifies the public redirect and trusted HTTPS readiness before selecting the release. Firewall reachability, certificate renewal, controlled reboot, durable logs and the persistence exercise still require the supervised checks below. + ```sh docker build --target api -t guestops-api:FULL_40_CHARACTER_SHA . docker build --target worker -t guestops-worker:FULL_40_CHARACTER_SHA . diff --git a/tests/test_ansible_handoff.py b/tests/test_ansible_handoff.py index 5bdf5af..e0cd4d4 100644 --- a/tests/test_ansible_handoff.py +++ b/tests/test_ansible_handoff.py @@ -15,13 +15,15 @@ class AnsibleHandoffTests(unittest.TestCase): preflight = text.index("Run offline deployment preflight") compose_start = text.index("Start the verified release without rebuilding") readiness = text.index("Wait for loopback readiness") + public_readiness = text.index("Verify public HTTPS readiness and certificate trust") current = text.index("Select the current successful release") self.assertLess(controller_verify, target_verify) self.assertLess(target_verify, api_build) self.assertLess(api_build, preflight) self.assertLess(preflight, compose_start) self.assertLess(compose_start, readiness) - self.assertLess(readiness, current) + self.assertLess(readiness, public_readiness) + self.assertLess(public_readiness, current) def test_playbook_uses_safe_release_controls(self): text = PLAYBOOK.read_text(encoding="utf-8") @@ -37,6 +39,10 @@ class AnsibleHandoffTests(unittest.TestCase): "AUTO_REPLY_ENABLE_LIVE", "no_log: true", "http://127.0.0.1:8080/health/ready", + "Reject public API or MongoDB listeners", + "Validate Nginx configuration", + "validate_certs: true", + "enabled: true", ): self.assertIn(required, text) self.assertNotIn("ansible.builtin.shell", text) -- 2.20.1 From 158d21ca045631c64a4b89309e1827bc122cdfc6 Mon Sep 17 00:00:00 2001 From: mathew Date: Thu, 1 Oct 2026 10:28:25 +0100 Subject: [PATCH 09/11] Integrate backup and monitoring operations --- MILESTONES.md | 4 +- deploy/ansible/README.md | 27 ++- deploy/ansible/guestops-operations.yml | 227 ++++++++++++++++++ deploy/ansible/guestops.yml | 2 +- .../systemd/guestops-backup-transfer.service | 6 +- deploy/systemd/guestops-backup.service | 6 +- .../systemd/guestops-monitor-status.service | 6 +- docs/operations.md | 2 +- tests/test_ansible_operations.py | 53 ++++ 9 files changed, 319 insertions(+), 14 deletions(-) create mode 100644 deploy/ansible/guestops-operations.yml create mode 100644 tests/test_ansible_operations.py diff --git a/MILESTONES.md b/MILESTONES.md index 1c8aa8d..7f6f6e8 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -21,7 +21,7 @@ This summary explains what each milestone delivers and where it currently stands | 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | | 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The runner-free deterministic packager is implemented and the Gitea Action is removed; the release-line identity must be confirmed, then the package, Ansible installation evidence, and approval record must be retained. | | 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** The verified Ansible handoff now covers commit-bound installation, boot services, Nginx validation, listener restrictions and public HTTPS; privileged installation, firewall review, controlled reboot and supervised persistence evidence remain open. | -| 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** Repository tooling is ready; installation and timed operational evidence are blocked until Milestones 9 and 10 pass. | +| 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** The Ansible operations handoff now installs validated systemd units, public-key-only backup support, transfer retry and restricted Zabbix status; secret provisioning, durable-log confirmation, manual backup, timed restore, rollback and independent evidence remain open. | | 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | | 13 | Rezlynx/Guestline adapter | The real PMS provider adapter, mappings, idempotency, reconciliation, and ambiguous-write handling. | **Planned.** Provider contract and sandbox access are still required before implementation and acceptance. | | 14 | Payment links and status | The real payment-provider integration, webhooks, expiry, replay protection, and reconciliation. | **Planned.** The provider path and sandbox acceptance plan still need to be confirmed and completed. | @@ -61,7 +61,7 @@ This summary explains what each milestone delivers and where it currently stands | 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | | 9 | Gitea and reproducible releases | A | In progress | `deploy/package_source.py` now packages only an explicit committed ref, verifies matched application versions, produces deterministic gzip output and a SHA-256 source record, and refuses overwrite. Hand that package to a version-selected Ansible playbook following the CMS/CMSFront pattern. Ansible must verify and install it, build commit-tagged images, record their immutable IDs, and deploy without a Gitea runner. Retain the package/install evidence off-host and resolve the release-line/tag identity before approval; the existing `0.1.0` tag remains attached to the foundation release. | | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The Ansible handoff verifies the source on both controller and host, enables Docker/Nginx at boot, installs and validates the reviewed proxy, rejects exposed API/MongoDB listeners, and requires trusted public HTTPS before selecting the release. Run it on the provisioned Debian host, review the firewall, complete the confirmation-gated persistence drill and controlled reboot, and retain independent evidence. | -| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling now includes opt-in systemd scheduling, checksum-verified rsync transfer, a restricted Zabbix status boundary, guarded local retention and a release-bound acceptance validator. Install and test it on Debian, configure the restricted store and alerts, name operational/review owners, and retain independently reviewed evidence from the timed restore and rollback drill. | +| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling includes opt-in systemd scheduling, checksum-verified rsync transfer, restricted Zabbix status, guarded local retention and a release-bound acceptance validator. The Ansible operations playbook now verifies the selected release and private-file modes, imports only the recovery public key, validates and installs the units, enables transfer/monitoring, leaves backup scheduling off until manual acceptance, and fetches non-sensitive evidence. Provision secrets and durable logs, configure central alerts/retention, run the manual backup plus timed restore and rollback drills, and retain independent approval. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. | | 14 | Payment links and status | C | Planned | Select/confirm the payment-provider path, complete sandbox and webhook acceptance, and prove expiry, replay protection, reconciliation, and support recovery. | diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md index fbbf5e5..7ff146b 100644 --- a/deploy/ansible/README.md +++ b/deploy/ansible/README.md @@ -28,4 +28,29 @@ ansible-playbook guestops.yml \ The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, and waits for loopback readiness. It then enables Docker and Nginx at boot, validates and installs the reviewed proxy site, rejects public API or MongoDB listeners, and checks the public redirect and certificate-backed HTTPS readiness before changing the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success. -This playbook does not provision DNS, TLS, Nginx, firewall rules, backup keys, monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities. +The release playbook installs the reviewed Nginx site but does not provision DNS, certificates, firewall rules, durable logging, backup keys, central monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities. + +## Backup and monitoring operations + +After the exact release is deployed, run `guestops-operations.yml`. It installs the repository systemd units, verifies their definitions, imports only the approved public recovery key, enables transfer retry and monitoring, connects the aggregate status file to Zabbix, and retains non-sensitive installation evidence. It never creates a recovery private key or writes transfer credentials. + +Before running it, provision these private host files through Ansible Vault or the established secret process: + +- `/etc/guestops/backup.env` mode `0600`; +- `/etc/guestops/backup-transfer.env` mode `0600`; +- `/etc/guestops/backup-transfer.key` mode `0600`; +- `/etc/guestops/backup-known-hosts` mode `0644`. + +The recovery public key must be available on the controller. Its private key and protected recovery copy must remain off the Debian host. Confirm durable restricted logging separately and pass `guestops_durable_logs_configured=true` only after that review. The daily backup timer defaults to disabled; set `guestops_enable_backup_schedule=true` only after the manual backup and service-recovery exercise passes. + +```sh +ansible-playbook guestops-operations.yml \ + -l guestops_sandbox \ + -e guestops_release_commit=FULL_40_CHARACTER_SHA \ + -e guestops_backup_recipient=FULL_40_CHARACTER_GPG_FINGERPRINT \ + -e guestops_recovery_public_key=/secure/recovery/guestops-public.asc \ + -e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 \ + -e guestops_durable_logs_configured=true +``` + +Run the manual encrypted backup, transfer, isolated restore and rollback exercises separately under the approved maintenance procedure. After they pass, rerun with `guestops_enable_backup_schedule=true`. Configure Zabbix trigger thresholds and escalation recipients in the central Zabbix environment; the playbook exposes only the non-sensitive `guestops.status` item. diff --git a/deploy/ansible/guestops-operations.yml b/deploy/ansible/guestops-operations.yml new file mode 100644 index 0000000..38fce5d --- /dev/null +++ b/deploy/ansible/guestops-operations.yml @@ -0,0 +1,227 @@ +--- +- name: Install GuestOps backup and monitoring operations + hosts: guestops + become: true + gather_facts: true + + vars: + guestops_root: /srv/guestops + guestops_current: "{{ guestops_root }}/current" + guestops_config_root: /etc/guestops + guestops_backup_root: /var/backups/guestops + guestops_gnupg_root: /var/lib/guestops-backup/gnupg + guestops_zabbix_service: zabbix-agent2 + guestops_zabbix_include_directory: /etc/zabbix/zabbix_agent2.d + guestops_enable_backup_schedule: false + guestops_durable_logs_configured: false + + pre_tasks: + - name: Validate required operational variables + ansible.builtin.assert: + that: + - guestops_release_commit is match('^[0-9a-f]{40}$') + - guestops_backup_recipient is match('^[A-Fa-f0-9]{40}$') + - guestops_recovery_public_key | length > 0 + - guestops_evidence_directory | length > 0 + - guestops_durable_logs_configured | bool + fail_msg: Release commit, recovery public key/fingerprint, evidence directory and reviewed durable logging are required. + + - name: Confirm selected release link + ansible.builtin.stat: + path: "{{ guestops_current }}" + follow: false + register: guestops_selected_release + + - name: Require deployed release before operations installation + ansible.builtin.assert: + that: + - guestops_selected_release.stat.exists + - guestops_selected_release.stat.islnk + - guestops_selected_release.stat.lnk_source == guestops_root + '/releases/' + guestops_release_commit + fail_msg: /srv/guestops/current must select the exact approved release commit. + + - name: Confirm selected release target exists + ansible.builtin.stat: + path: "{{ guestops_root }}/releases/{{ guestops_release_commit }}" + follow: true + register: guestops_release_target + + - name: Require selected release directory + ansible.builtin.assert: + that: + - guestops_release_target.stat.exists + - guestops_release_target.stat.isdir + fail_msg: The selected release target must be an installed directory. + + - name: Inspect pre-provisioned private operational files + ansible.builtin.stat: + path: "{{ item.path }}" + follow: false + loop: + - { path: /etc/guestops/backup.env, mode: "0600" } + - { path: /etc/guestops/backup-transfer.env, mode: "0600" } + - { path: /etc/guestops/backup-transfer.key, mode: "0600" } + - { path: /etc/guestops/backup-known-hosts, mode: "0644" } + register: guestops_operational_files + + - name: Require private backup and transfer configuration + ansible.builtin.assert: + that: + - item.stat.exists + - item.stat.isreg + - not item.stat.islnk + - item.stat.mode == item.item.mode + fail_msg: "{{ item.item.path }} must be a regular non-symlink file with mode {{ item.item.mode }}." + loop: "{{ guestops_operational_files.results }}" + no_log: true + + tasks: + - name: Create private backup directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - "{{ guestops_backup_root }}" + - "{{ guestops_gnupg_root }}" + + - name: Stage recovery public key + ansible.builtin.copy: + src: "{{ guestops_recovery_public_key }}" + dest: /etc/guestops/recovery-public.asc + owner: root + group: root + mode: "0600" + + - name: Import recovery public key only + ansible.builtin.command: + argv: + - gpg + - --batch + - --homedir + - "{{ guestops_gnupg_root }}" + - --import + - /etc/guestops/recovery-public.asc + no_log: true + changed_when: true + + - name: Verify configured recovery fingerprint + ansible.builtin.command: + argv: + - gpg + - --batch + - --homedir + - "{{ guestops_gnupg_root }}" + - --list-keys + - "{{ guestops_backup_recipient }}" + changed_when: false + no_log: true + + - name: Remove staged recovery public key + ansible.builtin.file: + path: /etc/guestops/recovery-public.asc + state: absent + + - name: Install GuestOps systemd units + ansible.builtin.copy: + src: "{{ playbook_dir }}/../systemd/{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - guestops-backup.service + - guestops-backup.timer + - guestops-backup-transfer.service + - guestops-backup-transfer.timer + - guestops-monitor-status.service + - guestops-monitor-status.timer + register: guestops_systemd_units + + - name: Verify installed systemd units + ansible.builtin.command: + argv: + - systemd-analyze + - verify + - /etc/systemd/system/guestops-backup.service + - /etc/systemd/system/guestops-backup.timer + - /etc/systemd/system/guestops-backup-transfer.service + - /etc/systemd/system/guestops-backup-transfer.timer + - /etc/systemd/system/guestops-monitor-status.service + - /etc/systemd/system/guestops-monitor-status.timer + changed_when: false + + - name: Reload systemd unit definitions + ansible.builtin.systemd_service: + daemon_reload: true + when: guestops_systemd_units.changed + + - name: Enable transfer retry and monitoring timers + ansible.builtin.systemd_service: + name: "{{ item }}" + enabled: true + state: started + loop: + - guestops-backup-transfer.timer + - guestops-monitor-status.timer + + - name: Select backup schedule state + ansible.builtin.systemd_service: + name: guestops-backup.timer + enabled: "{{ guestops_enable_backup_schedule | bool }}" + state: "{{ 'started' if guestops_enable_backup_schedule | bool else 'stopped' }}" + + - name: Require Zabbix include directory + ansible.builtin.stat: + path: "{{ guestops_zabbix_include_directory }}" + register: guestops_zabbix_directory + + - name: Confirm Zabbix agent configuration path + ansible.builtin.assert: + that: + - guestops_zabbix_directory.stat.exists + - guestops_zabbix_directory.stat.isdir + fail_msg: Override guestops_zabbix_include_directory for the installed Zabbix agent. + + - name: Install restricted Zabbix status item + ansible.builtin.copy: + src: "{{ playbook_dir }}/../systemd/zabbix-agent-guestops.conf.example" + dest: "{{ guestops_zabbix_include_directory }}/guestops.conf" + owner: root + group: root + mode: "0644" + register: guestops_zabbix_configuration + + - name: Restart Zabbix agent after configuration change + ansible.builtin.service: + name: "{{ guestops_zabbix_service }}" + enabled: true + state: restarted + when: guestops_zabbix_configuration.changed + + - name: Generate initial non-sensitive monitoring status + ansible.builtin.command: + argv: [systemctl, start, guestops-monitor-status.service] + changed_when: true + + - name: Read installed timer schedule + ansible.builtin.command: + argv: [systemctl, list-timers, --all, --no-pager, guestops-backup.timer, guestops-backup-transfer.timer, guestops-monitor-status.timer] + register: guestops_timer_status + changed_when: false + + - name: Retain monitoring status on the Ansible controller + ansible.builtin.fetch: + src: /run/guestops-monitor/status.json + dest: "{{ guestops_evidence_directory }}/monitor-status.json" + flat: true + + - name: Retain timer status on the Ansible controller + ansible.builtin.copy: + content: "{{ guestops_timer_status.stdout }}\n" + dest: "{{ guestops_evidence_directory }}/systemd-timers.txt" + mode: "0600" + delegate_to: localhost + become: false diff --git a/deploy/ansible/guestops.yml b/deploy/ansible/guestops.yml index a924bd3..102712f 100644 --- a/deploy/ansible/guestops.yml +++ b/deploy/ansible/guestops.yml @@ -5,7 +5,7 @@ gather_facts: true vars: - guestops_root: /opt/guestops + guestops_root: /srv/guestops guestops_config_root: /etc/guestops guestops_public_hostname: sandbox-guestops.futuresens.co.uk guestops_public_origin: "https://{{ guestops_public_hostname }}" diff --git a/deploy/systemd/guestops-backup-transfer.service b/deploy/systemd/guestops-backup-transfer.service index 44eac9e..351b549 100644 --- a/deploy/systemd/guestops-backup-transfer.service +++ b/deploy/systemd/guestops-backup-transfer.service @@ -2,15 +2,15 @@ Description=Transfer GuestOps encrypted backups to restricted storage Wants=network-online.target After=network-online.target guestops-backup.service -ConditionPathIsDirectory=/srv/guestops +ConditionPathIsDirectory=/srv/guestops/current ConditionPathIsDirectory=/var/backups/guestops [Service] Type=oneshot -WorkingDirectory=/srv/guestops +WorkingDirectory=/srv/guestops/current EnvironmentFile=/etc/guestops/backup-transfer.env UMask=0077 -ExecStart=/usr/bin/python3 /srv/guestops/deploy/backup_transfer.py --prune-verified --retention-days 7 +ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/backup_transfer.py --prune-verified --retention-days 7 NoNewPrivileges=true PrivateTmp=true ProtectHome=true diff --git a/deploy/systemd/guestops-backup.service b/deploy/systemd/guestops-backup.service index fa2685d..14bd15d 100644 --- a/deploy/systemd/guestops-backup.service +++ b/deploy/systemd/guestops-backup.service @@ -2,17 +2,17 @@ Description=GuestOps encrypted maintenance backup Requires=docker.service After=docker.service -ConditionPathIsDirectory=/srv/guestops +ConditionPathIsDirectory=/srv/guestops/current ConditionPathIsDirectory=/var/backups/guestops ConditionPathIsDirectory=/var/lib/guestops-backup/gnupg [Service] Type=oneshot -WorkingDirectory=/srv/guestops +WorkingDirectory=/srv/guestops/current EnvironmentFile=/etc/guestops/backup.env Environment=GNUPGHOME=/var/lib/guestops-backup/gnupg UMask=0077 -ExecStart=/usr/bin/python3 /srv/guestops/deploy/ops.py scheduled-backup --confirm-maintenance +ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/ops.py scheduled-backup --confirm-maintenance NoNewPrivileges=true PrivateTmp=true ProtectHome=true diff --git a/deploy/systemd/guestops-monitor-status.service b/deploy/systemd/guestops-monitor-status.service index c369c6e..ce9ddeb 100644 --- a/deploy/systemd/guestops-monitor-status.service +++ b/deploy/systemd/guestops-monitor-status.service @@ -2,14 +2,14 @@ Description=Write non-sensitive GuestOps monitoring status Requires=docker.service After=docker.service network-online.target -ConditionPathIsDirectory=/srv/guestops +ConditionPathIsDirectory=/srv/guestops/current ConditionPathIsDirectory=/var/backups/guestops [Service] Type=oneshot -WorkingDirectory=/srv/guestops +WorkingDirectory=/srv/guestops/current UMask=0022 -ExecStart=/usr/bin/python3 /srv/guestops/deploy/monitor_status.py +ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/monitor_status.py NoNewPrivileges=true PrivateTmp=true ProtectHome=true diff --git a/docs/operations.md b/docs/operations.md index f753440..04949aa 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -87,7 +87,7 @@ Copy the encrypted file off-server to restricted storage after every successful ### Optional systemd schedule -The repository includes an opt-in daily systemd service and timer. They are templates, not automatically installed. The service assumes the reviewed checkout is `/srv/guestops` and stages encrypted files in `/var/backups/guestops`; review and change both unit files together if the host uses different paths. +The repository includes an opt-in daily systemd service and timer. The service follows the Ansible-selected release at `/srv/guestops/current` and stages encrypted files in `/var/backups/guestops`. The application-owned `deploy/ansible/guestops-operations.yml` installs and verifies these units after the exact release is selected; keep the backup timer disabled until the supervised manual backup passes. Create the staging directory and environment file without storing a private key or passphrase on the server: diff --git a/tests/test_ansible_operations.py b/tests/test_ansible_operations.py new file mode 100644 index 0000000..7c0e5cf --- /dev/null +++ b/tests/test_ansible_operations.py @@ -0,0 +1,53 @@ +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +PLAYBOOK = ROOT / "deploy" / "ansible" / "guestops-operations.yml" + + +class AnsibleOperationsTests(unittest.TestCase): + def test_operations_require_release_and_private_inputs(self): + text = PLAYBOOK.read_text(encoding="utf-8") + for required in ( + "/srv/guestops/current", + "guestops_release_commit is match('^[0-9a-f]{40}$')", + "guestops_backup_recipient is match('^[A-Fa-f0-9]{40}$')", + "guestops_durable_logs_configured | bool", + "/etc/guestops/backup.env", + "/etc/guestops/backup-transfer.env", + "/etc/guestops/backup-transfer.key", + "/etc/guestops/backup-known-hosts", + "no_log: true", + ): + self.assertIn(required, text) + + def test_public_key_units_monitoring_and_evidence_are_integrated(self): + text = PLAYBOOK.read_text(encoding="utf-8") + ordered = ( + "Import recovery public key only", + "Verify installed systemd units", + "Enable transfer retry and monitoring timers", + "Install restricted Zabbix status item", + "Generate initial non-sensitive monitoring status", + "Retain monitoring status on the Ansible controller", + ) + positions = [text.index(item) for item in ordered] + self.assertEqual(positions, sorted(positions)) + self.assertIn("guestops_enable_backup_schedule: false", text) + self.assertNotIn("ansible.builtin.shell", text) + self.assertNotIn("PRIVATE KEY", text) + + def test_systemd_units_follow_selected_release(self): + for name in ( + "guestops-backup.service", + "guestops-backup-transfer.service", + "guestops-monitor-status.service", + ): + text = (ROOT / "deploy" / "systemd" / name).read_text(encoding="utf-8") + self.assertIn("/srv/guestops/current", text) + self.assertNotIn("WorkingDirectory=/srv/guestops\n", text) + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From c5c6ec1453437b158df0e9ecf68ea317575fa9b2 Mon Sep 17 00:00:00 2001 From: mathew Date: Thu, 1 Oct 2026 15:40:28 +0100 Subject: [PATCH 10/11] Integrate Gate B acceptance bundle --- MILESTONES.md | 12 +- deploy/gate_b_bundle.py | 220 ++++++++++++++++++++++++++++++++++++ docs/pilot-release.md | 26 +++++ tests/test_gate_b_bundle.py | 141 +++++++++++++++++++++++ 4 files changed, 393 insertions(+), 6 deletions(-) create mode 100644 deploy/gate_b_bundle.py create mode 100644 tests/test_gate_b_bundle.py diff --git a/MILESTONES.md b/MILESTONES.md index 7f6f6e8..337a7db 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -22,13 +22,13 @@ This summary explains what each milestone delivers and where it currently stands | 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The runner-free deterministic packager is implemented and the Gitea Action is removed; the release-line identity must be confirmed, then the package, Ansible installation evidence, and approval record must be retained. | | 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** The verified Ansible handoff now covers commit-bound installation, boot services, Nginx validation, listener restrictions and public HTTPS; privileged installation, firewall review, controlled reboot and supervised persistence evidence remain open. | | 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** The Ansible operations handoff now installs validated systemd units, public-key-only backup support, transfer retry and restricted Zabbix status; secret provisioning, durable-log confirmation, manual backup, timed restore, rollback and independent evidence remain open. | -| 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook and validator exist; the live synthetic-data exercise and independent review remain outstanding. | +| 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook, record validator and release-bound Gate B bundle integration exist; the live synthetic-data exercise and independent review remain outstanding. | | 13 | Rezlynx/Guestline adapter | The real PMS provider adapter, mappings, idempotency, reconciliation, and ambiguous-write handling. | **Planned.** Provider contract and sandbox access are still required before implementation and acceptance. | | 14 | Payment links and status | The real payment-provider integration, webhooks, expiry, replay protection, and reconciliation. | **Planned.** The provider path and sandbox acceptance plan still need to be confirmed and completed. | -| 15 | Knowledge, AI, and FAQ activation | Supervised knowledge-quality, AI-draft, FAQ test-mode, staff-training, and stop-control acceptance. | **Implemented; acceptance required.** The evaluation tooling exists; the supervised evaluation and independent approval remain outstanding. | +| 15 | Knowledge, AI, and FAQ activation | Supervised knowledge-quality, AI-draft, FAQ test-mode, staff-training, and stop-control acceptance. | **Implemented; acceptance required.** Evaluation tooling and cross-release bundle validation exist; the supervised evaluation, staff training and independent approval remain outstanding. | | 16 | Identity, preferences, and privacy | Account/session controls, hotel preferences, privacy inventory, retention decisions, and audit review. | **Implemented; acceptance required.** Legal and operational decisions, identity checks, and independent review remain outstanding. | | 17 | Inbox usability and desktop parity | Stable pagination, protected unsaved drafts, hotel-timezone display, and desktop workflow parity. | **Implemented; acceptance required.** Automated checks pass; the supervised desktop exercise and independent approval remain outstanding. | -| 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** Validators and targets exist; Gate A/B prerequisites, capacity evidence, incident rehearsal, pilot, and named approvals remain open. | +| 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** The integrated bundle validator now enforces one archive, release record, image set, environment, capacity report, pilot record and approval decision; live prerequisites, incident rehearsal, five-day pilot and named approvals remain open. | | 19 | Account security and self-service | TOTP MFA, recovery codes, transactional email, granular roles, preferences, and security notifications. | **Implemented on the development branch; acceptance required.** Keep it separate until `0.2.1` is approved and tagged, then review, merge, and version it as `0.3.0`. | ## Status key @@ -62,13 +62,13 @@ This summary explains what each milestone delivers and where it currently stands | 9 | Gitea and reproducible releases | A | In progress | `deploy/package_source.py` now packages only an explicit committed ref, verifies matched application versions, produces deterministic gzip output and a SHA-256 source record, and refuses overwrite. Hand that package to a version-selected Ansible playbook following the CMS/CMSFront pattern. Ansible must verify and install it, build commit-tagged images, record their immutable IDs, and deploy without a Gitea runner. Retain the package/install evidence off-host and resolve the release-line/tag identity before approval; the existing `0.1.0` tag remains attached to the foundation release. | | 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The Ansible handoff verifies the source on both controller and host, enables Docker/Nginx at boot, installs and validates the reviewed proxy, rejects exposed API/MongoDB listeners, and requires trusted public HTTPS before selecting the release. Run it on the provisioned Debian host, review the firewall, complete the confirmation-gated persistence drill and controlled reboot, and retain independent evidence. | | 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling includes opt-in systemd scheduling, checksum-verified rsync transfer, restricted Zabbix status, guarded local retention and a release-bound acceptance validator. The Ansible operations playbook now verifies the selected release and private-file modes, imports only the recovery public key, validates and installs the units, enables transfer/monitoring, leaves backup scheduling off until manual acceptance, and fetches non-sensitive evidence. Provision secrets and durable logs, configure central alerts/retention, run the manual backup plus timed restore and rollback drills, and retain independent approval. | -| 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | +| 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented and wired into the Gate B bundle validator. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. | | 14 | Payment links and status | C | Planned | Select/confirm the payment-provider path, complete sandbox and webhook acceptance, and prove expiry, replay protection, reconciliation, and support recovery. | -| 15 | Knowledge, AI, and FAQ activation | B | Implemented / acceptance required | Owners can run a bounded no-send batch evaluation, and a release-bound acceptance record enforces positive/negative coverage, zero FAQ errors, separate AI review, staff training, stop-control evidence and named monitoring/rollback owners. Complete the supervised evaluation and retain independent approval. | +| 15 | Knowledge, AI, and FAQ activation | B | Implemented / acceptance required | Owners can run a bounded no-send batch evaluation, and a release-bound acceptance record enforces positive/negative coverage, zero FAQ errors, separate AI review, staff training, stop-control evidence and named monitoring/rollback owners. The integrated Gate B bundle rejects release or environment mismatches. Complete the supervised evaluation and retain independent approval. | | 16 | Identity, preferences, and privacy | B/C | Implemented / acceptance required | Login throttling trusts the client address only after one-hop processing by the configured proxy. A release-bound review now covers owner-controlled preferences, account/session controls, data inventory, retention/deletion/legal-hold ownership, provider decisions, audit evidence and known identity limitations. Complete the legal/operational decisions and independently approve the record. | | 17 | Inbox usability and desktop parity | B | Implemented / acceptance required | The inbox uses tenant-scoped stable cursor pagination in pages of 50 and protects unsaved drafts during route/history navigation, reload, conversation selection, filtering and search. Operational timestamps use the saved hotel timezone, and a release-bound desktop-parity acceptance record is implemented. The implementation and preview HTTP suite pass; run the supervised exercise against the approved release and retain independent approval. | -| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.1` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision record validators with agreed targets. Retain the exact source-package and Ansible-install evidence, complete Gate A and Gate B prerequisites, run the probe and supervised exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | +| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.1` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision validators plus an integrated bundle check that binds every prerequisite to one release and verifies the retained capacity/pilot checksums. Retain the exact package and Ansible evidence, complete the live Gate A/B exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | ## Delivery sequence diff --git a/deploy/gate_b_bundle.py b/deploy/gate_b_bundle.py new file mode 100644 index 0000000..a6f26d0 --- /dev/null +++ b/deploy/gate_b_bundle.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Validate the complete release-bound GuestOps Gate B acceptance bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import sys +from urllib.parse import urlparse + + +DEPLOY = Path(__file__).resolve().parent +if str(DEPLOY) not in sys.path: + sys.path.insert(0, str(DEPLOY)) + +import automation_acceptance +import backup_restore_acceptance +import debian_acceptance +import desktop_acceptance +import google_acceptance +import identity_privacy_acceptance +import incident_exercise +import pilot_approval +import pilot_run +import verify_release +import verify_source_package + + +VERSION = "0.2.1" +SHA256 = re.compile(r"[0-9a-f]{64}") +RECORD_KEYS = { + "debian-host", "persistence", "backup-restore", "google-mailbox", + "automation", "identity-privacy", "inbox-usability", "capacity", + "incident-support", "pilot-findings", "pilot-approval", +} + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def load_json(path: Path) -> object: + require(path.is_file() and not path.is_symlink(), f"Required bundle file is missing or is a symlink: {path.name}") + return json.loads(path.read_text(encoding="utf-8")) + + +def release_binding(record: object, name: str, commit: str, release_sha: str) -> None: + require(isinstance(record, dict), f"{name} must be a JSON object.") + require(record.get("releaseCommit") == commit, f"{name} uses a different releaseCommit.") + require(record.get("releaseRecordSha256") == release_sha, + f"{name} uses a different releaseRecordSha256.") + + +def environment_origin(record: object, name: str) -> str: + require(isinstance(record, dict), f"{name} must be a JSON object.") + value = str(record.get("environment", "")).rstrip("/") + parsed = urlparse(value) + require(parsed.scheme == "https" and parsed.hostname and parsed.path in ("", "/"), + f"{name} has no valid HTTPS environment.") + return value + + +def validate_capacity(report: object, commit: str, release_sha: str) -> None: + require(isinstance(report, dict), "capacity must be a JSON object.") + require(report.get("schemaVersion") == 1 and report.get("kind") == "guestops-read-only-capacity", + "capacity has an unsupported schema or kind.") + require(report.get("releaseCommit") == commit, "capacity uses a different releaseCommit.") + require(report.get("releaseRecordSha256") == release_sha, + "capacity uses a different releaseRecordSha256.") + require(report.get("paths") == ["/health/ready", "/api/hotel", "/api/conversations/page"], + "capacity must use the exact read-only path set.") + for field in ("concurrency", "requests", "successes", "failures"): + require(isinstance(report.get(field), int) and not isinstance(report.get(field), bool), + f"capacity.{field} must be an integer.") + require(1 <= report["concurrency"] <= 20 and report["requests"] > 0, + "capacity has invalid concurrency or request count.") + require(report["successes"] + report["failures"] == report["requests"], + "capacity success and failure counts do not match requests.") + expected_error = round(report["failures"] / report["requests"], 6) + require(report.get("errorRate") == expected_error, "capacity.errorRate does not match its counts.") + latency = report.get("latencyMs") + require(isinstance(latency, dict) and set(latency) == {"median", "p95", "maximum"} + and all(isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 + for value in latency.values()), "capacity.latencyMs is invalid.") + require(latency["median"] <= latency["p95"] <= latency["maximum"], + "capacity latency percentiles are out of order.") + + +def validate_bundle( + source_archive: Path, + source_record_path: Path, + release_record_path: Path, + records: dict[str, tuple[Path, object]], + host_metrics_path: Path, +) -> dict[str, object]: + require(set(records) == RECORD_KEYS, "Gate B bundle requires the exact record set.") + release_record = load_json(release_record_path) + require(isinstance(release_record, dict), "release-record must be a JSON object.") + commit = str(release_record.get("commit", "")) + version = str(release_record.get("version", "")) + require(version == VERSION, f"Gate B bundle version must be {VERSION}.") + release_result = verify_release.validate(source_archive, release_record_path, commit, version) + source_result = verify_source_package.validate(source_archive, source_record_path, commit, version) + release_sha = str(release_result["releaseRecord"]["sha256"]) + require(source_result["artifact"]["sha256"] == release_result["archive"]["sha256"], + "Source and release records identify different archives.") + + values = {name: value for name, (_, value) in records.items()} + for name, record in values.items(): + release_binding(record, name, commit, release_sha) + + debian_acceptance.validate_pair(values["debian-host"], values["persistence"], commit, release_sha) + backup_restore_acceptance.validate(values["backup-restore"], commit, release_sha) + google_acceptance.validate(values["google-mailbox"]) + automation_acceptance.validate(values["automation"]) + identity_privacy_acceptance.validate(values["identity-privacy"]) + desktop_acceptance.validate(values["inbox-usability"]) + validate_capacity(values["capacity"], commit, release_sha) + incident_exercise.validate(values["incident-support"]) + pilot_run.validate(values["pilot-findings"]) + pilot_approval.validate(values["pilot-approval"]) + + environment_names = { + "debian-host", "persistence", "backup-restore", "google-mailbox", + "automation", "identity-privacy", "inbox-usability", "incident-support", + "pilot-findings", + } + origins = {environment_origin(values[name], name) for name in environment_names} + require(len(origins) == 1, "Gate B records use different environments.") + origin = next(iter(origins)) + require(values["capacity"].get("originHost") == urlparse(origin).hostname, + "capacity originHost does not match the accepted environment.") + + archive_sha = str(release_result["archive"]["sha256"]) + for name in ("debian-host", "persistence", "backup-restore"): + require(values[name].get("archiveSha256") == archive_sha, + f"{name} uses a different archiveSha256.") + approved_images = release_record["images"] + for name in ("debian-host", "persistence", "backup-restore"): + images = values[name].get("images", {}) + require(images.get("api") == approved_images["api"] and images.get("worker") == approved_images["worker"], + f"{name} uses different API or worker image identities.") + + approval_capacity = values["pilot-approval"]["capacity"] + capacity = values["capacity"] + require(approval_capacity["reportSha256"] == digest(records["capacity"][0]), + "Pilot approval capacity report checksum does not match the retained report.") + require(approval_capacity["hostMetricsSha256"] == digest(host_metrics_path), + "Pilot approval host metrics checksum does not match the retained file.") + require(approval_capacity["observedConcurrency"] == capacity["concurrency"], + "Pilot approval concurrency does not match the capacity report.") + require(approval_capacity["observedP95Ms"] == capacity["latencyMs"]["p95"], + "Pilot approval p95 does not match the capacity report.") + require(approval_capacity["observedErrorRate"] == capacity["errorRate"], + "Pilot approval error rate does not match the capacity report.") + require(values["pilot-approval"]["pilot"]["recordSha256"] == digest(records["pilot-findings"][0]), + "Pilot approval checksum does not match the retained pilot run record.") + + summary_records = {name: digest(path) for name, (path, _) in sorted(records.items())} + return { + "archiveSha256": archive_sha, + "environmentHost": urlparse(origin).hostname, + "releaseCommit": commit, + "releaseRecordSha256": release_sha, + "records": summary_records, + "schemaVersion": 1, + "sourceRecordSha256": source_result["sourceRecord"]["sha256"], + "validated": True, + "version": version, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-archive", required=True, type=Path) + parser.add_argument("--source-record", required=True, type=Path) + parser.add_argument("--release-record", required=True, type=Path) + parser.add_argument("--host-metrics", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + for name in sorted(RECORD_KEYS): + parser.add_argument("--" + name, required=True, type=Path) + args = parser.parse_args() + try: + require(args.host_metrics.is_file() and not args.host_metrics.is_symlink(), + "Host metrics file is missing or is a symlink.") + require(not args.output.exists() and args.output.parent.is_dir(), + "Output must be a new file in an existing restricted directory.") + records = {} + for name in RECORD_KEYS: + path = getattr(args, name.replace("-", "_")) + records[name] = (path, load_json(path)) + summary = validate_bundle( + args.source_archive, + args.source_record, + args.release_record, + records, + args.host_metrics, + ) + descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + output.write(json.dumps(summary, indent=2, sort_keys=True) + "\n") + print("Gate B bundle is structurally complete, consistently release-bound and approved. " + "This validates records and checksums, not the underlying restricted evidence.") + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"Gate B bundle rejected: {error}", file=sys.stderr) + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/docs/pilot-release.md b/docs/pilot-release.md index ce67efb..ffb2201 100644 --- a/docs/pilot-release.md +++ b/docs/pilot-release.md @@ -62,6 +62,32 @@ python3 deploy/pilot_approval.py /secure/acceptance/pilot-approval.json The validator requires the exact Gate B evidence set, or that set plus independently accepted PMS and payment-provider evidence for Gate C. It also verifies that observed concurrency meets the pre-agreed target and that p95 latency and error rate remain within their pre-agreed bounds. Structural validation does not inspect evidence or authorize rollout by itself. +## Integrated Gate B bundle validation + +After every individual record passes independent review, validate the complete bundle together. This prevents records from different commits, release records, archives, image builds or sandbox environments from being combined into one approval. It also binds the final decision to the retained capacity report, host metrics and five-day pilot record by checksum. + +```sh +python3 deploy/gate_b_bundle.py \ + --source-archive /secure/releases/GuestOps-0.2.1-COMMIT.tar.gz \ + --source-record /secure/releases/GuestOps-0.2.1-COMMIT.source.json \ + --release-record /secure/releases/release-record.json \ + --host-metrics /secure/acceptance/capacity-host-metrics.json \ + --debian-host /secure/acceptance/debian-host.json \ + --persistence /secure/acceptance/persistence.json \ + --backup-restore /secure/acceptance/backup-restore.json \ + --google-mailbox /secure/acceptance/google-mailbox.json \ + --automation /secure/acceptance/automation.json \ + --identity-privacy /secure/acceptance/identity-privacy.json \ + --inbox-usability /secure/acceptance/inbox-usability.json \ + --capacity /secure/acceptance/capacity.json \ + --incident-support /secure/acceptance/incident-support.json \ + --pilot-findings /secure/acceptance/pilot-run.json \ + --pilot-approval /secure/acceptance/pilot-approval.json \ + --output /secure/acceptance/gate-b-bundle-summary.json +``` + +The output contains only release identity and file checksums, uses mode `0600`, and refuses to overwrite an existing summary. Retain it with the individual validator outputs. A passing bundle proves structural consistency, not that screenshots, provider activity, approvals, findings or other referenced evidence are genuine. + Complete the [incident and rollback exercise](incident-exercise.md) before marking `incident-support` as passed. Its record must use the same release identifiers and target gate as this decision. Reference the retained exercise record and validator output; do not substitute a local automated-test result for the supervised exercise. Complete the [desktop-parity acceptance exercise](desktop-acceptance.md) before marking `inbox-usability` as passed. Bind it to the same release identifiers and retain its independently reviewed record outside the repository. diff --git a/tests/test_gate_b_bundle.py b/tests/test_gate_b_bundle.py new file mode 100644 index 0000000..c9742ce --- /dev/null +++ b/tests/test_gate_b_bundle.py @@ -0,0 +1,141 @@ +import contextlib +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("gate_b_bundle", ROOT / "deploy" / "gate_b_bundle.py") +bundle = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bundle) + + +COMMIT = "a" * 40 +API = {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "b" * 64} +WORKER = {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "c" * 64} +ORIGIN = "https://sandbox-guestops.futuresens.co.uk" + + +class GateBBundleTests(unittest.TestCase): + def fixture(self, directory: str): + root = Path(directory) + archive = root / "GuestOps-0.2.1-aaaaaaaaaaaa.tar.gz" + source = root / "GuestOps-0.2.1-aaaaaaaaaaaa.source.json" + release = root / "release-record.json" + metrics = root / "host-metrics.json" + archive.write_bytes(b"approved source") + artifact = { + "name": archive.name, + "size": archive.stat().st_size, + "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + } + source.write_text(json.dumps({"schemaVersion": 1, "version": "0.2.1", "commit": COMMIT, "artifact": artifact}), encoding="utf-8") + release.write_text(json.dumps({ + "schemaVersion": 1, "version": "0.2.1", "commit": COMMIT, + "artifact": artifact, "images": {"api": API, "worker": WORKER}, + }), encoding="utf-8") + release_sha = hashlib.sha256(release.read_bytes()).hexdigest() + metrics.write_bytes(b'{"cpu":40,"memory":35}') + + values = {} + paths = {} + for name in bundle.RECORD_KEYS: + value = {"releaseCommit": COMMIT, "releaseRecordSha256": release_sha} + if name in { + "debian-host", "persistence", "backup-restore", "google-mailbox", + "automation", "identity-privacy", "inbox-usability", "incident-support", + "pilot-findings", + }: + value["environment"] = ORIGIN + if name in {"debian-host", "persistence", "backup-restore"}: + value["archiveSha256"] = artifact["sha256"] + value["images"] = {"api": API, "worker": WORKER} + values[name] = value + + values["capacity"].update({ + "schemaVersion": 1, "kind": "guestops-read-only-capacity", + "originHost": "sandbox-guestops.futuresens.co.uk", + "paths": ["/health/ready", "/api/hotel", "/api/conversations/page"], + "concurrency": 10, "requests": 100, "successes": 100, "failures": 0, + "errorRate": 0.0, "latencyMs": {"median": 100, "p95": 250, "maximum": 300}, + }) + pilot_path = root / "pilot-findings.json" + pilot_path.write_text(json.dumps(values["pilot-findings"]), encoding="utf-8") + paths["pilot-findings"] = pilot_path + capacity_path = root / "capacity.json" + capacity_path.write_text(json.dumps(values["capacity"]), encoding="utf-8") + paths["capacity"] = capacity_path + values["pilot-approval"].update({ + "capacity": { + "reportSha256": hashlib.sha256(capacity_path.read_bytes()).hexdigest(), + "hostMetricsSha256": hashlib.sha256(metrics.read_bytes()).hexdigest(), + "observedConcurrency": 10, "observedP95Ms": 250, "observedErrorRate": 0.0, + }, + "pilot": {"recordSha256": hashlib.sha256(pilot_path.read_bytes()).hexdigest()}, + }) + for name in bundle.RECORD_KEYS - paths.keys(): + path = root / f"{name}.json" + path.write_text(json.dumps(values[name]), encoding="utf-8") + paths[name] = path + records = {name: (paths[name], values[name]) for name in bundle.RECORD_KEYS} + return archive, source, release, metrics, records + + @contextlib.contextmanager + def mocked_individual_validators(self): + patches = [ + patch.object(bundle.debian_acceptance, "validate_pair"), + patch.object(bundle.backup_restore_acceptance, "validate"), + patch.object(bundle.google_acceptance, "validate"), + patch.object(bundle.automation_acceptance, "validate"), + patch.object(bundle.identity_privacy_acceptance, "validate"), + patch.object(bundle.desktop_acceptance, "validate"), + patch.object(bundle.incident_exercise, "validate"), + patch.object(bundle.pilot_run, "validate"), + patch.object(bundle.pilot_approval, "validate"), + ] + with contextlib.ExitStack() as stack: + for item in patches: + stack.enter_context(item) + yield + + def test_complete_bundle_is_bound_to_one_release(self): + with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): + archive, source, release, metrics, records = self.fixture(directory) + result = bundle.validate_bundle(archive, source, release, records, metrics) + self.assertTrue(result["validated"]) + self.assertEqual(result["releaseCommit"], COMMIT) + self.assertEqual(set(result["records"]), bundle.RECORD_KEYS) + + def test_different_environment_or_checksum_is_rejected(self): + with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): + archive, source, release, metrics, records = self.fixture(directory) + records["google-mailbox"][1]["environment"] = "https://other.example.invalid" + with self.assertRaisesRegex(ValueError, "different environments"): + bundle.validate_bundle(archive, source, release, records, metrics) + + with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): + archive, source, release, metrics, records = self.fixture(directory) + records["pilot-approval"][1]["capacity"]["reportSha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "capacity report checksum"): + bundle.validate_bundle(archive, source, release, records, metrics) + + def test_capacity_counts_and_latency_are_checked(self): + report = { + "schemaVersion": 1, "kind": "guestops-read-only-capacity", + "releaseCommit": COMMIT, "releaseRecordSha256": "d" * 64, + "paths": ["/health/ready", "/api/hotel", "/api/conversations/page"], + "concurrency": 10, "requests": 10, "successes": 9, "failures": 1, + "errorRate": 0.1, "latencyMs": {"median": 10, "p95": 20, "maximum": 30}, + } + bundle.validate_capacity(report, COMMIT, "d" * 64) + report["failures"] = 2 + with self.assertRaisesRegex(ValueError, "do not match requests"): + bundle.validate_capacity(report, COMMIT, "d" * 64) + + +if __name__ == "__main__": + unittest.main() -- 2.20.1 From db8422afda51b2ebd094eb7cc84be78bb6f3d70a Mon Sep 17 00:00:00 2001 From: mathew Date: Wed, 21 Oct 2026 11:55:33 +0100 Subject: [PATCH 11/11] GuestOps asible playbook --- .env.example | 9 +- Dockerfile | 6 +- MILESTONES.md | 24 +- README.md | 4 +- compose.yml | 37 +- deploy/ansible/README.md | 56 --- deploy/ansible/guestops-operations.yml | 227 ---------- deploy/ansible/guestops.yml | 402 ------------------ deploy/backup-restore-acceptance.example.json | 81 ---- deploy/backup_restore_acceptance.py | 220 ---------- deploy/backup_transfer.py | 238 ----------- deploy/capacity_probe.py | 128 ------ deploy/debian-host-acceptance.example.json | 45 -- deploy/debian_acceptance.py | 206 --------- deploy/gate_b_bundle.py | 220 ---------- deploy/google-acceptance.example.json | 2 +- deploy/mongo-init.js | 2 - deploy/monitor_status.py | 189 -------- deploy/nginx.conf | 31 -- deploy/ops.py | 344 --------------- deploy/package_source.py | 36 +- deploy/persistence-acceptance.example.json | 31 -- deploy/release_record.py | 76 ---- deploy/systemd/backup-transfer.env.example | 6 - deploy/systemd/backup.env.example | 2 - .../systemd/guestops-backup-transfer.service | 19 - deploy/systemd/guestops-backup-transfer.timer | 11 - deploy/systemd/guestops-backup.service | 21 - deploy/systemd/guestops-backup.timer | 11 - .../systemd/guestops-monitor-status.service | 21 - deploy/systemd/guestops-monitor-status.timer | 11 - .../zabbix-agent-guestops.conf.example | 2 - deploy/verify_release.py | 118 ----- deploy/verify_source_package.py | 104 ----- docs/accounts.md | 2 +- docs/deployment.md | 145 ++----- docs/google-acceptance.md | 2 +- docs/incident-exercise.md | 2 +- docs/migration.md | 4 +- docs/operations.md | 208 --------- docs/pilot-release.md | 93 ---- src/GuestOps.Api/GoogleMailbox.cs | 2 +- tests/production_smoke.py | 2 +- tests/test_ansible_handoff.py | 53 --- tests/test_ansible_operations.py | 53 --- tests/test_backup_restore_acceptance.py | 129 ------ tests/test_backup_transfer.py | 81 ---- tests/test_capacity_probe.py | 26 -- tests/test_debian_acceptance.py | 113 ----- tests/test_desktop_acceptance.py | 2 +- tests/test_gate_b_acceptance.py | 2 +- tests/test_gate_b_bundle.py | 141 ------ tests/test_google_acceptance.py | 2 +- tests/test_incident_exercise.py | 2 +- tests/test_monitor_status.py | 75 ---- tests/test_ops.py | 186 -------- tests/test_package_source.py | 17 +- tests/test_pilot_run.py | 2 +- tests/test_release_record.py | 82 ---- tests/test_verify_release.py | 87 ---- tests/test_verify_source_package.py | 72 ---- 61 files changed, 108 insertions(+), 4417 deletions(-) delete mode 100644 deploy/ansible/README.md delete mode 100644 deploy/ansible/guestops-operations.yml delete mode 100644 deploy/ansible/guestops.yml delete mode 100644 deploy/backup-restore-acceptance.example.json delete mode 100644 deploy/backup_restore_acceptance.py delete mode 100644 deploy/backup_transfer.py delete mode 100644 deploy/capacity_probe.py delete mode 100644 deploy/debian-host-acceptance.example.json delete mode 100644 deploy/debian_acceptance.py delete mode 100644 deploy/gate_b_bundle.py delete mode 100644 deploy/mongo-init.js delete mode 100644 deploy/monitor_status.py delete mode 100644 deploy/nginx.conf delete mode 100644 deploy/ops.py delete mode 100644 deploy/persistence-acceptance.example.json delete mode 100644 deploy/release_record.py delete mode 100644 deploy/systemd/backup-transfer.env.example delete mode 100644 deploy/systemd/backup.env.example delete mode 100644 deploy/systemd/guestops-backup-transfer.service delete mode 100644 deploy/systemd/guestops-backup-transfer.timer delete mode 100644 deploy/systemd/guestops-backup.service delete mode 100644 deploy/systemd/guestops-backup.timer delete mode 100644 deploy/systemd/guestops-monitor-status.service delete mode 100644 deploy/systemd/guestops-monitor-status.timer delete mode 100644 deploy/systemd/zabbix-agent-guestops.conf.example delete mode 100644 deploy/verify_release.py delete mode 100644 deploy/verify_source_package.py delete mode 100644 docs/operations.md delete mode 100644 docs/pilot-release.md delete mode 100644 tests/test_ansible_handoff.py delete mode 100644 tests/test_ansible_operations.py delete mode 100644 tests/test_backup_restore_acceptance.py delete mode 100644 tests/test_backup_transfer.py delete mode 100644 tests/test_capacity_probe.py delete mode 100644 tests/test_debian_acceptance.py delete mode 100644 tests/test_gate_b_bundle.py delete mode 100644 tests/test_monitor_status.py delete mode 100644 tests/test_ops.py delete mode 100644 tests/test_release_record.py delete mode 100644 tests/test_verify_release.py delete mode 100644 tests/test_verify_source_package.py diff --git a/.env.example b/.env.example index 2fc0dee..8500578 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,10 @@ -# Generate separate 32-byte hex values with: openssl rand -hex 32 # Keep .env private (chmod 600). Never commit real values. -MONGO_ROOT_PASSWORD= -MONGO_APP_PASSWORD= +# The MongoDB URI must use an application-scoped user and a host address that +# is reachable from Docker; 127.0.0.1 inside a container is the container. +MONGO_CONNECTION_STRING= +MONGO_DATABASE=guestops +GUESTOPS_PUBLIC_ORIGIN=https://guestops-sandbox.futuresens.co.uk +GUESTOPS_ALLOWED_HOSTS=guestops-sandbox.futuresens.co.uk;localhost;127.0.0.1 GOOGLE_CLIENT_ID= GOOGLE_CLIENT_SECRET= GOOGLE_ENABLE_SENDING=false diff --git a/Dockerfile b/Dockerfile index 8714577..e80dcd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,11 @@ RUN dotnet publish src/GuestOps.Worker/GuestOps.Worker.csproj -c Release -o /pub FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS api WORKDIR /app -RUN mkdir -p /var/lib/guestops/keys && chown -R app:app /var/lib/guestops && chmod 700 /var/lib/guestops/keys +RUN groupadd --gid 25001 guestops-secrets \ + && usermod --append --groups guestops-secrets app \ + && mkdir -p /var/lib/guestops/keys \ + && chown -R app:app /var/lib/guestops \ + && chmod 700 /var/lib/guestops/keys COPY --from=backend /publish/api ./ COPY --from=frontend /build/web/dist ./wwwroot/ ENV ASPNETCORE_URLS=http://+:8080 ASPNETCORE_ENVIRONMENT=Production Keys__Path=/var/lib/guestops/keys diff --git a/MILESTONES.md b/MILESTONES.md index 337a7db..595e4e2 100644 --- a/MILESTONES.md +++ b/MILESTONES.md @@ -18,17 +18,17 @@ This summary explains what each milestone delivers and where it currently stands | 5 | FAQ automation | Knowledge-based FAQ drafting, test mode, approval controls, and guarded live automation. | **Implemented; acceptance required.** Live mode remains disabled pending quality, false-positive, monitoring, and rollback acceptance. | | 6 | Team onboarding and account recovery | Staff invitations, password setup/reset, access disable/restore, and administrator recovery. | **Implemented; acceptance required.** Deployed link delivery, expiry, recovery, and administrator procedures still need operational evidence. | | 7 | Google connection recovery | OAuth reconnect, checkpoint recovery, grant revocation handling, and worker restart safety. | **Implemented; acceptance required.** Dedicated Google-account and worker-restart exercises have not yet been accepted. | -| 8 | Operational readiness tooling | Release verification, diagnostics, encrypted backup, restore, preflight, and persistence tools. | **Implemented; acceptance required.** The tools must still be run against the exact release on the Debian host. | -| 9 | Gitea and reproducible releases | A checksummed source package, Ansible-controlled installation, recorded image identities, retained artifacts, and approval tagging. | **In progress.** The runner-free deterministic packager is implemented and the Gitea Action is removed; the release-line identity must be confirmed, then the package, Ansible installation evidence, and approval record must be retained. | -| 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, persistent database and key volumes, and reboot/recreation proof. | **In progress.** The verified Ansible handoff now covers commit-bound installation, boot services, Nginx validation, listener restrictions and public HTTPS; privileged installation, firewall review, controlled reboot and supervised persistence evidence remain open. | -| 11 | Backups, monitoring, and recovery | Scheduled encrypted backups, verified off-host transfer, Zabbix monitoring, restore, and rollback rehearsal. | **In progress.** The Ansible operations handoff now installs validated systemd units, public-key-only backup support, transfer retry and restricted Zabbix status; secret provisioning, durable-log confirmation, manual backup, timed restore, rollback and independent evidence remain open. | +| 8 | Operational readiness | Host backup, monitoring, and recovery are provided by the existing central infrastructure. | **External.** GuestOps no longer carries a second application-specific operational stack. | +| 9 | Versioned releases | A deterministic version-named source archive and Ansible-controlled installation. | **Implemented; acceptance required.** Package the tested commit as `guestops_.tar.gz` and deploy it through the central playbook. | +| 10 | Debian deployment and persistence | Secure Debian/Compose deployment, HTTPS, external MongoDB, and persistent Data Protection keys. | **In progress.** The single Ansible playbook builds the API/worker, configures Nginx, restricts the listener and checks public readiness; sandbox deployment remains to be exercised. | +| 11 | Backups, monitoring, and recovery | Existing central MongoDB backup, Filebeat, host monitoring, and version-based rollback. | **External.** Rollback selects an older archive and reruns the same deployment playbook. | | 12 | Google mailbox acceptance | End-to-end Gmail consent, import, recovery, reviewed sending, reconciliation, and revocation evidence. | **In progress.** The runbook, record validator and release-bound Gate B bundle integration exist; the live synthetic-data exercise and independent review remain outstanding. | | 13 | Rezlynx/Guestline adapter | The real PMS provider adapter, mappings, idempotency, reconciliation, and ambiguous-write handling. | **Planned.** Provider contract and sandbox access are still required before implementation and acceptance. | | 14 | Payment links and status | The real payment-provider integration, webhooks, expiry, replay protection, and reconciliation. | **Planned.** The provider path and sandbox acceptance plan still need to be confirmed and completed. | | 15 | Knowledge, AI, and FAQ activation | Supervised knowledge-quality, AI-draft, FAQ test-mode, staff-training, and stop-control acceptance. | **Implemented; acceptance required.** Evaluation tooling and cross-release bundle validation exist; the supervised evaluation, staff training and independent approval remain outstanding. | | 16 | Identity, preferences, and privacy | Account/session controls, hotel preferences, privacy inventory, retention decisions, and audit review. | **Implemented; acceptance required.** Legal and operational decisions, identity checks, and independent review remain outstanding. | | 17 | Inbox usability and desktop parity | Stable pagination, protected unsaved drafts, hotel-timezone display, and desktop workflow parity. | **Implemented; acceptance required.** Automated checks pass; the supervised desktop exercise and independent approval remain outstanding. | -| 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** The integrated bundle validator now enforces one archive, release record, image set, environment, capacity report, pilot record and approval decision; live prerequisites, incident rehearsal, five-day pilot and named approvals remain open. | +| 18 | Pilot, capacity, and release approval | Capacity proof, incident exercise, five-business-day hotel pilot, findings closure, and Gate B approval. | **In progress.** Live prerequisites, incident rehearsal, five-day pilot and named approvals remain open. | | 19 | Account security and self-service | TOTP MFA, recovery codes, transactional email, granular roles, preferences, and security notifications. | **Implemented on the development branch; acceptance required.** Keep it separate until `0.2.1` is approved and tagged, then review, merge, and version it as `0.3.0`. | ## Status key @@ -58,17 +58,17 @@ This summary explains what each milestone delivers and where it currently stands | 5 | FAQ automation | B | Implemented / acceptance required | Draft and approval controls are promoted to local `main`. Keep live automation disabled until knowledge quality, thresholds, and rollback behaviour pass acceptance. | | 6 | Team onboarding and account recovery | B | Implemented / acceptance required | Invitation, password reset, and recovery flows are promoted to local `main`; verify deployed links, mail delivery, token expiry, and administrator recovery procedures. | | 7 | Google connection recovery | B | Implemented / acceptance required | Connection epochs, checkpoint recovery, and revocation handling are promoted to local `main`; complete real Google acceptance and worker-restart exercises. | -| 8 | Operational readiness tooling | A | Implemented / acceptance required | Backup, restore, release, and diagnostic tooling is promoted to local `main`; execute it on the actual Debian host and retain evidence. | -| 9 | Gitea and reproducible releases | A | In progress | `deploy/package_source.py` now packages only an explicit committed ref, verifies matched application versions, produces deterministic gzip output and a SHA-256 source record, and refuses overwrite. Hand that package to a version-selected Ansible playbook following the CMS/CMSFront pattern. Ansible must verify and install it, build commit-tagged images, record their immutable IDs, and deploy without a Gitea runner. Retain the package/install evidence off-host and resolve the release-line/tag identity before approval; the existing `0.1.0` tag remains attached to the foundation release. | -| 10 | Debian deployment and persistence | A | In progress | Compose uses separate named database and shared key volumes, private host configuration, loopback-only API access and bounded logs. The Ansible handoff verifies the source on both controller and host, enables Docker/Nginx at boot, installs and validates the reviewed proxy, rejects exposed API/MongoDB listeners, and requires trusted public HTTPS before selecting the release. Run it on the provisioned Debian host, review the firewall, complete the confirmation-gated persistence drill and controlled reboot, and retain independent evidence. | -| 11 | Backups, monitoring, and recovery | A | In progress | Encrypted backup and isolated restore tooling includes opt-in systemd scheduling, checksum-verified rsync transfer, restricted Zabbix status, guarded local retention and a release-bound acceptance validator. The Ansible operations playbook now verifies the selected release and private-file modes, imports only the recovery public key, validates and installs the units, enables transfer/monitoring, leaves backup scheduling off until manual acceptance, and fetches non-sensitive evidence. Provision secrets and durable logs, configure central alerts/retention, run the manual backup plus timed restore and rollback drills, and retain independent approval. | +| 8 | Operational readiness tooling | A | External | Use the existing host backup, Filebeat, monitoring and recovery processes instead of a second GuestOps-specific operational stack. | +| 9 | Gitea and versioned releases | A | In progress | `deploy/package_source.py` packages an explicit committed ref as immutable `guestops_.tar.gz`, verifies matching .NET/frontend versions, and refuses overwrite. Deploy it through the central version-selected Ansible playbook. | +| 10 | Debian deployment and persistence | A | In progress | Compose runs API and worker against the existing MongoDB, preserves a named Data Protection key volume, and binds the API to loopback. Exercise the single Ansible playbook on sandbox and confirm a second deployment preserves the keys. | +| 11 | Backups, monitoring, and recovery | A | External | The existing central MongoDB backup, Filebeat, host monitoring and archive retention remain authoritative. Rollback selects an earlier GuestOps version and reruns the deployment playbook. | | 12 | Google mailbox and reviewed-reply acceptance | B | In progress | The synthetic-data provider runbook, exact scenario set and restricted-record validator are implemented and wired into the Gate B bundle validator. Complete every scenario against the accepted Debian release and dedicated Google sandbox accounts, independently review the evidence, and retain the validated record. | | 13 | Rezlynx/Guestline adapter | C | Planned | Obtain the provider contract and sandbox, implement the adapter and mapping, and accept idempotency, stale-data, ambiguous-write, and reconciliation paths. | | 14 | Payment links and status | C | Planned | Select/confirm the payment-provider path, complete sandbox and webhook acceptance, and prove expiry, replay protection, reconciliation, and support recovery. | | 15 | Knowledge, AI, and FAQ activation | B | Implemented / acceptance required | Owners can run a bounded no-send batch evaluation, and a release-bound acceptance record enforces positive/negative coverage, zero FAQ errors, separate AI review, staff training, stop-control evidence and named monitoring/rollback owners. The integrated Gate B bundle rejects release or environment mismatches. Complete the supervised evaluation and retain independent approval. | | 16 | Identity, preferences, and privacy | B/C | Implemented / acceptance required | Login throttling trusts the client address only after one-hop processing by the configured proxy. A release-bound review now covers owner-controlled preferences, account/session controls, data inventory, retention/deletion/legal-hold ownership, provider decisions, audit evidence and known identity limitations. Complete the legal/operational decisions and independently approve the record. | | 17 | Inbox usability and desktop parity | B | Implemented / acceptance required | The inbox uses tenant-scoped stable cursor pagination in pages of 50 and protects unsaved drafts during route/history navigation, reload, conversation selection, filtering and search. Operational timestamps use the saved hotel timezone, and a release-bound desktop-parity acceptance record is implemented. The implementation and preview HTTP suite pass; run the supervised exercise against the approved release and retain independent approval. | -| 18 | Pilot, capacity, and release approval | B/C | In progress | The `0.2.1` Gate B candidate has bounded capacity, five-business-day pilot, incident and final-decision validators plus an integrated bundle check that binds every prerequisite to one release and verifies the retained capacity/pilot checksums. Retain the exact package and Ansible evidence, complete the live Gate A/B exercises, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | +| 18 | Pilot, capacity, and release approval | B/C | In progress | Complete the live Gate A/B exercises against the selected version, resolve or contain findings, and retain separate hotel-owner and technical approval. Gate C remains dependent on milestones 13 and 14. | ## Delivery sequence @@ -82,10 +82,10 @@ Milestones 13 (Guestline/Rezlynx) and 14 (payments) can progress as parallel pro - [ ] Merge the Action removal and release-process update to the intended default branch; that resulting commit becomes the new package candidate. - [x] Preserve the published `0.2.0` tag unchanged and use the clean `0.2.1` candidate line from `main`, excluding Milestone 19 application code. -- [ ] Create and checksum the approved source archive with `deploy/package_source.py`, add/select it in the GuestOps Ansible playbook, and retain the package and installation evidence. +- [ ] Create the immutable versioned archive with `deploy/package_source.py`, copy it into the Ansible files directory, and select its version in group vars. - [ ] Create and archive the immutable `0.2.1` approval tag only after Gate B approval. - [ ] Deploy to the target Debian environment with persistent MongoDB and data-protection keys. -- [ ] Run and record backup, restore, restart, monitoring, and rollback exercises. +- [ ] Confirm the existing central backup and monitoring processes include GuestOps, then test rollback by selecting the previous version. - [ ] Complete real Google mailbox acceptance without using production guest data. - [ ] Resolve or explicitly contain the milestone 16–17 pilot findings listed above. - [ ] Obtain the Guestline/Rezlynx interface contract and sandbox access. diff --git a/README.md b/README.md index c04c7fd..327fcd3 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ Open http://127.0.0.1:5173 and select **Open preview workspace**. Each preview l For MongoDB-backed operation, disable Preview and set `Mongo__ConnectionString`, `Mongo__Database`, and a persistent private `Keys__Path`. See [deployment](docs/deployment.md). -Operational tooling includes an owner-only Workspace health page and Linux deployment preflight, encrypted backup and isolated restore drill. See [operations and recovery](docs/operations.md) before the hotel pilot. +The owner-only Workspace health page remains available in the application. Host monitoring, MongoDB backup and disaster recovery are provided by the existing central infrastructure. -Release packaging is runner-free. `deploy/package_source.py` creates a deterministic archive and checksum record from one explicit committed Git ref for handoff to the Futuresens Ansible deployment. It never packages uncommitted working-tree files. +Release packaging is runner-free. `deploy/package_source.py` creates an immutable `guestops_.tar.gz` archive from one explicit committed Git ref for handoff to the Futuresens Ansible deployment. It never packages uncommitted working-tree files. ## Verification diff --git a/compose.yml b/compose.yml index a69ced6..998bea8 100644 --- a/compose.yml +++ b/compose.yml @@ -1,9 +1,9 @@ name: guestops x-app-env: &app-env - Mongo__ConnectionString: mongodb://guestops:${MONGO_APP_PASSWORD:?Set MONGO_APP_PASSWORD}@mongo:27017/guestops?authSource=guestops - Mongo__Database: guestops + Mongo__ConnectionString: ${MONGO_CONNECTION_STRING:?Set MONGO_CONNECTION_STRING} + Mongo__Database: ${MONGO_DATABASE:-guestops} Keys__Path: /var/lib/guestops/keys - PublicUrl: https://sandbox-guestops.futuresens.co.uk + PublicUrl: ${GUESTOPS_PUBLIC_ORIGIN:?Set GUESTOPS_PUBLIC_ORIGIN} Google__ClientId: ${GOOGLE_CLIENT_ID:-} Google__ClientSecret: ${GOOGLE_CLIENT_SECRET:-} AutoReply__EnableLive: ${AUTO_REPLY_ENABLE_LIVE:-false} @@ -22,7 +22,7 @@ services: environment: <<: *app-env ASPNETCORE_ENVIRONMENT: Production - AllowedHosts: sandbox-guestops.futuresens.co.uk;localhost;127.0.0.1 + AllowedHosts: ${GUESTOPS_ALLOWED_HOSTS:?Set GUESTOPS_ALLOWED_HOSTS} Proxy__KnownAddress: ${GUESTOPS_GATEWAY:-172.30.87.1} Ai__ApiKey: ${AI_API_KEY:-} Ai__Model: ${AI_MODEL:-} @@ -32,8 +32,9 @@ services: - app-keys:/var/lib/guestops/keys - ${PMS_CONFIG_FILE_HOST:-./deploy/pms.example.json}:/run/guestops/pms.json:ro - ${PAYMENTS_CONFIG_FILE_HOST:-./deploy/payments.example.json}:/run/guestops/payments.json:ro - depends_on: - mongo: { condition: service_healthy } + labels: + co.elastic.logs/enabled: "true" + service.name: guestops-api logging: *logging mem_limit: 1g security_opt: ["no-new-privileges:true"] @@ -43,31 +44,13 @@ services: restart: unless-stopped environment: *app-env volumes: ["app-keys:/var/lib/guestops/keys"] - depends_on: - mongo: { condition: service_healthy } + labels: + co.elastic.logs/enabled: "true" + service.name: guestops-worker logging: *logging mem_limit: 768m security_opt: ["no-new-privileges:true"] - mongo: - image: mongo:8.0 - restart: unless-stopped - environment: - MONGO_INITDB_ROOT_USERNAME: admin - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD:?Set MONGO_ROOT_PASSWORD} - MONGO_APP_PASSWORD: ${MONGO_APP_PASSWORD:?Set MONGO_APP_PASSWORD} - volumes: - - mongo-data:/data/db - - ./deploy/mongo-init.js:/docker-entrypoint-initdb.d/guestops.js:ro - command: ["mongod", "--wiredTigerCacheSizeGB", "1"] - healthcheck: - test: ["CMD-SHELL", "mongosh --quiet --eval 'db.adminCommand({ping:1}).ok' | grep 1"] - interval: 10s - timeout: 5s - retries: 12 - logging: *logging - mem_limit: 2500m volumes: - mongo-data: app-keys: networks: default: diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md deleted file mode 100644 index 7ff146b..0000000 --- a/deploy/ansible/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# GuestOps Ansible handoff - -This playbook is the application-owned deployment contract. Copy or import it into the private Futuresens Ansible repository and bind the `guestops` inventory group there. Do not commit inventory secrets, `.env` contents, provider credentials, hostnames other than the public service name, or restricted evidence paths to GuestOps. - -Required extra variables: - -- `guestops_release_version`: matched application version, for example `0.2.1`; -- `guestops_release_commit`: full 40-character packaged commit; -- `guestops_archive`: controller path to the generated `.tar.gz`; -- `guestops_archive_sha256`: approved lowercase SHA-256; -- `guestops_source_record`: controller path to the matching `.source.json`; -- `guestops_evidence_directory`: existing restricted controller directory for the fetched release record. - -The target must already have Docker Engine with Compose, Nginx, Python 3, `ss`, a trusted certificate under `/etc/letsencrypt/live/sandbox-guestops.futuresens.co.uk`, and `/etc/guestops/guestops.env` owned by root with mode `0600`. The private environment file supplies MongoDB passwords, image references, provider-file paths and disabled-by-default feature flags. Provision it through Ansible Vault or the existing Futuresens secret process, never through this repository. - -Example invocation from the central Ansible checkout: - -```sh -ansible-playbook guestops.yml \ - -l guestops_sandbox \ - -e guestops_release_version=0.2.1 \ - -e guestops_release_commit=FULL_40_CHARACTER_SHA \ - -e guestops_archive=/secure/releases/GuestOps-0.2.1-COMMIT.tar.gz \ - -e guestops_archive_sha256=LOWERCASE_SHA256 \ - -e guestops_source_record=/secure/releases/GuestOps-0.2.1-COMMIT.source.json \ - -e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 -``` - -The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, and waits for loopback readiness. It then enables Docker and Nginx at boot, validates and installs the reviewed proxy site, rejects public API or MongoDB listeners, and checks the public redirect and certificate-backed HTTPS readiness before changing the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success. - -The release playbook installs the reviewed Nginx site but does not provision DNS, certificates, firewall rules, durable logging, backup keys, central monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities. - -## Backup and monitoring operations - -After the exact release is deployed, run `guestops-operations.yml`. It installs the repository systemd units, verifies their definitions, imports only the approved public recovery key, enables transfer retry and monitoring, connects the aggregate status file to Zabbix, and retains non-sensitive installation evidence. It never creates a recovery private key or writes transfer credentials. - -Before running it, provision these private host files through Ansible Vault or the established secret process: - -- `/etc/guestops/backup.env` mode `0600`; -- `/etc/guestops/backup-transfer.env` mode `0600`; -- `/etc/guestops/backup-transfer.key` mode `0600`; -- `/etc/guestops/backup-known-hosts` mode `0644`. - -The recovery public key must be available on the controller. Its private key and protected recovery copy must remain off the Debian host. Confirm durable restricted logging separately and pass `guestops_durable_logs_configured=true` only after that review. The daily backup timer defaults to disabled; set `guestops_enable_backup_schedule=true` only after the manual backup and service-recovery exercise passes. - -```sh -ansible-playbook guestops-operations.yml \ - -l guestops_sandbox \ - -e guestops_release_commit=FULL_40_CHARACTER_SHA \ - -e guestops_backup_recipient=FULL_40_CHARACTER_GPG_FINGERPRINT \ - -e guestops_recovery_public_key=/secure/recovery/guestops-public.asc \ - -e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 \ - -e guestops_durable_logs_configured=true -``` - -Run the manual encrypted backup, transfer, isolated restore and rollback exercises separately under the approved maintenance procedure. After they pass, rerun with `guestops_enable_backup_schedule=true`. Configure Zabbix trigger thresholds and escalation recipients in the central Zabbix environment; the playbook exposes only the non-sensitive `guestops.status` item. diff --git a/deploy/ansible/guestops-operations.yml b/deploy/ansible/guestops-operations.yml deleted file mode 100644 index 38fce5d..0000000 --- a/deploy/ansible/guestops-operations.yml +++ /dev/null @@ -1,227 +0,0 @@ ---- -- name: Install GuestOps backup and monitoring operations - hosts: guestops - become: true - gather_facts: true - - vars: - guestops_root: /srv/guestops - guestops_current: "{{ guestops_root }}/current" - guestops_config_root: /etc/guestops - guestops_backup_root: /var/backups/guestops - guestops_gnupg_root: /var/lib/guestops-backup/gnupg - guestops_zabbix_service: zabbix-agent2 - guestops_zabbix_include_directory: /etc/zabbix/zabbix_agent2.d - guestops_enable_backup_schedule: false - guestops_durable_logs_configured: false - - pre_tasks: - - name: Validate required operational variables - ansible.builtin.assert: - that: - - guestops_release_commit is match('^[0-9a-f]{40}$') - - guestops_backup_recipient is match('^[A-Fa-f0-9]{40}$') - - guestops_recovery_public_key | length > 0 - - guestops_evidence_directory | length > 0 - - guestops_durable_logs_configured | bool - fail_msg: Release commit, recovery public key/fingerprint, evidence directory and reviewed durable logging are required. - - - name: Confirm selected release link - ansible.builtin.stat: - path: "{{ guestops_current }}" - follow: false - register: guestops_selected_release - - - name: Require deployed release before operations installation - ansible.builtin.assert: - that: - - guestops_selected_release.stat.exists - - guestops_selected_release.stat.islnk - - guestops_selected_release.stat.lnk_source == guestops_root + '/releases/' + guestops_release_commit - fail_msg: /srv/guestops/current must select the exact approved release commit. - - - name: Confirm selected release target exists - ansible.builtin.stat: - path: "{{ guestops_root }}/releases/{{ guestops_release_commit }}" - follow: true - register: guestops_release_target - - - name: Require selected release directory - ansible.builtin.assert: - that: - - guestops_release_target.stat.exists - - guestops_release_target.stat.isdir - fail_msg: The selected release target must be an installed directory. - - - name: Inspect pre-provisioned private operational files - ansible.builtin.stat: - path: "{{ item.path }}" - follow: false - loop: - - { path: /etc/guestops/backup.env, mode: "0600" } - - { path: /etc/guestops/backup-transfer.env, mode: "0600" } - - { path: /etc/guestops/backup-transfer.key, mode: "0600" } - - { path: /etc/guestops/backup-known-hosts, mode: "0644" } - register: guestops_operational_files - - - name: Require private backup and transfer configuration - ansible.builtin.assert: - that: - - item.stat.exists - - item.stat.isreg - - not item.stat.islnk - - item.stat.mode == item.item.mode - fail_msg: "{{ item.item.path }} must be a regular non-symlink file with mode {{ item.item.mode }}." - loop: "{{ guestops_operational_files.results }}" - no_log: true - - tasks: - - name: Create private backup directories - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: root - group: root - mode: "0700" - loop: - - "{{ guestops_backup_root }}" - - "{{ guestops_gnupg_root }}" - - - name: Stage recovery public key - ansible.builtin.copy: - src: "{{ guestops_recovery_public_key }}" - dest: /etc/guestops/recovery-public.asc - owner: root - group: root - mode: "0600" - - - name: Import recovery public key only - ansible.builtin.command: - argv: - - gpg - - --batch - - --homedir - - "{{ guestops_gnupg_root }}" - - --import - - /etc/guestops/recovery-public.asc - no_log: true - changed_when: true - - - name: Verify configured recovery fingerprint - ansible.builtin.command: - argv: - - gpg - - --batch - - --homedir - - "{{ guestops_gnupg_root }}" - - --list-keys - - "{{ guestops_backup_recipient }}" - changed_when: false - no_log: true - - - name: Remove staged recovery public key - ansible.builtin.file: - path: /etc/guestops/recovery-public.asc - state: absent - - - name: Install GuestOps systemd units - ansible.builtin.copy: - src: "{{ playbook_dir }}/../systemd/{{ item }}" - dest: "/etc/systemd/system/{{ item }}" - owner: root - group: root - mode: "0644" - loop: - - guestops-backup.service - - guestops-backup.timer - - guestops-backup-transfer.service - - guestops-backup-transfer.timer - - guestops-monitor-status.service - - guestops-monitor-status.timer - register: guestops_systemd_units - - - name: Verify installed systemd units - ansible.builtin.command: - argv: - - systemd-analyze - - verify - - /etc/systemd/system/guestops-backup.service - - /etc/systemd/system/guestops-backup.timer - - /etc/systemd/system/guestops-backup-transfer.service - - /etc/systemd/system/guestops-backup-transfer.timer - - /etc/systemd/system/guestops-monitor-status.service - - /etc/systemd/system/guestops-monitor-status.timer - changed_when: false - - - name: Reload systemd unit definitions - ansible.builtin.systemd_service: - daemon_reload: true - when: guestops_systemd_units.changed - - - name: Enable transfer retry and monitoring timers - ansible.builtin.systemd_service: - name: "{{ item }}" - enabled: true - state: started - loop: - - guestops-backup-transfer.timer - - guestops-monitor-status.timer - - - name: Select backup schedule state - ansible.builtin.systemd_service: - name: guestops-backup.timer - enabled: "{{ guestops_enable_backup_schedule | bool }}" - state: "{{ 'started' if guestops_enable_backup_schedule | bool else 'stopped' }}" - - - name: Require Zabbix include directory - ansible.builtin.stat: - path: "{{ guestops_zabbix_include_directory }}" - register: guestops_zabbix_directory - - - name: Confirm Zabbix agent configuration path - ansible.builtin.assert: - that: - - guestops_zabbix_directory.stat.exists - - guestops_zabbix_directory.stat.isdir - fail_msg: Override guestops_zabbix_include_directory for the installed Zabbix agent. - - - name: Install restricted Zabbix status item - ansible.builtin.copy: - src: "{{ playbook_dir }}/../systemd/zabbix-agent-guestops.conf.example" - dest: "{{ guestops_zabbix_include_directory }}/guestops.conf" - owner: root - group: root - mode: "0644" - register: guestops_zabbix_configuration - - - name: Restart Zabbix agent after configuration change - ansible.builtin.service: - name: "{{ guestops_zabbix_service }}" - enabled: true - state: restarted - when: guestops_zabbix_configuration.changed - - - name: Generate initial non-sensitive monitoring status - ansible.builtin.command: - argv: [systemctl, start, guestops-monitor-status.service] - changed_when: true - - - name: Read installed timer schedule - ansible.builtin.command: - argv: [systemctl, list-timers, --all, --no-pager, guestops-backup.timer, guestops-backup-transfer.timer, guestops-monitor-status.timer] - register: guestops_timer_status - changed_when: false - - - name: Retain monitoring status on the Ansible controller - ansible.builtin.fetch: - src: /run/guestops-monitor/status.json - dest: "{{ guestops_evidence_directory }}/monitor-status.json" - flat: true - - - name: Retain timer status on the Ansible controller - ansible.builtin.copy: - content: "{{ guestops_timer_status.stdout }}\n" - dest: "{{ guestops_evidence_directory }}/systemd-timers.txt" - mode: "0600" - delegate_to: localhost - become: false diff --git a/deploy/ansible/guestops.yml b/deploy/ansible/guestops.yml deleted file mode 100644 index 102712f..0000000 --- a/deploy/ansible/guestops.yml +++ /dev/null @@ -1,402 +0,0 @@ ---- -- name: Install a verified GuestOps source release - hosts: guestops - become: true - gather_facts: true - - vars: - guestops_root: /srv/guestops - guestops_config_root: /etc/guestops - guestops_public_hostname: sandbox-guestops.futuresens.co.uk - guestops_public_origin: "https://{{ guestops_public_hostname }}" - guestops_release_root: "{{ guestops_root }}/releases/{{ guestops_release_commit }}" - guestops_staging_archive: "/var/tmp/{{ guestops_archive | basename }}" - guestops_staging_record: "/var/tmp/{{ guestops_source_record | basename }}" - guestops_api_image: "guestops-api:{{ guestops_release_commit }}" - guestops_worker_image: "guestops-worker:{{ guestops_release_commit }}" - - pre_tasks: - - name: Validate required release variables - ansible.builtin.assert: - that: - - guestops_release_version is match('^[0-9]+\\.[0-9]+\\.[0-9]+$') - - guestops_release_commit is match('^[0-9a-f]{40}$') - - guestops_archive_sha256 is match('^[0-9a-f]{64}$') - - guestops_archive | length > 0 - - guestops_source_record | length > 0 - - guestops_evidence_directory | length > 0 - - guestops_public_hostname is match('^[A-Za-z0-9.-]+$') - - guestops_public_hostname == 'sandbox-guestops.futuresens.co.uk' - fail_msg: Release version, full commit, archive, SHA-256, source record and evidence directory are required. - - - name: Verify source package on the Ansible controller - ansible.builtin.command: - argv: - - python3 - - "{{ playbook_dir }}/../verify_source_package.py" - - --archive - - "{{ guestops_archive }}" - - --record - - "{{ guestops_source_record }}" - - --expected-commit - - "{{ guestops_release_commit }}" - - --expected-version - - "{{ guestops_release_version }}" - delegate_to: localhost - become: false - changed_when: false - - - name: Check private GuestOps environment exists - ansible.builtin.stat: - path: "{{ guestops_config_root }}/guestops.env" - register: guestops_environment - - - name: Require a private pre-provisioned environment file - ansible.builtin.assert: - that: - - guestops_environment.stat.exists - - guestops_environment.stat.isreg - - guestops_environment.stat.mode == '0600' - fail_msg: /etc/guestops/guestops.env must already exist with mode 0600. - - tasks: - - name: Require supported Debian host - ansible.builtin.assert: - that: - - ansible_facts.system == 'Linux' - - ansible_facts.distribution == 'Debian' - - ansible_facts.distribution_major_version | int >= 12 - - ansible_facts.processor_vcpus | default(0) | int >= 4 - - ansible_facts.memtotal_mb | default(0) | int >= 7300 - fail_msg: GuestOps requires Debian 12 or newer with at least 4 CPUs and 7.3 GiB RAM. - - - name: Enable Docker at boot - ansible.builtin.service: - name: docker - enabled: true - state: started - - - name: Enable Nginx at boot - ansible.builtin.service: - name: nginx - enabled: true - state: started - - - name: Create release and evidence directories - ansible.builtin.file: - path: "{{ item.path }}" - state: directory - owner: root - group: root - mode: "{{ item.mode }}" - loop: - - { path: "{{ guestops_root }}/releases", mode: "0755" } - - { path: "{{ guestops_release_root }}", mode: "0755" } - - { path: "{{ guestops_config_root }}/release-records", mode: "0700" } - - - name: Copy source archive to the host - ansible.builtin.copy: - src: "{{ guestops_archive }}" - dest: "{{ guestops_staging_archive }}" - owner: root - group: root - mode: "0600" - - - name: Copy source record to the host - ansible.builtin.copy: - src: "{{ guestops_source_record }}" - dest: "{{ guestops_staging_record }}" - owner: root - group: root - mode: "0600" - - - name: Calculate transferred archive checksum - ansible.builtin.stat: - path: "{{ guestops_staging_archive }}" - checksum_algorithm: sha256 - register: guestops_transferred_archive - - - name: Reject a changed transferred archive - ansible.builtin.assert: - that: - - guestops_transferred_archive.stat.checksum == guestops_archive_sha256 - fail_msg: Transferred GuestOps archive does not match the approved SHA-256. - - - name: Extract committed application files - ansible.builtin.unarchive: - src: "{{ guestops_staging_archive }}" - dest: "{{ guestops_release_root }}" - remote_src: true - extra_opts: - - --strip-components=1 - - - name: Verify transferred package and record on the target - ansible.builtin.command: - argv: - - python3 - - deploy/verify_source_package.py - - --archive - - "{{ guestops_staging_archive }}" - - --record - - "{{ guestops_staging_record }}" - - --expected-commit - - "{{ guestops_release_commit }}" - - --expected-version - - "{{ guestops_release_version }}" - - --output - - "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source-verification.json" - chdir: "{{ guestops_release_root }}" - changed_when: false - - - name: Retain approved source record on the target - ansible.builtin.copy: - src: "{{ guestops_staging_record }}" - dest: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source.json" - remote_src: true - owner: root - group: root - mode: "0600" - - - name: Select release images and safe pilot defaults in the private environment - ansible.builtin.lineinfile: - path: "{{ guestops_config_root }}/guestops.env" - regexp: "^{{ item.name }}=" - line: "{{ item.name }}={{ item.value }}" - create: false - mode: "0600" - loop: - - { name: GUESTOPS_API_IMAGE, value: "{{ guestops_api_image }}" } - - { name: GUESTOPS_WORKER_IMAGE, value: "{{ guestops_worker_image }}" } - - { name: GOOGLE_ENABLE_SENDING, value: "false" } - - { name: AUTO_REPLY_ENABLE_LIVE, value: "false" } - no_log: true - - - name: Copy private deployment environment into the release - ansible.builtin.copy: - src: "{{ guestops_config_root }}/guestops.env" - dest: "{{ guestops_release_root }}/.env" - remote_src: true - owner: root - group: root - mode: "0600" - - - name: Build API image from the verified source - ansible.builtin.command: - argv: - - docker - - build - - --target - - api - - --tag - - "{{ guestops_api_image }}" - - . - chdir: "{{ guestops_release_root }}" - register: guestops_api_build - changed_when: true - - - name: Build worker image from the verified source - ansible.builtin.command: - argv: - - docker - - build - - --target - - worker - - --tag - - "{{ guestops_worker_image }}" - - . - chdir: "{{ guestops_release_root }}" - register: guestops_worker_build - changed_when: true - - - name: Inspect API image identity - ansible.builtin.command: - argv: [docker, image, inspect, --format, "{{ '{{.Id}}' }}", "{{ guestops_api_image }}"] - register: guestops_api_identity - changed_when: false - - - name: Inspect worker image identity - ansible.builtin.command: - argv: [docker, image, inspect, --format, "{{ '{{.Id}}' }}", "{{ guestops_worker_image }}"] - register: guestops_worker_identity - changed_when: false - - - name: Create post-build release record - ansible.builtin.command: - argv: - - python3 - - deploy/release_record.py - - --artifact - - "{{ guestops_staging_archive }}" - - --commit - - "{{ guestops_release_commit }}" - - --api-image - - "{{ guestops_api_image }}" - - --api-id - - "{{ guestops_api_identity.stdout }}" - - --worker-image - - "{{ guestops_worker_image }}" - - --worker-id - - "{{ guestops_worker_identity.stdout }}" - - --output - - "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.json" - chdir: "{{ guestops_release_root }}" - - - name: Run offline deployment preflight - ansible.builtin.command: - argv: - - python3 - - deploy/ops.py - - preflight - - --offline - chdir: "{{ guestops_release_root }}" - changed_when: false - - - name: Validate Compose without exposing expanded configuration - ansible.builtin.command: - argv: - - docker - - compose - - --env-file - - "{{ guestops_config_root }}/guestops.env" - - config - - --quiet - chdir: "{{ guestops_release_root }}" - changed_when: false - - - name: Start the verified release without rebuilding - ansible.builtin.command: - argv: - - docker - - compose - - --env-file - - "{{ guestops_config_root }}/guestops.env" - - up - - --detach - - --no-build - chdir: "{{ guestops_release_root }}" - environment: - GUESTOPS_API_IMAGE: "{{ guestops_api_image }}" - GUESTOPS_WORKER_IMAGE: "{{ guestops_worker_image }}" - changed_when: true - - - name: Wait for loopback readiness - ansible.builtin.uri: - url: http://127.0.0.1:8080/health/ready - method: GET - status_code: 200 - return_content: false - register: guestops_readiness - retries: 30 - delay: 2 - until: guestops_readiness.status == 200 - - - name: Confirm TLS certificate files exist - ansible.builtin.stat: - path: "{{ item }}" - follow: true - loop: - - "/etc/letsencrypt/live/{{ guestops_public_hostname }}/fullchain.pem" - - "/etc/letsencrypt/live/{{ guestops_public_hostname }}/privkey.pem" - register: guestops_certificates - - - name: Require the pre-provisioned TLS certificate - ansible.builtin.assert: - that: - - guestops_certificates.results | map(attribute='stat.exists') | min - - guestops_certificates.results | map(attribute='stat.isreg') | min - fail_msg: A valid pre-provisioned Let's Encrypt certificate is required before enabling Nginx. - - - name: Install reviewed GuestOps Nginx site - ansible.builtin.copy: - src: "{{ playbook_dir }}/../nginx.conf" - dest: /etc/nginx/sites-available/guestops.conf - owner: root - group: root - mode: "0644" - register: guestops_nginx_site - - - name: Enable GuestOps Nginx site - ansible.builtin.file: - src: /etc/nginx/sites-available/guestops.conf - dest: /etc/nginx/sites-enabled/guestops.conf - state: link - register: guestops_nginx_enabled - - - name: Validate Nginx configuration - ansible.builtin.command: - argv: [nginx, -t] - changed_when: false - - - name: Reload Nginx after reviewed configuration change - ansible.builtin.service: - name: nginx - enabled: true - state: reloaded - when: guestops_nginx_site.changed or guestops_nginx_enabled.changed - - - name: Inspect host TCP listeners - ansible.builtin.command: - argv: [ss, -ltnH] - register: guestops_tcp_listeners - changed_when: false - - - name: Reject public API or MongoDB listeners - ansible.builtin.assert: - that: - - guestops_tcp_listeners.stdout_lines | select('search', ':8080(\\s|$)') | reject('search', '127\\.0\\.0\\.1:8080(\\s|$)') | list | length == 0 - - guestops_tcp_listeners.stdout_lines | select('search', ':27017(\\s|$)') | list | length == 0 - fail_msg: API port 8080 must be loopback-only and MongoDB must have no host listener. - - - name: Verify public HTTP redirects to HTTPS - ansible.builtin.uri: - url: "http://{{ guestops_public_hostname }}/health/ready" - method: GET - follow_redirects: none - status_code: [301, 302, 307, 308] - return_content: false - delegate_to: localhost - become: false - - - name: Verify public HTTPS readiness and certificate trust - ansible.builtin.uri: - url: "{{ guestops_public_origin }}/health/ready" - method: GET - status_code: 200 - return_content: true - validate_certs: true - register: guestops_public_readiness - retries: 10 - delay: 3 - until: - - guestops_public_readiness.status == 200 - - guestops_public_readiness.json is defined - - guestops_public_readiness.json.status == 'ready' - delegate_to: localhost - become: false - - - name: Select the current successful release - ansible.builtin.file: - src: "{{ guestops_release_root }}" - dest: "{{ guestops_root }}/current" - state: link - force: true - - - name: Retain release record on the Ansible controller - ansible.builtin.fetch: - src: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.json" - dest: "{{ guestops_evidence_directory }}/release-record.json" - flat: true - - - name: Retain target source verification on the Ansible controller - ansible.builtin.fetch: - src: "{{ guestops_config_root }}/release-records/{{ guestops_release_commit }}.source-verification.json" - dest: "{{ guestops_evidence_directory }}/source-verification.json" - flat: true - - - name: Remove transferred staging files after successful deployment - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - "{{ guestops_staging_archive }}" - - "{{ guestops_staging_record }}" diff --git a/deploy/backup-restore-acceptance.example.json b/deploy/backup-restore-acceptance.example.json deleted file mode 100644 index 5a51045..0000000 --- a/deploy/backup-restore-acceptance.example.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "schemaVersion": 1, - "system": "guestops-backup-recovery", - "evidenceId": "backup-restore", - "dataClassification": "synthetic-only", - "releaseVersion": "0.2.1", - "releaseCommit": "0000000000000000000000000000000000000000", - "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "environment": "https://sandbox-guestops.futuresens.co.uk", - "hostIdentifier": "guestops-sandbox-01", - "images": { - "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, - "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, - "mongo": {"reference": "mongo:8.0", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} - }, - "owners": { - "backup": "REPLACE", - "monitoring": "REPLACE", - "recoveryOperator": "REPLACE", - "technicalEscalation": "REPLACE", - "retention": "REPLACE", - "independentReviewer": "REPLACE" - }, - "startedAt": "2026-10-01T09:00:00Z", - "endedAt": "2026-10-01T13:00:00Z", - "reviewedAt": "2026-10-01T14:00:00Z", - "recoveryObjectives": { - "targetRpoHours": 24, - "observedRpoHours": 25, - "targetRtoMinutes": 240, - "observedRtoMinutes": 241 - }, - "retention": { - "localVerifiedDays": 7, - "offHostDaily": 35, - "offHostMonthly": 12, - "legalHoldOverrideTested": false - }, - "backup": { - "createdAt": "2026-10-01T08:00:00Z", - "sha256": "0000000000000000000000000000000000000000000000000000000000000000", - "transferredSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "privateKeyPresentOnHost": false - }, - "rollback": { - "previousReleaseCommit": "1111111111111111111111111111111111111111", - "previousArchiveSha256": "1111111111111111111111111111111111111111111111111111111111111111", - "previousImages": { - "api": {"reference": "guestops-api:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"}, - "worker": {"reference": "guestops-worker:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"} - }, - "persistentVolumesReplaced": false, - "restoredReleaseCommit": "0000000000000000000000000000000000000000", - "unresolvedOperations": 1, - "finalControls": { - "googleSending": "disabled", - "faqLiveMode": "disabled", - "pmsWrites": "disabled", - "paymentCreation": "disabled" - } - }, - "monitoringState": "not-configured", - "unresolvedCriticalFindings": 1, - "scenarios": [ - {"id": "alert-escalation", "status": "not-run", "evidence": []}, - {"id": "atomic-off-host-transfer", "status": "not-run", "evidence": []}, - {"id": "controlled-return-to-service", "status": "not-run", "evidence": []}, - {"id": "data-protection-recovery", "status": "not-run", "evidence": []}, - {"id": "database-inventory", "status": "not-run", "evidence": []}, - {"id": "durable-secret-free-logs", "status": "not-run", "evidence": []}, - {"id": "encrypted-manual-backup", "status": "not-run", "evidence": []}, - {"id": "image-rollback", "status": "not-run", "evidence": []}, - {"id": "isolated-restore", "status": "not-run", "evidence": []}, - {"id": "monitoring-coverage", "status": "not-run", "evidence": []}, - {"id": "off-host-checksum", "status": "not-run", "evidence": []}, - {"id": "production-service-recovery", "status": "not-run", "evidence": []}, - {"id": "retention-and-legal-hold", "status": "not-run", "evidence": []}, - {"id": "scheduled-backup", "status": "not-run", "evidence": []} - ] -} diff --git a/deploy/backup_restore_acceptance.py b/deploy/backup_restore_acceptance.py deleted file mode 100644 index ea74e55..0000000 --- a/deploy/backup_restore_acceptance.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -"""Validate a restricted GuestOps backup, monitoring and recovery record.""" - -from __future__ import annotations - -import argparse -import datetime as dt -import json -from pathlib import Path -import re -from urllib.parse import urlparse - - -VERSION = "0.2.1" -SCENARIOS = { - "encrypted-manual-backup", - "scheduled-backup", - "production-service-recovery", - "atomic-off-host-transfer", - "off-host-checksum", - "retention-and-legal-hold", - "monitoring-coverage", - "alert-escalation", - "durable-secret-free-logs", - "isolated-restore", - "data-protection-recovery", - "database-inventory", - "image-rollback", - "controlled-return-to-service", -} -SHA256 = re.compile(r"[0-9a-f]{64}") -GIT_SHA = re.compile(r"[0-9a-f]{40}") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def timestamp(value: object, field: str) -> dt.datetime: - require(isinstance(value, str) and value.endswith("Z"), - f"{field} must be a UTC timestamp ending in Z.") - try: - parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00") - except ValueError as error: - raise ValueError(f"{field} is not a valid timestamp.") from error - require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.") - return parsed - - -def safe_name(value: object, field: str) -> str: - name = str(value or "").strip() - require(2 <= len(name) <= 120 and "@" not in name and "/" not in name and "\\" not in name, - f"{field} requires a name without an email address or path.") - return name - - -def validate(record: object, expected_commit: str, expected_release_sha256: str) -> None: - require(isinstance(record, dict), "Acceptance record must be a JSON object.") - require(record.get("schemaVersion") == 1, "Unsupported backup/recovery schema.") - require(record.get("system") == "guestops-backup-recovery", - "system must be guestops-backup-recovery.") - require(record.get("evidenceId") == "backup-restore", - "evidenceId must be backup-restore.") - require(record.get("dataClassification") == "synthetic-only", - "Recovery acceptance must use synthetic data only.") - require(record.get("releaseVersion") == VERSION, f"releaseVersion must be {VERSION}.") - require(GIT_SHA.fullmatch(str(expected_commit)) is not None, - "Expected release commit must be a full lowercase Git SHA.") - require(SHA256.fullmatch(str(expected_release_sha256)) is not None, - "Expected release-record checksum must be a lowercase SHA-256 digest.") - require(record.get("releaseCommit") == expected_commit, - "releaseCommit does not match the approved candidate.") - require(record.get("releaseRecordSha256") == expected_release_sha256, - "releaseRecordSha256 does not match the retained release record.") - require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None, - "archiveSha256 must be a lowercase SHA-256 digest.") - - origin = urlparse(str(record.get("environment", ""))) - require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/") - and not origin.query and not origin.fragment and origin.username is None - and origin.password is None, - "environment must be an HTTPS origin without credentials, path, query or fragment.") - safe_name(record.get("hostIdentifier"), "hostIdentifier") - - images = record.get("images") - require(isinstance(images, dict) and set(images) == {"api", "worker", "mongo"}, - "images must contain exactly api, worker and mongo.") - for name in ("api", "worker"): - image = images[name] - require(isinstance(image, dict) and set(image) == {"reference", "id"}, - f"images.{name} must contain exactly reference and id.") - require(image["reference"] == f"guestops-{name}:{expected_commit}", - f"images.{name}.reference must use the full approved commit.") - require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, - f"images.{name}.id must be immutable.") - mongo = images["mongo"] - require(isinstance(mongo, dict) and set(mongo) == {"reference", "id"} - and mongo["reference"] == "mongo:8.0" - and re.fullmatch(r"sha256:[0-9a-f]{64}", str(mongo["id"])) is not None, - "images.mongo must identify the immutable mongo:8.0 image.") - - owners = record.get("owners") - owner_keys = {"backup", "monitoring", "recoveryOperator", "technicalEscalation", - "retention", "independentReviewer"} - require(isinstance(owners, dict) and set(owners) == owner_keys, - "owners must contain the exact operational and review roles.") - names = {key: safe_name(value, f"owners.{key}") for key, value in owners.items()} - reviewer = names["independentReviewer"].casefold() - require(reviewer not in {names[key].casefold() for key in owner_keys - {"independentReviewer"}}, - "independentReviewer must be different from every operational owner.") - - started = timestamp(record.get("startedAt"), "startedAt") - ended = timestamp(record.get("endedAt"), "endedAt") - reviewed = timestamp(record.get("reviewedAt"), "reviewedAt") - require(started <= ended <= reviewed, "Acceptance timestamps are out of order.") - - recovery = record.get("recoveryObjectives") - require(isinstance(recovery, dict) and set(recovery) == { - "targetRpoHours", "observedRpoHours", "targetRtoMinutes", "observedRtoMinutes", - }, "recoveryObjectives must contain exact target and observed RPO/RTO values.") - for field in recovery: - require(isinstance(recovery[field], (int, float)) and not isinstance(recovery[field], bool) - and recovery[field] >= 0, f"recoveryObjectives.{field} must be non-negative.") - require(recovery["targetRpoHours"] == 24 and recovery["observedRpoHours"] <= 24, - "Observed RPO must meet the approved 24-hour target.") - require(recovery["targetRtoMinutes"] == 240 and recovery["observedRtoMinutes"] <= 240, - "Observed RTO must meet the approved four-hour target.") - - retention = record.get("retention") - require(retention == { - "localVerifiedDays": 7, - "offHostDaily": 35, - "offHostMonthly": 12, - "legalHoldOverrideTested": True, - }, "Retention must record seven local days, 35 daily and 12 monthly off-host copies, and legal-hold testing.") - - backup = record.get("backup") - require(isinstance(backup, dict) and set(backup) == { - "createdAt", "sha256", "transferredSha256", "privateKeyPresentOnHost", - }, "backup must contain exact creation, checksum, transfer and private-key fields.") - created = timestamp(backup["createdAt"], "backup.createdAt") - require(created <= started, "The accepted backup must exist when the timed exercise starts.") - require(abs(recovery["observedRpoHours"] - (started - created).total_seconds() / 3600) < 0.01, - "Observed RPO must match the backup and exercise timestamps.") - require(abs(recovery["observedRtoMinutes"] - (ended - started).total_seconds() / 60) < 0.01, - "Observed RTO must match the exercise timestamps.") - require(SHA256.fullmatch(str(backup["sha256"])) is not None - and backup["transferredSha256"] == backup["sha256"], - "Local and transferred backup checksums must match.") - require(backup["privateKeyPresentOnHost"] is False, - "The recovery private key must not be present on the Debian host.") - - rollback = record.get("rollback") - require(isinstance(rollback, dict) and set(rollback) == { - "previousReleaseCommit", "previousArchiveSha256", "previousImages", - "persistentVolumesReplaced", "restoredReleaseCommit", "unresolvedOperations", "finalControls", - }, "rollback must contain the exact rehearsal and final-state fields.") - require(GIT_SHA.fullmatch(str(rollback["previousReleaseCommit"])) is not None - and rollback["previousReleaseCommit"] != expected_commit, - "Rollback must use a different retained previous release.") - require(SHA256.fullmatch(str(rollback["previousArchiveSha256"])) is not None, - "Rollback requires the previous archive checksum.") - previous_images = rollback["previousImages"] - require(isinstance(previous_images, dict) and set(previous_images) == {"api", "worker"}, - "Rollback requires exact previous API and worker images.") - for name in ("api", "worker"): - image = previous_images[name] - require(isinstance(image, dict) and set(image) == {"reference", "id"} - and image["reference"] == f"guestops-{name}:{rollback['previousReleaseCommit']}" - and re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, - f"rollback.previousImages.{name} must use the retained previous release identity.") - require(rollback["persistentVolumesReplaced"] is False, - "Image rollback must not replace persistent volumes.") - require(rollback["restoredReleaseCommit"] == expected_commit, - "The exercise must finish on the approved candidate.") - require(rollback["unresolvedOperations"] == 0, - "The exercise must finish without unresolved operations.") - require(rollback["finalControls"] == { - "googleSending": "disabled", "faqLiveMode": "disabled", - "pmsWrites": "disabled", "paymentCreation": "disabled", - }, "The exercise must finish with all unaccepted external writes disabled.") - - scenarios = record.get("scenarios") - require(isinstance(scenarios, list), "scenarios must be a list.") - ids = [item.get("id") for item in scenarios if isinstance(item, dict)] - require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == SCENARIOS, - "Acceptance record requires the exact backup/recovery scenario set.") - for item in scenarios: - scenario_id = item["id"] - require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.") - evidence = item.get("evidence") - require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all( - isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value - and "\\" not in value and not value.startswith("/") for value in evidence - ), f"Scenario {scenario_id} requires safe opaque evidence references.") - require(record.get("monitoringState") == "healthy", - "Monitoring must be healthy at acceptance completion.") - require(record.get("unresolvedCriticalFindings") == 0, - "Acceptance cannot pass with unresolved critical findings.") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("record", type=Path) - parser.add_argument("--expected-commit", required=True) - parser.add_argument("--expected-release-record-sha256", required=True) - args = parser.parse_args() - validate(json.loads(args.record.read_text(encoding="utf-8")), - args.expected_commit, args.expected_release_record_sha256) - print("Backup, monitoring and recovery acceptance record is structurally complete and passed. " - "This validates the record, not its restricted evidence.") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Backup/recovery acceptance record rejected: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/backup_transfer.py b/deploy/backup_transfer.py deleted file mode 100644 index ee445fc..0000000 --- a/deploy/backup_transfer.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env python3 -"""Atomically transfer encrypted GuestOps backups to restricted storage.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import re -import shlex -import shutil -import stat -import subprocess -import tempfile -import time -import uuid - - -BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg") -SAFE_HOST = re.compile(r"[A-Za-z0-9.-]{1,253}") -SAFE_USER = re.compile(r"[A-Za-z_][A-Za-z0-9_-]{0,31}") -SAFE_REMOTE_PATH = re.compile(r"/[A-Za-z0-9._/-]{1,500}") -SHA256 = re.compile(r"[0-9a-f]{64}") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise RuntimeError(message) - - -def digest(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def private_directory(value: str) -> Path: - requested = Path(value) - require(requested.is_absolute() and not requested.is_symlink(), - "BACKUP_DIRECTORY must be an absolute, non-symlink path.") - directory = requested.resolve() - require(directory.is_dir(), "BACKUP_DIRECTORY must exist.") - require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0, - "BACKUP_DIRECTORY must not be accessible to group or other users.") - return directory - - -def regular_file(value: str, field: str, *, private: bool) -> Path: - requested = Path(value) - require(requested.is_absolute() and not requested.is_symlink(), - f"{field} must be an absolute, non-symlink path.") - path = requested.resolve() - require(path.is_file(), f"{field} must be an existing regular file.") - if private: - require(stat.S_IMODE(path.stat().st_mode) & 0o077 == 0, - f"{field} must not be accessible to group or other users.") - return path - - -def configuration(environment: dict[str, str]) -> dict[str, object]: - directory = private_directory(environment.get("BACKUP_DIRECTORY", "")) - host = environment.get("BACKUP_REMOTE_HOST", "") - user = environment.get("BACKUP_REMOTE_USER", "") - remote = environment.get("BACKUP_REMOTE_DIRECTORY", "") - require(SAFE_HOST.fullmatch(host) is not None, "BACKUP_REMOTE_HOST is invalid.") - require(SAFE_USER.fullmatch(user) is not None, "BACKUP_REMOTE_USER is invalid.") - require(SAFE_REMOTE_PATH.fullmatch(remote) is not None and "//" not in remote - and "/../" not in remote + "/" and not remote.endswith("/.."), - "BACKUP_REMOTE_DIRECTORY must be a safe absolute path.") - identity = regular_file(environment.get("BACKUP_SSH_IDENTITY", ""), - "BACKUP_SSH_IDENTITY", private=True) - known_hosts = regular_file(environment.get("BACKUP_SSH_KNOWN_HOSTS", ""), - "BACKUP_SSH_KNOWN_HOSTS", private=False) - require(shutil.which("ssh") is not None and shutil.which("rsync") is not None, - "ssh and rsync are required.") - return { - "directory": directory, - "host": host, - "user": user, - "remote": remote.rstrip("/"), - "identity": identity, - "known_hosts": known_hosts, - } - - -def ssh_base(config: dict[str, object]) -> list[str]: - return [ - "ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", - "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", - "-o", f"UserKnownHostsFile={config['known_hosts']}", - "-i", str(config["identity"]), f"{config['user']}@{config['host']}", - ] - - -def run(args: list[str], *, environment: dict[str, str] | None = None) -> bytes: - result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, timeout=900, env=environment) - require(result.returncode == 0, - f"{Path(args[0]).name} step failed; review the restricted operator logs.") - return result.stdout - - -def remote_digest(config: dict[str, object], remote_path: str) -> str | None: - command = f"if test -f {remote_path} && test ! -L {remote_path}; then sha256sum -- {remote_path}; fi" - output = run([*ssh_base(config), command]).decode("utf-8", "strict").strip() - if not output: - return None - value = output.split()[0] - require(SHA256.fullmatch(value) is not None, "Remote checksum response was invalid.") - return value - - -def marker_path(backup: Path) -> Path: - return backup.with_name(backup.name + ".transferred.json") - - -def write_marker(backup: Path, checksum: str) -> None: - marker = marker_path(backup) - payload = json.dumps({ - "schemaVersion": 1, - "backup": backup.name, - "sha256": checksum, - "verifiedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - }, sort_keys=True) + "\n" - fd, temporary = tempfile.mkstemp(prefix=marker.name + ".", dir=marker.parent) - try: - os.fchmod(fd, 0o600) - with os.fdopen(fd, "w", encoding="utf-8") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, marker) - finally: - try: - os.unlink(temporary) - except FileNotFoundError: - pass - - -def transfer_one(config: dict[str, object], backup: Path) -> None: - require(backup.is_file() and not backup.is_symlink() - and BACKUP_NAME.fullmatch(backup.name) is not None, - "Refusing to transfer an unexpected backup path.") - checksum = digest(backup) - remote_final = f"{config['remote']}/{backup.name}" - existing = remote_digest(config, remote_final) - if existing is not None: - require(existing == checksum, "A remote backup with this name has a different checksum.") - write_marker(backup, checksum) - return - - remote_partial = f"{config['remote']}/.{backup.name}.partial-{uuid.uuid4().hex}" - rsh = shlex.join([ - "ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", - "-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15", - "-o", f"UserKnownHostsFile={config['known_hosts']}", - "-i", str(config["identity"]), - ]) - rsync_environment = os.environ.copy() - rsync_environment["RSYNC_RSH"] = rsh - try: - run(["rsync", "--archive", "--chmod=F600", "--protect-args", "--", - str(backup), f"{config['user']}@{config['host']}:{remote_partial}"], - environment=rsync_environment) - require(remote_digest(config, remote_partial) == checksum, - "Transferred backup checksum does not match the local file.") - command = (f"test ! -e {remote_final} && mv -T -- {remote_partial} {remote_final} " - f"&& chmod 600 -- {remote_final}") - run([*ssh_base(config), command]) - require(remote_digest(config, remote_final) == checksum, - "Final remote backup checksum does not match the local file.") - write_marker(backup, checksum) - except Exception: - # The name contains a fresh random suffix and is the only remote object this run may remove. - subprocess.run([*ssh_base(config), f"rm -f -- {remote_partial}"], stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) - raise - - -def transfer_pending(config: dict[str, object]) -> int: - directory = config["directory"] - backups = sorted(path for path in directory.iterdir() - if path.is_file() and not path.is_symlink() - and BACKUP_NAME.fullmatch(path.name) is not None) - for backup in backups: - marker = marker_path(backup) - if marker.is_file() and not marker.is_symlink(): - try: - recorded = json.loads(marker.read_text(encoding="utf-8")) - if recorded.get("sha256") == digest(backup): - continue - except (OSError, ValueError, json.JSONDecodeError): - pass - transfer_one(config, backup) - return len(backups) - - -def prune_verified(config: dict[str, object], retention_days: int, now: float | None = None) -> int: - require(1 <= retention_days <= 365, "Local retention must be between 1 and 365 days.") - threshold = (time.time() if now is None else now) - retention_days * 86400 - removed = 0 - for backup in config["directory"].iterdir(): - if not backup.is_file() or backup.is_symlink() or BACKUP_NAME.fullmatch(backup.name) is None: - continue - marker = marker_path(backup) - if backup.stat().st_mtime >= threshold or not marker.is_file() or marker.is_symlink(): - continue - try: - recorded = json.loads(marker.read_text(encoding="utf-8")) - except (OSError, ValueError, json.JSONDecodeError): - continue - if recorded.get("backup") != backup.name or recorded.get("sha256") != digest(backup): - continue - backup.unlink() - marker.unlink() - removed += 1 - return removed - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--prune-verified", action="store_true") - parser.add_argument("--retention-days", type=int, default=7) - args = parser.parse_args() - config = configuration(dict(os.environ)) - observed = transfer_pending(config) - removed = prune_verified(config, args.retention_days) if args.prune_verified else 0 - print(f"Backup transfer completed: {observed} encrypted backup(s) inspected; " - f"{removed} verified local backup(s) expired.") - - -if __name__ == "__main__": - try: - main() - except (OSError, RuntimeError, subprocess.SubprocessError, json.JSONDecodeError) as error: - print(f"Backup transfer failed: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/capacity_probe.py b/deploy/capacity_probe.py deleted file mode 100644 index 0ecce7d..0000000 --- a/deploy/capacity_probe.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -"""Run a bounded, read-only capacity probe against an approved GuestOps sandbox.""" - -from __future__ import annotations - -import argparse -import concurrent.futures -import datetime as dt -import http.cookiejar -import json -import os -from pathlib import Path -import re -import statistics -import time -import urllib.error -import urllib.parse -import urllib.request - - -PATHS = ("/health/ready", "/api/hotel", "/api/conversations/page") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def percentile(values: list[float], fraction: float) -> float: - ordered = sorted(values) - position = max(0, min(len(ordered) - 1, int(len(ordered) * fraction + 0.999999) - 1)) - return ordered[position] - - -def summarize(results: list[tuple[bool, float]], concurrency: int) -> dict[str, object]: - require(len(results) > 0, "At least one probe result is required.") - latencies = [latency for _, latency in results] - successes = sum(1 for success, _ in results if success) - return { - "concurrency": concurrency, - "requests": len(results), - "successes": successes, - "failures": len(results) - successes, - "errorRate": round((len(results) - successes) / len(results), 6), - "latencyMs": { - "median": round(statistics.median(latencies), 2), - "p95": round(percentile(latencies, 0.95), 2), - "maximum": round(max(latencies), 2), - }, - } - - -def login(origin: str, email: str, password: str) -> str: - jar = http.cookiejar.CookieJar() - opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(jar)) - with opener.open(origin + "/api/session", timeout=15) as response: - csrf = json.load(response)["csrfToken"] - body = json.dumps({"email": email, "password": password}).encode() - request = urllib.request.Request(origin + "/api/auth/login", data=body, method="POST", headers={"Content-Type": "application/json", "X-CSRF-TOKEN": csrf}) - with opener.open(request, timeout=15) as response: - require(response.status == 200, "Sandbox login failed.") - cookies = "; ".join(f"{cookie.name}={cookie.value}" for cookie in jar) - require("guestops.session=" in cookies, "Sandbox did not issue a GuestOps session cookie.") - return cookies - - -def request_once(origin: str, cookie: str, number: int) -> tuple[bool, float]: - path = PATHS[number % len(PATHS)] - request = urllib.request.Request(origin + path, headers={"Cookie": cookie, "Accept": "application/json"}) - started = time.perf_counter() - try: - with urllib.request.urlopen(request, timeout=20) as response: - success = response.status == 200 - response.read(1024) # Bound local processing; never retain response or guest content. - except (OSError, urllib.error.HTTPError): - success = False - return success, (time.perf_counter() - started) * 1000 - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--origin", required=True) - parser.add_argument("--requests", type=int, default=100) - parser.add_argument("--concurrency", type=int, default=5) - parser.add_argument("--release-commit", required=True) - parser.add_argument("--release-record-sha256", required=True) - parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--confirm-sandbox", action="store_true") - args = parser.parse_args() - - parsed = urllib.parse.urlparse(args.origin) - require(args.confirm_sandbox, "Use --confirm-sandbox after confirming the target and maintenance window.") - require(parsed.scheme == "https" and parsed.hostname and parsed.path in ("", "/") and not parsed.query and not parsed.fragment and not parsed.username, - "Origin must be an HTTPS origin without credentials, path, query or fragment.") - require(parsed.hostname != "localhost" and not parsed.hostname.startswith("127."), "Use the deployed HTTPS sandbox, not a development server.") - require(1 <= args.concurrency <= 20 and 1 <= args.requests <= 2000, "Probe bounds are 1–20 concurrent workers and 1–2000 requests.") - require(re.fullmatch(r"[0-9a-f]{40}", args.release_commit) is not None, "Use a full lowercase release commit SHA.") - require(re.fullmatch(r"[0-9a-f]{64}", args.release_record_sha256) is not None, "Use the release-record SHA-256.") - require(not args.output.exists() and args.output.parent.is_dir(), "Output must be a new file in an existing restricted directory.") - email = os.environ.get("CAPACITY_EMAIL", "") - password = os.environ.get("CAPACITY_PASSWORD", "") - require(email and password, "Set CAPACITY_EMAIL and CAPACITY_PASSWORD for a dedicated sandbox staff account.") - - cookie = login(args.origin.rstrip("/"), email, password) - with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: - results = list(pool.map(lambda number: request_once(args.origin.rstrip("/"), cookie, number), range(args.requests))) - report = { - "schemaVersion": 1, - "kind": "guestops-read-only-capacity", - "recordedAt": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), - "originHost": parsed.hostname, - "releaseCommit": args.release_commit, - "releaseRecordSha256": args.release_record_sha256, - "paths": list(PATHS), - **summarize(results, args.concurrency), - } - descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as output: - output.write(json.dumps(report, indent=2, sort_keys=True) + "\n") - print(f"Capacity probe completed: {report['successes']}/{report['requests']} successful; p95 {report['latencyMs']['p95']} ms. Review against the approved target before release.") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Capacity probe stopped: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/debian-host-acceptance.example.json b/deploy/debian-host-acceptance.example.json deleted file mode 100644 index 48e226f..0000000 --- a/deploy/debian-host-acceptance.example.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "schemaVersion": 1, - "system": "guestops-debian-host", - "evidenceId": "debian-host", - "releaseVersion": "0.2.1", - "releaseCommit": "0000000000000000000000000000000000000000", - "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "environment": "https://sandbox-guestops.futuresens.co.uk", - "hostIdentifier": "guestops-sandbox-01", - "images": { - "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, - "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} - }, - "hostFacts": { - "debianMajor": 12, - "cpuCores": 4, - "memoryBytes": 8140382208, - "freeDiskBytes": 14275686400, - "publicTcpPorts": [] - }, - "featureControls": { - "googleSending": "disabled", - "faqLiveMode": "disabled", - "pmsWrites": "disabled", - "paymentCreation": "disabled" - }, - "operator": "REPLACE", - "reviewedBy": "REPLACE", - "startedAt": "2026-09-30T09:00:00Z", - "endedAt": "2026-09-30T10:00:00Z", - "reviewedAt": "2026-09-30T11:00:00Z", - "unresolvedCriticalFindings": 1, - "scenarios": [ - {"id": "boot-services", "status": "not-run", "evidence": []}, - {"id": "controlled-reboot", "status": "not-run", "evidence": []}, - {"id": "durable-log-retrieval", "status": "not-run", "evidence": []}, - {"id": "host-baseline", "status": "not-run", "evidence": []}, - {"id": "https-and-redirect", "status": "not-run", "evidence": []}, - {"id": "network-exposure", "status": "not-run", "evidence": []}, - {"id": "proxy-trust", "status": "not-run", "evidence": []}, - {"id": "secret-free-logs", "status": "not-run", "evidence": []}, - {"id": "workspace-health", "status": "not-run", "evidence": []} - ] -} diff --git a/deploy/debian_acceptance.py b/deploy/debian_acceptance.py deleted file mode 100644 index 250d8fa..0000000 --- a/deploy/debian_acceptance.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Validate restricted GuestOps Debian-host and persistence acceptance records.""" - -from __future__ import annotations - -import argparse -import datetime as dt -import json -from pathlib import Path -import re -from urllib.parse import urlparse - - -VERSION = "0.2.1" -SYSTEMS = { - "guestops-debian-host": { - "evidenceId": "debian-host", - "scenarios": { - "host-baseline", - "network-exposure", - "https-and-redirect", - "proxy-trust", - "boot-services", - "controlled-reboot", - "workspace-health", - "durable-log-retrieval", - "secret-free-logs", - }, - }, - "guestops-persistence": { - "evidenceId": "persistence", - "scenarios": { - "separate-volume-layout", - "service-restart", - "container-recreation", - "database-inventory", - "data-protection-key", - "image-identity", - "post-reboot-persistence", - }, - }, -} -SHA256 = re.compile(r"[0-9a-f]{64}") -GIT_SHA = re.compile(r"[0-9a-f]{40}") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def timestamp(value: object, field: str) -> dt.datetime: - require(isinstance(value, str) and value.endswith("Z"), - f"{field} must be a UTC timestamp ending in Z.") - try: - parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00") - except ValueError as error: - raise ValueError(f"{field} is not a valid timestamp.") from error - require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.") - return parsed - - -def safe_text(value: object, field: str, minimum: int = 2, maximum: int = 160) -> str: - text = str(value or "").strip() - require(minimum <= len(text) <= maximum and "@" not in text and "\\" not in text, - f"{field} must be safe text without an email address or local path.") - return text - - -def validate_common(record: object, expected_commit: str, expected_release_sha256: str) -> str: - require(isinstance(record, dict), "Acceptance record must be a JSON object.") - require(record.get("schemaVersion") == 1, "Unsupported Debian acceptance schema.") - system = record.get("system") - require(system in SYSTEMS, "Unknown Debian acceptance record system.") - require(record.get("evidenceId") == SYSTEMS[system]["evidenceId"], - f"{system} has the wrong evidenceId.") - require(record.get("releaseVersion") == VERSION, - f"releaseVersion must be {VERSION}.") - require(GIT_SHA.fullmatch(str(expected_commit)) is not None, - "Expected release commit must be a full lowercase Git SHA.") - require(SHA256.fullmatch(str(expected_release_sha256)) is not None, - "Expected release-record checksum must be a lowercase SHA-256 digest.") - require(record.get("releaseCommit") == expected_commit, - "releaseCommit does not match the approved candidate.") - require(record.get("releaseRecordSha256") == expected_release_sha256, - "releaseRecordSha256 does not match the retained release record.") - require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None, - "archiveSha256 must be a lowercase SHA-256 digest.") - - origin = urlparse(str(record.get("environment", ""))) - require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/") - and not origin.query and not origin.fragment and origin.username is None - and origin.password is None, - "environment must be an HTTPS origin without credentials, path, query or fragment.") - host = safe_text(record.get("hostIdentifier"), "hostIdentifier") - - images = record.get("images") - require(isinstance(images, dict) and set(images) == {"api", "worker"}, - "images must contain exactly api and worker.") - for name in ("api", "worker"): - image = images[name] - require(isinstance(image, dict) and set(image) == {"reference", "id"}, - f"images.{name} must contain exactly reference and id.") - require(image["reference"] == f"guestops-{name}:{expected_commit}", - f"images.{name}.reference must use the full approved commit.") - require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None, - f"images.{name}.id must be an immutable image ID.") - - operator = safe_text(record.get("operator"), "operator") - reviewer = safe_text(record.get("reviewedBy"), "reviewedBy") - require(operator.casefold() != reviewer.casefold(), - "operator and reviewedBy must be different people.") - started = timestamp(record.get("startedAt"), "startedAt") - ended = timestamp(record.get("endedAt"), "endedAt") - reviewed = timestamp(record.get("reviewedAt"), "reviewedAt") - require(started <= ended <= reviewed, "Acceptance timestamps are out of order.") - - scenarios = record.get("scenarios") - require(isinstance(scenarios, list), "scenarios must be a list.") - ids = [item.get("id") for item in scenarios if isinstance(item, dict)] - required = SYSTEMS[system]["scenarios"] - require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == required, - f"{system} requires its exact acceptance scenario set.") - for item in scenarios: - scenario_id = item["id"] - require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.") - evidence = item.get("evidence") - require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all( - isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value - and "\\" not in value and not value.startswith("/") for value in evidence - ), f"Scenario {scenario_id} requires safe opaque evidence references.") - - require(record.get("unresolvedCriticalFindings") == 0, - "Acceptance cannot pass with unresolved critical findings.") - return host - - -def validate_host(record: object, expected_commit: str, expected_release_sha256: str) -> str: - host = validate_common(record, expected_commit, expected_release_sha256) - require(record.get("system") == "guestops-debian-host", - "First record must be guestops-debian-host.") - facts = record.get("hostFacts") - require(isinstance(facts, dict) and set(facts) == { - "debianMajor", "cpuCores", "memoryBytes", "freeDiskBytes", "publicTcpPorts", - }, "hostFacts must contain the exact reviewed host facts.") - require(isinstance(facts["debianMajor"], int) and facts["debianMajor"] >= 12, - "Debian 12 or newer is required.") - require(isinstance(facts["cpuCores"], int) and facts["cpuCores"] >= 4, - "At least four CPU cores are required.") - require(isinstance(facts["memoryBytes"], int) and facts["memoryBytes"] >= 7_500_000_000, - "At least 7.5 GB of memory is required.") - require(isinstance(facts["freeDiskBytes"], int) and facts["freeDiskBytes"] >= 8 * 1024**3, - "At least 8 GiB of free disk space is required.") - require(facts["publicTcpPorts"] == [80, 443], - "Only TCP ports 80 and 443 may be public.") - require(record.get("featureControls") == { - "googleSending": "disabled", - "faqLiveMode": "disabled", - "pmsWrites": "disabled", - "paymentCreation": "disabled", - }, "Unaccepted external writes and FAQ live mode must remain disabled.") - return host - - -def validate_persistence(record: object, expected_commit: str, expected_release_sha256: str) -> str: - host = validate_common(record, expected_commit, expected_release_sha256) - require(record.get("system") == "guestops-persistence", - "Second record must be guestops-persistence.") - require(record.get("drillCommand") == "python3 deploy/ops.py persistence-drill --confirm-restart", - "drillCommand must identify the confirmation-gated persistence drill.") - return host - - -def validate_pair(host_record: object, persistence_record: object, - expected_commit: str, expected_release_sha256: str) -> None: - host = validate_host(host_record, expected_commit, expected_release_sha256) - persistence_host = validate_persistence( - persistence_record, expected_commit, expected_release_sha256) - require(host == persistence_host, "Both records must identify the same host.") - for field in ("environment", "releaseVersion", "releaseCommit", "releaseRecordSha256", - "archiveSha256", "images"): - require(host_record.get(field) == persistence_record.get(field), - f"Both records must use the same {field}.") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("host_record", type=Path) - parser.add_argument("persistence_record", type=Path) - parser.add_argument("--expected-commit", required=True) - parser.add_argument("--expected-release-record-sha256", required=True) - args = parser.parse_args() - host_record = json.loads(args.host_record.read_text(encoding="utf-8")) - persistence_record = json.loads(args.persistence_record.read_text(encoding="utf-8")) - validate_pair(host_record, persistence_record, - args.expected_commit, args.expected_release_record_sha256) - print("Debian-host and persistence acceptance records are structurally complete and passed. " - "This validates the records, not their restricted evidence.") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Debian acceptance records rejected: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/gate_b_bundle.py b/deploy/gate_b_bundle.py deleted file mode 100644 index a6f26d0..0000000 --- a/deploy/gate_b_bundle.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -"""Validate the complete release-bound GuestOps Gate B acceptance bundle.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -import re -import sys -from urllib.parse import urlparse - - -DEPLOY = Path(__file__).resolve().parent -if str(DEPLOY) not in sys.path: - sys.path.insert(0, str(DEPLOY)) - -import automation_acceptance -import backup_restore_acceptance -import debian_acceptance -import desktop_acceptance -import google_acceptance -import identity_privacy_acceptance -import incident_exercise -import pilot_approval -import pilot_run -import verify_release -import verify_source_package - - -VERSION = "0.2.1" -SHA256 = re.compile(r"[0-9a-f]{64}") -RECORD_KEYS = { - "debian-host", "persistence", "backup-restore", "google-mailbox", - "automation", "identity-privacy", "inbox-usability", "capacity", - "incident-support", "pilot-findings", "pilot-approval", -} - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def digest(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def load_json(path: Path) -> object: - require(path.is_file() and not path.is_symlink(), f"Required bundle file is missing or is a symlink: {path.name}") - return json.loads(path.read_text(encoding="utf-8")) - - -def release_binding(record: object, name: str, commit: str, release_sha: str) -> None: - require(isinstance(record, dict), f"{name} must be a JSON object.") - require(record.get("releaseCommit") == commit, f"{name} uses a different releaseCommit.") - require(record.get("releaseRecordSha256") == release_sha, - f"{name} uses a different releaseRecordSha256.") - - -def environment_origin(record: object, name: str) -> str: - require(isinstance(record, dict), f"{name} must be a JSON object.") - value = str(record.get("environment", "")).rstrip("/") - parsed = urlparse(value) - require(parsed.scheme == "https" and parsed.hostname and parsed.path in ("", "/"), - f"{name} has no valid HTTPS environment.") - return value - - -def validate_capacity(report: object, commit: str, release_sha: str) -> None: - require(isinstance(report, dict), "capacity must be a JSON object.") - require(report.get("schemaVersion") == 1 and report.get("kind") == "guestops-read-only-capacity", - "capacity has an unsupported schema or kind.") - require(report.get("releaseCommit") == commit, "capacity uses a different releaseCommit.") - require(report.get("releaseRecordSha256") == release_sha, - "capacity uses a different releaseRecordSha256.") - require(report.get("paths") == ["/health/ready", "/api/hotel", "/api/conversations/page"], - "capacity must use the exact read-only path set.") - for field in ("concurrency", "requests", "successes", "failures"): - require(isinstance(report.get(field), int) and not isinstance(report.get(field), bool), - f"capacity.{field} must be an integer.") - require(1 <= report["concurrency"] <= 20 and report["requests"] > 0, - "capacity has invalid concurrency or request count.") - require(report["successes"] + report["failures"] == report["requests"], - "capacity success and failure counts do not match requests.") - expected_error = round(report["failures"] / report["requests"], 6) - require(report.get("errorRate") == expected_error, "capacity.errorRate does not match its counts.") - latency = report.get("latencyMs") - require(isinstance(latency, dict) and set(latency) == {"median", "p95", "maximum"} - and all(isinstance(value, (int, float)) and not isinstance(value, bool) and value >= 0 - for value in latency.values()), "capacity.latencyMs is invalid.") - require(latency["median"] <= latency["p95"] <= latency["maximum"], - "capacity latency percentiles are out of order.") - - -def validate_bundle( - source_archive: Path, - source_record_path: Path, - release_record_path: Path, - records: dict[str, tuple[Path, object]], - host_metrics_path: Path, -) -> dict[str, object]: - require(set(records) == RECORD_KEYS, "Gate B bundle requires the exact record set.") - release_record = load_json(release_record_path) - require(isinstance(release_record, dict), "release-record must be a JSON object.") - commit = str(release_record.get("commit", "")) - version = str(release_record.get("version", "")) - require(version == VERSION, f"Gate B bundle version must be {VERSION}.") - release_result = verify_release.validate(source_archive, release_record_path, commit, version) - source_result = verify_source_package.validate(source_archive, source_record_path, commit, version) - release_sha = str(release_result["releaseRecord"]["sha256"]) - require(source_result["artifact"]["sha256"] == release_result["archive"]["sha256"], - "Source and release records identify different archives.") - - values = {name: value for name, (_, value) in records.items()} - for name, record in values.items(): - release_binding(record, name, commit, release_sha) - - debian_acceptance.validate_pair(values["debian-host"], values["persistence"], commit, release_sha) - backup_restore_acceptance.validate(values["backup-restore"], commit, release_sha) - google_acceptance.validate(values["google-mailbox"]) - automation_acceptance.validate(values["automation"]) - identity_privacy_acceptance.validate(values["identity-privacy"]) - desktop_acceptance.validate(values["inbox-usability"]) - validate_capacity(values["capacity"], commit, release_sha) - incident_exercise.validate(values["incident-support"]) - pilot_run.validate(values["pilot-findings"]) - pilot_approval.validate(values["pilot-approval"]) - - environment_names = { - "debian-host", "persistence", "backup-restore", "google-mailbox", - "automation", "identity-privacy", "inbox-usability", "incident-support", - "pilot-findings", - } - origins = {environment_origin(values[name], name) for name in environment_names} - require(len(origins) == 1, "Gate B records use different environments.") - origin = next(iter(origins)) - require(values["capacity"].get("originHost") == urlparse(origin).hostname, - "capacity originHost does not match the accepted environment.") - - archive_sha = str(release_result["archive"]["sha256"]) - for name in ("debian-host", "persistence", "backup-restore"): - require(values[name].get("archiveSha256") == archive_sha, - f"{name} uses a different archiveSha256.") - approved_images = release_record["images"] - for name in ("debian-host", "persistence", "backup-restore"): - images = values[name].get("images", {}) - require(images.get("api") == approved_images["api"] and images.get("worker") == approved_images["worker"], - f"{name} uses different API or worker image identities.") - - approval_capacity = values["pilot-approval"]["capacity"] - capacity = values["capacity"] - require(approval_capacity["reportSha256"] == digest(records["capacity"][0]), - "Pilot approval capacity report checksum does not match the retained report.") - require(approval_capacity["hostMetricsSha256"] == digest(host_metrics_path), - "Pilot approval host metrics checksum does not match the retained file.") - require(approval_capacity["observedConcurrency"] == capacity["concurrency"], - "Pilot approval concurrency does not match the capacity report.") - require(approval_capacity["observedP95Ms"] == capacity["latencyMs"]["p95"], - "Pilot approval p95 does not match the capacity report.") - require(approval_capacity["observedErrorRate"] == capacity["errorRate"], - "Pilot approval error rate does not match the capacity report.") - require(values["pilot-approval"]["pilot"]["recordSha256"] == digest(records["pilot-findings"][0]), - "Pilot approval checksum does not match the retained pilot run record.") - - summary_records = {name: digest(path) for name, (path, _) in sorted(records.items())} - return { - "archiveSha256": archive_sha, - "environmentHost": urlparse(origin).hostname, - "releaseCommit": commit, - "releaseRecordSha256": release_sha, - "records": summary_records, - "schemaVersion": 1, - "sourceRecordSha256": source_result["sourceRecord"]["sha256"], - "validated": True, - "version": version, - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--source-archive", required=True, type=Path) - parser.add_argument("--source-record", required=True, type=Path) - parser.add_argument("--release-record", required=True, type=Path) - parser.add_argument("--host-metrics", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - for name in sorted(RECORD_KEYS): - parser.add_argument("--" + name, required=True, type=Path) - args = parser.parse_args() - try: - require(args.host_metrics.is_file() and not args.host_metrics.is_symlink(), - "Host metrics file is missing or is a symlink.") - require(not args.output.exists() and args.output.parent.is_dir(), - "Output must be a new file in an existing restricted directory.") - records = {} - for name in RECORD_KEYS: - path = getattr(args, name.replace("-", "_")) - records[name] = (path, load_json(path)) - summary = validate_bundle( - args.source_archive, - args.source_record, - args.release_record, - records, - args.host_metrics, - ) - descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as output: - output.write(json.dumps(summary, indent=2, sort_keys=True) + "\n") - print("Gate B bundle is structurally complete, consistently release-bound and approved. " - "This validates records and checksums, not the underlying restricted evidence.") - except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Gate B bundle rejected: {error}", file=sys.stderr) - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/deploy/google-acceptance.example.json b/deploy/google-acceptance.example.json index 8f55076..ca2f36f 100644 --- a/deploy/google-acceptance.example.json +++ b/deploy/google-acceptance.example.json @@ -2,7 +2,7 @@ "acceptedAt": "2026-01-01T00:00:00Z", "acceptedBy": "REPLACE WITH APPROVER", "endedAt": "2026-01-01T00:00:00Z", - "environment": "https://sandbox-guestops.futuresens.co.uk", + "environment": "https://guestops-sandbox.futuresens.co.uk", "mailboxLabel": "sandbox mailbox A", "operator": "REPLACE WITH OPERATOR", "releaseCommit": "0000000000000000000000000000000000000000", diff --git a/deploy/mongo-init.js b/deploy/mongo-init.js deleted file mode 100644 index 090549d..0000000 --- a/deploy/mongo-init.js +++ /dev/null @@ -1,2 +0,0 @@ -const guestops = db.getSiblingDB('guestops'); -guestops.createUser({user: 'guestops', pwd: process.env.MONGO_APP_PASSWORD, roles: [{role: 'readWrite', db: 'guestops'}]}); diff --git a/deploy/monitor_status.py b/deploy/monitor_status.py deleted file mode 100644 index 368d393..0000000 --- a/deploy/monitor_status.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python3 -"""Write non-sensitive GuestOps host status for a read-only monitoring agent.""" - -from __future__ import annotations - -import argparse -import datetime as dt -import json -import os -from pathlib import Path -import re -import shutil -import socket -import ssl -import stat -import subprocess -import tempfile -import urllib.request -from urllib.parse import urlparse - - -BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg") -MARKER_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg\.transferred\.json") -AUTH = ('const c=new Mongo("mongodb://127.0.0.1");' - 'c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,' - 'process.env.MONGO_INITDB_ROOT_PASSWORD);') -HEARTBEAT = ('const x=c.getDB("guestops").workerheartbeat.findOne({_id:"worker"});' - 'print(JSON.stringify(x&&x.At?x.At:null));') - - -def require(condition: bool, message: str) -> None: - if not condition: - raise RuntimeError(message) - - -def run(args: list[str], root: Path, timeout: int = 30) -> bytes: - result = subprocess.run(args, cwd=root, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - require(result.returncode == 0, f"{Path(args[0]).name} probe failed.") - return result.stdout - - -def utc_now() -> dt.datetime: - return dt.datetime.now(dt.timezone.utc) - - -def age_seconds(path: Path, pattern: re.Pattern[str], now: dt.datetime) -> int | None: - if not path.is_dir() or path.is_symlink(): - return None - times = [item.stat().st_mtime for item in path.iterdir() - if item.is_file() and not item.is_symlink() and pattern.fullmatch(item.name)] - return max(0, int(now.timestamp() - max(times))) if times else None - - -def https_status(origin: str, now: dt.datetime) -> dict[str, object]: - parsed = urlparse(origin) - require(parsed.scheme == "https" and parsed.hostname and parsed.port in (None, 443) - and parsed.path in ("", "/") and not parsed.query and not parsed.fragment - and parsed.username is None and parsed.password is None, - "GUESTOPS_ORIGIN must be an HTTPS origin without credentials or a path.") - context = ssl.create_default_context() - with socket.create_connection((parsed.hostname, 443), timeout=10) as connection: - with context.wrap_socket(connection, server_hostname=parsed.hostname) as secured: - certificate = secured.getpeercert() - expires = dt.datetime.strptime(certificate["notAfter"], "%b %d %H:%M:%S %Y %Z").replace( - tzinfo=dt.timezone.utc) - with urllib.request.urlopen(origin.rstrip("/") + "/health/ready", timeout=15, - context=context) as response: - ready = response.status == 200 and json.load(response) == {"status": "ready"} - return {"ready": ready, "certificateDaysRemaining": max(0, int((expires - now).total_seconds() // 86400))} - - -def parse_compose(value: bytes) -> dict[str, dict[str, str]]: - text = value.decode("utf-8", "strict").strip() - if not text: - return {} - try: - parsed = json.loads(text) - rows = parsed if isinstance(parsed, list) else [parsed] - except json.JSONDecodeError: - rows = [json.loads(line) for line in text.splitlines() if line.strip()] - result = {} - for row in rows: - if not isinstance(row, dict): - continue - service = str(row.get("Service", "")) - if service in {"api", "worker", "mongo"}: - result[service] = { - "state": str(row.get("State", "unknown")).lower(), - "health": str(row.get("Health", "none") or "none").lower(), - } - return result - - -def worker_heartbeat_age(root: Path, now: dt.datetime) -> int | None: - output = run(["docker", "compose", "exec", "-T", "mongo", "mongosh", "--quiet", - "--nodb", "--eval", AUTH + HEARTBEAT], root).decode("utf-8", "strict").strip() - value = json.loads(output) - if value is None: - return None - require(isinstance(value, str), "Worker heartbeat response was invalid.") - parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) - require(parsed.tzinfo is not None, "Worker heartbeat must contain a timezone.") - return max(0, int((now - parsed.astimezone(dt.timezone.utc)).total_seconds())) - - -def build_status(root: Path, backup_directory: Path, origin: str, - now: dt.datetime | None = None) -> tuple[dict[str, object], list[str]]: - moment = now or utc_now() - errors: list[str] = [] - try: - https = https_status(origin, moment) - except Exception: - https = {"ready": False, "certificateDaysRemaining": None} - errors.append("https-probe-failed") - try: - containers = parse_compose(run(["docker", "compose", "ps", "--format", "json"], root)) - if set(containers) != {"api", "worker", "mongo"}: - errors.append("container-set-incomplete") - except Exception: - containers = {} - errors.append("container-probe-failed") - try: - heartbeat_age = worker_heartbeat_age(root, moment) - if heartbeat_age is None: - errors.append("worker-heartbeat-missing") - except Exception: - heartbeat_age = None - errors.append("worker-heartbeat-probe-failed") - disk = shutil.disk_usage(root) - status = { - "schemaVersion": 1, - "generatedAt": moment.isoformat().replace("+00:00", "Z"), - "https": https, - "containers": containers, - "workerHeartbeatAgeSeconds": heartbeat_age, - "backupAgeSeconds": age_seconds(backup_directory, BACKUP_NAME, moment), - "verifiedTransferAgeSeconds": age_seconds(backup_directory, MARKER_NAME, moment), - "diskFreePercent": round(disk.free * 100 / disk.total, 2), - "persistentJournal": Path("/var/log/journal").is_dir(), - "errors": sorted(set(errors)), - } - return status, errors - - -def write_status(path: Path, status: dict[str, object]) -> None: - require(path.is_absolute() and not path.is_symlink(), - "Status output must be an absolute, non-symlink path.") - parent = path.parent.resolve() - require(parent.is_dir() and not path.parent.is_symlink(), "Status output directory must exist.") - fd, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=parent) - try: - os.fchmod(fd, 0o644) - with os.fdopen(fd, "w", encoding="utf-8") as stream: - json.dump(status, stream, sort_keys=True, separators=(",", ":")) - stream.write("\n") - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, path) - finally: - try: - os.unlink(temporary) - except FileNotFoundError: - pass - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) - parser.add_argument("--backup-directory", type=Path, default=Path("/var/backups/guestops")) - parser.add_argument("--origin", default="https://sandbox-guestops.futuresens.co.uk") - parser.add_argument("--output", type=Path, default=Path("/run/guestops-monitor/status.json")) - args = parser.parse_args() - root = args.root.resolve() - require(root.is_dir(), "GuestOps root must exist.") - backup_directory = args.backup_directory.resolve() - status, errors = build_status(root, backup_directory, args.origin) - write_status(args.output, status) - print("GuestOps monitoring status updated." if not errors - else "GuestOps monitoring status updated with failed probes.") - raise SystemExit(1 if errors else 0) - - -if __name__ == "__main__": - try: - main() - except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as error: - print(f"GuestOps monitoring probe failed: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/nginx.conf b/deploy/nginx.conf deleted file mode 100644 index b77b541..0000000 --- a/deploy/nginx.conf +++ /dev/null @@ -1,31 +0,0 @@ -# Install within the existing Nginx configuration. Obtain the certificate first. -server { - listen 80; - server_name sandbox-guestops.futuresens.co.uk; - location /.well-known/acme-challenge/ { root /var/www/letsencrypt; } - location / { return 301 https://$host$request_uri; } -} -server { - listen 443 ssl; - server_name sandbox-guestops.futuresens.co.uk; - ssl_certificate /etc/letsencrypt/live/sandbox-guestops.futuresens.co.uk/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/sandbox-guestops.futuresens.co.uk/privkey.pem; - ssl_protocols TLSv1.2 TLSv1.3; - add_header Strict-Transport-Security "max-age=31536000" always; - client_max_body_size 128k; - # OAuth authorization codes and state must not appear in access logs. - location = /api/integrations/google/callback { - access_log off; - proxy_pass http://127.0.0.1:8080; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto https; - } - location / { - proxy_pass http://127.0.0.1:8080; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_read_timeout 60s; - } -} diff --git a/deploy/ops.py b/deploy/ops.py deleted file mode 100644 index 602f2d7..0000000 --- a/deploy/ops.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 -"""GuestOps dedicated-Compose operations. Never prints credentials or provider data.""" -import argparse -import contextlib -import hashlib -import json -import os -from pathlib import Path -import re -import shutil -import stat -import subprocess -import sys -import tarfile -import tempfile -import time -import urllib.request -import uuid - -ROOT = Path(__file__).resolve().parents[1] -FILES = {"mongo.archive.gz", "keys.tar.gz", "configuration.json", "manifest.json"} -LIMIT = 8 * 1024**3 - - -def require(condition, message): - if not condition: - raise RuntimeError(message) - - -def run(args, *, output=None, input_file=None, timeout=900): - # Provider output may contain secrets; errors identify only the program. - result = subprocess.run(args, cwd=ROOT, stdin=input_file or subprocess.DEVNULL, - stdout=output or subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - require(result.returncode == 0, f"{args[0]} step failed. Check the private operator environment; no command output was logged.") - return result.stdout or b"" - - -def compose(*args, **kwargs): - return run(["docker", "compose", *args], **kwargs) - - -def digest(path): - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - -def image_id(image): - require(not image.startswith("-"), "Invalid image reference.") - return run(["docker", "image", "inspect", "--format", "{{.Id}}", image]).decode().strip() - - -def provider_configured(config, target): - section = "Pms" if target == "/run/guestops/pms.json" else "Payments" - # Only the exact shipped empty template is public; unknown settings fail closed. - return config not in ({}, {section: {"Hotels": {}}}) - - -def persistence_layout(config): - """Return the resolved named volumes required to survive recreation.""" - services = config["services"] - declared = config.get("volumes", {}) - - def volume_for(service, target): - matches = [v for v in services[service].get("volumes", []) if v["target"] == target] - require(len(matches) == 1 and matches[0]["type"] == "volume" and matches[0].get("source"), - f"{service} requires one named persistent volume at {target}.") - source = matches[0]["source"] - require(source in declared, f"{service} volume {source} must be declared at the top level.") - return declared[source].get("name") or source - - api_keys = volume_for("api", "/var/lib/guestops/keys") - worker_keys = volume_for("worker", "/var/lib/guestops/keys") - require(api_keys == worker_keys, "API and worker key volumes differ.") - mongo_data = volume_for("mongo", "/data/db") - require(api_keys != mongo_data, "Database and key data require separate named volumes.") - return {"keys": api_keys, "database": mongo_data} - - -AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);' -INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));' - - -def mongo(script): - return compose("exec", "-T", "mongo", "mongosh", "--quiet", "--nodb", "--eval", AUTH + script) - - -def configuration(): - config = json.loads(compose("config", "--format", "json")) - services = config["services"] - require(set(services) == {"api", "worker", "mongo"}, "This tool supports the dedicated three-service GuestOps Compose deployment only.") - require(not services["mongo"].get("ports"), "MongoDB must not have published ports.") - ports = services["api"].get("ports", []) - require(len(ports) == 1 and ports[0].get("host_ip") == "127.0.0.1" and int(ports[0]["target"]) == 8080, "API must publish only port 8080 on loopback.") - require(not services["worker"].get("ports"), "Worker must not publish ports.") - for name in ("api", "worker"): - env = services[name]["environment"] - require(env.get("Mongo__Database") == "guestops" and "@mongo:27017/guestops?" in env.get("Mongo__ConnectionString", ""), "Backup supports only the dedicated Compose guestops database.") - require(str(env.get("Preview", "false")).lower() != "true", "Production preview is forbidden.") - require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.") - require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.") - require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.") - persistence_layout(config) - return config - - -def wait_for(action, message, attempts=60): - for attempt in range(attempts): - try: - action() - return - except Exception: - if attempt == attempts - 1: - raise RuntimeError(message) - time.sleep(1) - - -def volume_identity(name): - require(not name.startswith("-"), "Invalid volume name.") - details = json.loads(run(["docker", "volume", "inspect", name])) - require(len(details) == 1 and details[0].get("Name") == name, "Named volume inspection failed.") - return {"name": name, "driver": details[0].get("Driver"), "scope": details[0].get("Scope")} - - -def readiness(): - with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=10) as response: - require(json.load(response) == {"status": "ready"}, "Loopback readiness failed.") - - -def persistence_drill(args): - require(args.confirm_restart, "Persistence drill requires --confirm-restart: all services will be restarted and application containers recreated.") - config = configuration() - layout = persistence_layout(config) - before_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()} - before_inventory = json.loads(mongo(INVENTORY)) - probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip() - require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.") - - # Restart the database first, then its clients, so recovery is exercised in a known order. - compose("restart", "-t", "150", "mongo") - wait_for(lambda: mongo('const r=c.getDB("admin").runCommand({ping:1});if(!r.ok)quit(1);'), "MongoDB did not recover after restart.") - compose("restart", "-t", "150", "api", "worker") - wait_for(readiness, "API readiness did not recover after restart.") - - # Recreate stateless containers as an image upgrade would, without rebuilding or changing data volumes. - compose("up", "-d", "--no-build", "--force-recreate", "api", "worker") - wait_for(readiness, "API readiness did not recover after container recreation.") - compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + probe, "api", "--verify-backup-probe") - - after_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()} - after_inventory = json.loads(mongo(INVENTORY)) - require(after_volumes == before_volumes, "A persistent volume identity changed during the drill.") - require(after_inventory["collections"] == before_inventory["collections"], "Database collection counts or indexes changed during the drill.") - print("Persistence drill passed: named volumes, database counts/indexes, data-protection keys and readiness survived restart and container recreation.") - - -def preflight(args): - config = configuration() - env_file = ROOT / ".env" - require(env_file.is_file() and not env_file.is_symlink(), "Create a private .env file before deployment.") - require(stat.S_IMODE(env_file.stat().st_mode) & 0o077 == 0, ".env must not be accessible to group or other users.") - require(shutil.disk_usage(ROOT).free >= 8 * 1024**3, "Keep at least 8 GiB free before deployment; allow extra room for backups.") - for name, service in config["services"].items(): - image_id(service["image"]) - for volume in service.get("volumes", []): - if volume["type"] == "bind": - require(Path(volume["source"]).exists(), f"A required {name} bind mount is missing.") - if volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"): - provider = Path(volume["source"]) - if provider_configured(json.loads(provider.read_text()), volume["target"]): - require(stat.S_IMODE(provider.stat().st_mode) & 0o077 == 0, "Configured provider files must not be accessible to group or other users.") - if not args.offline: - url = config["services"]["api"]["environment"]["PublicUrl"] - require(re.fullmatch(r"https://[A-Za-z0-9.-]+", url), "PublicUrl must be an HTTPS hostname without a path.") - with urllib.request.urlopen(url + "/health/ready", timeout=15) as response: - require(response.url == url + "/health/ready" and json.load(response) == {"status": "ready"}, "Public HTTPS readiness failed.") - print("Preflight passed: private database, loopback API, production settings, images, mounts and disk headroom" + ("; HTTPS not checked." if args.offline else "; public HTTPS and database readiness checked.")) - - -@contextlib.contextmanager -def maintenance_lock(): - import fcntl - with (ROOT / ".guestops-maintenance.lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - yield - - -def backup(args): - require(args.confirm_maintenance, "Backup requires --confirm-maintenance: API and workers will briefly stop.") - require(re.fullmatch(r"[A-Fa-f0-9]{40}", args.recipient), "Use a verified full 40-character GPG recipient fingerprint.") - run(["gpg", "--batch", "--list-keys", args.recipient]) - destination = Path(args.output).resolve() - require(not destination.exists(), "Backup output already exists; choose a new filename.") - require(destination.parent.is_dir(), "Create the private backup directory first.") - require(stat.S_IMODE(destination.parent.stat().st_mode) & 0o077 == 0, "Backup directory must be private (mode 700).") - config = configuration() - runtime = {} - for service in ("api", "worker", "mongo"): - cid = compose("ps", "--status", "running", "-q", service).decode().strip() - require(re.fullmatch(r"[a-f0-9]{12,64}", cid), f"Exactly one running {service} container is required.") - runtime[service] = json.loads(run(["docker", "inspect", cid]))[0] - require(shutil.disk_usage(destination.parent).free >= 3 * json.loads(mongo(INVENTORY))["bytes"] + 1024**3, "Insufficient free space for a consistent encrypted backup.") - partial = destination.with_name(destination.name + ".partial-" + uuid.uuid4().hex) - with maintenance_lock(), tempfile.TemporaryDirectory(prefix="guestops-backup-", dir=destination.parent) as folder: - folder = Path(folder) - stopped = False - ttl = None - try: - # Set before stopping so partial stop failures also attempt recovery. - stopped = True - compose("stop", "-t", "150", "api", "worker") - ttl = json.loads(mongo('print(JSON.stringify(c.getDB("admin").runCommand({getParameter:1,ttlMonitorEnabled:1}).ttlMonitorEnabled));')) - require(isinstance(ttl, bool), "Cannot determine MongoDB TTL monitor state.") - mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:false});if(!r.ok)quit(1);') - inventory = json.loads(mongo(INVENTORY)) - dump = 'set -eu; case "$MONGO_INITDB_ROOT_PASSWORD" in ""|*[!0-9a-fA-F]*) exit 1;; esac; umask 077; cfg=$(mktemp); trap \'rm -f "$cfg"\' EXIT; printf \'password: "%s"\\n\' "$MONGO_INITDB_ROOT_PASSWORD" > "$cfg"; mongodump --config "$cfg" --username "$MONGO_INITDB_ROOT_USERNAME" --authenticationDatabase admin --db guestops --archive --gzip' - with (folder / "mongo.archive.gz").open("wb") as output: - compose("exec", "-T", "mongo", "sh", "-c", dump, output=output) - probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip() - require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.") - with (folder / "keys.tar.gz").open("wb") as output: - compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "tar", "api", "-C", "/var/lib/guestops/keys", "-czf", "-", ".", output=output) - mounted = {} - for volume in config["services"]["api"].get("volumes", []): - if volume["type"] == "bind" and volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"): - mounted[volume["target"]] = Path(volume["source"]).read_text() - (folder / "configuration.json").write_text(json.dumps({"compose": config, "runtime": runtime, "providerFiles": mounted})) - manifest = {"format": 1, "createdAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "inventory": inventory, "probe": probe, "apiImageId": runtime["api"]["Image"], "mongoImageId": runtime["mongo"]["Image"], "sha256": {name: digest(folder / name) for name in FILES - {"manifest.json"}}} - (folder / "manifest.json").write_text(json.dumps(manifest)) - finally: - try: - if ttl is not None: - mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:' + str(ttl).lower() + '});if(!r.ok)quit(1);') - finally: - if stopped: - compose("start", "api", "worker") - try: - with tarfile.open(folder / "bundle.tar", "w") as archive: - for name in sorted(FILES): - archive.add(folder / name, arcname=name, recursive=False) - run(["gpg", "--batch", "--trust-model", "always", "--recipient", args.recipient, "--output", str(partial), "--encrypt", str(folder / "bundle.tar")]) - os.link(partial, destination) # Atomic publication, never overwrite an existing backup. - finally: - partial.unlink(missing_ok=True) - print("Encrypted backup complete. Copy it off-server and perform a restore drill.") - - -def scheduled_backup(args): - require(args.confirm_maintenance, "Scheduled backup requires --confirm-maintenance because API and workers will briefly stop.") - recipient = os.environ.get("BACKUP_RECIPIENT", "") - directory_value = os.environ.get("BACKUP_DIRECTORY", "") - require(re.fullmatch(r"[A-Fa-f0-9]{40}", recipient), "BACKUP_RECIPIENT must be a verified full GPG fingerprint.") - require(directory_value != "", "BACKUP_DIRECTORY must name the private off-checkout staging directory.") - requested_directory = Path(directory_value) - require(requested_directory.is_absolute() and not requested_directory.is_symlink(), "BACKUP_DIRECTORY must be an absolute, non-symlink path.") - directory = requested_directory.resolve() - require(directory.is_dir(), "BACKUP_DIRECTORY must be an existing real directory.") - require(ROOT not in directory.parents and directory != ROOT, "Scheduled backups must be staged outside the application checkout.") - require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0, "BACKUP_DIRECTORY must be private (mode 700).") - timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) - destination = directory / f"guestops-{timestamp}.tar.gpg" - backup(argparse.Namespace(confirm_maintenance=True, recipient=recipient, output=str(destination))) - print(f"Scheduled backup staged as {destination.name}. Off-host transfer and alert verification remain required.") - - -def unpack(bundle, folder): - with tarfile.open(bundle, "r:") as archive: - members = archive.getmembers() - require(len(members) == len(FILES) and {m.name for m in members} == FILES, "Unexpected backup members.") - require(all(m.isfile() and 0 <= m.size <= LIMIT for m in members) and sum(m.size for m in members) <= LIMIT, "Invalid or oversized backup members.") - for member in members: - with archive.extractfile(member) as source, (folder / member.name).open("xb") as output: - shutil.copyfileobj(source, output) - manifest = json.loads((folder / "manifest.json").read_text()) - require(manifest.get("format") == 1 and set(manifest.get("sha256", {})) == FILES - {"manifest.json"}, "Unsupported backup format.") - for name, expected in manifest["sha256"].items(): - require(digest(folder / name) == expected, "Backup checksum verification failed.") - return manifest - - -def restore_drill(args): - backup_file = Path(args.backup).resolve() - require(backup_file.is_file(), "Backup file is missing.") - with tempfile.TemporaryDirectory(prefix="guestops-restore-", dir=ROOT) as temp: - folder = Path(temp) - run(["gpg", "--batch", "--max-output", str(LIMIT), "--output", str(folder / "bundle.tar"), "--decrypt", str(backup_file)]) - require((folder / "bundle.tar").stat().st_size <= LIMIT, "Decrypted bundle exceeds the 8 GiB pilot limit.") - manifest = unpack(folder / "bundle.tar", folder) - require(image_id(args.api_image) == manifest["apiImageId"] and image_id(args.mongo_image) == manifest["mongoImageId"], "Load the exact trusted API and MongoDB images used for this backup.") - require(shutil.disk_usage(ROOT).free > 3 * manifest["inventory"]["bytes"] + 1024**3, "Insufficient restore drill space.") - keys = folder / "keys" - keys.mkdir(mode=0o700) - with tarfile.open(folder / "keys.tar.gz", "r:gz") as archive: - total = 0 - for member in archive: - if member.name in (".", "./") and member.isdir(): - continue - name = member.name.removeprefix("./") - total += member.size - require(member.isfile() and re.fullmatch(r"[A-Za-z0-9_-]+\.xml", name) and member.size < 1024**2 and total < 16 * 1024**2, "Invalid key archive.") - with archive.extractfile(member) as source, (keys / name).open("xb") as output: - shutil.copyfileobj(source, output) - run(["docker", "run", "--rm", "--pull", "never", "--network", "none", "--user", "0:0", "--read-only", "--mount", f"type=bind,src={keys},dst=/var/lib/guestops/keys,readonly", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + manifest["probe"], args.api_image, "--verify-backup-probe"]) - cid = run(["docker", "run", "-d", "--pull", "never", "--network", "none", "--memory", "1g", "--label", "guestops.restore-drill=true", args.mongo_image, "--bind_ip", "127.0.0.1", "--setParameter", "ttlMonitorEnabled=false"]).decode().strip() - require(re.fullmatch(r"[a-f0-9]{64}", cid), "Unexpected restore container identifier.") - try: - for attempt in range(30): - try: - run(["docker", "exec", cid, "mongosh", "--quiet", "--eval", "db.adminCommand({ping:1})"], timeout=10) - break - except RuntimeError: - if attempt == 29: - raise - time.sleep(1) - with (folder / "mongo.archive.gz").open("rb") as source: - run(["docker", "exec", "-i", cid, "mongorestore", "--archive", "--gzip", "--nsInclude", "guestops.*"], input_file=source) - restored = json.loads(run(["docker", "exec", cid, "mongosh", "--quiet", "--nodb", "--eval", 'const c=new Mongo("mongodb://127.0.0.1");' + INVENTORY])) - require(restored["collections"] == manifest["inventory"]["collections"], "Restored collection counts or indexes differ.") - finally: - run(["docker", "rm", "-f", "-v", cid]) # Only the exact container created above and its anonymous volumes. - print("Restore drill passed: collection counts, indexes and actual key decryption verified in isolated containers. Production was not restored or modified.") - - -def main(): - require(os.name == "posix", "Run deployment operations on Linux.") - os.umask(0o077) - parser = argparse.ArgumentParser(description=__doc__) - subs = parser.add_subparsers(dest="command", required=True) - check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true") - persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true") - save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true") - scheduled = subs.add_parser("scheduled-backup"); scheduled.add_argument("--confirm-maintenance", action="store_true") - drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0") - args = parser.parse_args() - {"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "scheduled-backup": scheduled_backup, "restore-drill": restore_drill}[args.command](args) - - -if __name__ == "__main__": - try: - main() - except Exception as error: - # Never include subprocess output, config values or parsed guest data. - print(str(error) if isinstance(error, RuntimeError) else "Operation failed (" + type(error).__name__ + "). No sensitive details were logged.", file=sys.stderr) - sys.exit(1) - diff --git a/deploy/package_source.py b/deploy/package_source.py index b3b48f3..b3d50f3 100644 --- a/deploy/package_source.py +++ b/deploy/package_source.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse import gzip -import hashlib import io import json from pathlib import Path @@ -44,11 +43,7 @@ def versions(repository: Path, commit: str) -> tuple[str, str]: return dotnet_version, web_version -def sha256_bytes(value: bytes) -> str: - return hashlib.sha256(value).hexdigest() - - -def build_package(repository: Path, ref: str, output_directory: Path) -> tuple[Path, Path, dict[str, object]]: +def build_package(repository: Path, ref: str, output_directory: Path) -> tuple[Path, dict[str, str]]: repository = repository.resolve() output_directory = output_directory.resolve() if not (repository / ".git").exists(): @@ -66,7 +61,7 @@ def build_package(repository: Path, ref: str, output_directory: Path) -> tuple[P if SEMVER.fullmatch(dotnet_version) is None: raise ValueError("The committed version must use MAJOR.MINOR.PATCH.") - prefix = f"GuestOps-{dotnet_version}/" + prefix = f"guestops_{dotnet_version}/" tar_bytes = bytes( git( repository, @@ -82,28 +77,13 @@ def build_package(repository: Path, ref: str, output_directory: Path) -> tuple[P stream.write(tar_bytes) archive_bytes = compressed.getvalue() - stem = f"GuestOps-{dotnet_version}-{commit[:12]}" - archive = output_directory / f"{stem}.tar.gz" - record_path = output_directory / f"{stem}.source.json" - if archive.exists() or record_path.exists(): - raise ValueError("The output package or source record already exists; releases are not overwritten.") + archive = output_directory / f"guestops_{dotnet_version}.tar.gz" + if archive.exists(): + raise ValueError("The output package already exists; published versions are not overwritten.") output_directory.mkdir(parents=True, exist_ok=True) archive.write_bytes(archive_bytes) - record: dict[str, object] = { - "artifact": { - "name": archive.name, - "sha256": sha256_bytes(archive_bytes), - "size": len(archive_bytes), - }, - "commit": commit, - "schemaVersion": 1, - "version": dotnet_version, - } - record_path.write_text( - json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - return archive, record_path, record + return archive, {"commit": commit, "version": dotnet_version} def main() -> None: @@ -113,11 +93,11 @@ def main() -> None: parser.add_argument("--output-directory", required=True, type=Path) args = parser.parse_args() try: - archive, record, details = build_package(args.repository, args.ref, args.output_directory) + archive, details = build_package(args.repository, args.ref, args.output_directory) except (OSError, ValueError, ET.ParseError, json.JSONDecodeError, subprocess.SubprocessError) as error: print(f"Source packaging failed: {error}", file=sys.stderr) raise SystemExit(1) - print(json.dumps({"archive": str(archive), "record": str(record), **details}, indent=2)) + print(json.dumps({"archive": str(archive), **details}, indent=2)) if __name__ == "__main__": diff --git a/deploy/persistence-acceptance.example.json b/deploy/persistence-acceptance.example.json deleted file mode 100644 index 8dc4ff7..0000000 --- a/deploy/persistence-acceptance.example.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "schemaVersion": 1, - "system": "guestops-persistence", - "evidenceId": "persistence", - "releaseVersion": "0.2.1", - "releaseCommit": "0000000000000000000000000000000000000000", - "releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000", - "environment": "https://sandbox-guestops.futuresens.co.uk", - "hostIdentifier": "guestops-sandbox-01", - "images": { - "api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}, - "worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"} - }, - "drillCommand": "python3 deploy/ops.py persistence-drill --confirm-restart", - "operator": "REPLACE", - "reviewedBy": "REPLACE", - "startedAt": "2026-09-30T09:00:00Z", - "endedAt": "2026-09-30T10:00:00Z", - "reviewedAt": "2026-09-30T11:00:00Z", - "unresolvedCriticalFindings": 1, - "scenarios": [ - {"id": "container-recreation", "status": "not-run", "evidence": []}, - {"id": "data-protection-key", "status": "not-run", "evidence": []}, - {"id": "database-inventory", "status": "not-run", "evidence": []}, - {"id": "image-identity", "status": "not-run", "evidence": []}, - {"id": "post-reboot-persistence", "status": "not-run", "evidence": []}, - {"id": "separate-volume-layout", "status": "not-run", "evidence": []}, - {"id": "service-restart", "status": "not-run", "evidence": []} - ] -} diff --git a/deploy/release_record.py b/deploy/release_record.py deleted file mode 100644 index 54ad727..0000000 --- a/deploy/release_record.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Bind a reviewed GuestOps source package to its installed image identities.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] - - -def project_versions() -> tuple[str, str]: - props = (ROOT / "Directory.Build.props").read_text(encoding="utf-8") - match = re.search(r"([^<]+)", props) - if match is None: - raise ValueError("Directory.Build.props does not contain a Version element") - package = json.loads((ROOT / "web" / "package.json").read_text(encoding="utf-8")) - return match.group(1), str(package["version"]) - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--artifact", required=True, type=Path) - parser.add_argument("--commit", required=True) - parser.add_argument("--api-image", required=True) - parser.add_argument("--api-id", required=True) - parser.add_argument("--worker-image", required=True) - parser.add_argument("--worker-id", required=True) - parser.add_argument("--output", required=True, type=Path) - args = parser.parse_args() - - commit = args.commit.lower() - if re.fullmatch(r"[0-9a-f]{40}", commit) is None: - parser.error("--commit must be a full 40-character Git SHA") - if not args.artifact.is_file(): - parser.error("--artifact must name an existing file") - - dotnet_version, web_version = project_versions() - if dotnet_version != web_version: - parser.error( - f"release versions differ: .NET={dotnet_version}, web={web_version}" - ) - - record = { - "artifact": { - "name": args.artifact.name, - "sha256": sha256(args.artifact), - "size": args.artifact.stat().st_size, - }, - "commit": commit, - "images": { - "api": {"id": args.api_id, "reference": args.api_image}, - "worker": {"id": args.worker_id, "reference": args.worker_image}, - }, - "schemaVersion": 1, - "version": dotnet_version, - } - args.output.write_text( - json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - - -if __name__ == "__main__": - main() diff --git a/deploy/systemd/backup-transfer.env.example b/deploy/systemd/backup-transfer.env.example deleted file mode 100644 index 35eb787..0000000 --- a/deploy/systemd/backup-transfer.env.example +++ /dev/null @@ -1,6 +0,0 @@ -BACKUP_DIRECTORY=/var/backups/guestops -BACKUP_REMOTE_HOST=restricted-store.example.invalid -BACKUP_REMOTE_USER=guestops_upload -BACKUP_REMOTE_DIRECTORY=/restricted/guestops/backups -BACKUP_SSH_IDENTITY=/etc/guestops/backup-transfer.key -BACKUP_SSH_KNOWN_HOSTS=/etc/guestops/backup-known-hosts diff --git a/deploy/systemd/backup.env.example b/deploy/systemd/backup.env.example deleted file mode 100644 index cabeb3b..0000000 --- a/deploy/systemd/backup.env.example +++ /dev/null @@ -1,2 +0,0 @@ -BACKUP_RECIPIENT=0123456789ABCDEF0123456789ABCDEF01234567 -BACKUP_DIRECTORY=/var/backups/guestops diff --git a/deploy/systemd/guestops-backup-transfer.service b/deploy/systemd/guestops-backup-transfer.service deleted file mode 100644 index 351b549..0000000 --- a/deploy/systemd/guestops-backup-transfer.service +++ /dev/null @@ -1,19 +0,0 @@ -[Unit] -Description=Transfer GuestOps encrypted backups to restricted storage -Wants=network-online.target -After=network-online.target guestops-backup.service -ConditionPathIsDirectory=/srv/guestops/current -ConditionPathIsDirectory=/var/backups/guestops - -[Service] -Type=oneshot -WorkingDirectory=/srv/guestops/current -EnvironmentFile=/etc/guestops/backup-transfer.env -UMask=0077 -ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/backup_transfer.py --prune-verified --retention-days 7 -NoNewPrivileges=true -PrivateTmp=true -ProtectHome=true -ProtectSystem=full -ReadWritePaths=/var/backups/guestops -TimeoutStartSec=30min diff --git a/deploy/systemd/guestops-backup-transfer.timer b/deploy/systemd/guestops-backup-transfer.timer deleted file mode 100644 index 1b7dd29..0000000 --- a/deploy/systemd/guestops-backup-transfer.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Retry GuestOps off-host backup transfer - -[Timer] -OnBootSec=10min -OnUnitActiveSec=15min -Persistent=true -Unit=guestops-backup-transfer.service - -[Install] -WantedBy=timers.target diff --git a/deploy/systemd/guestops-backup.service b/deploy/systemd/guestops-backup.service deleted file mode 100644 index 14bd15d..0000000 --- a/deploy/systemd/guestops-backup.service +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=GuestOps encrypted maintenance backup -Requires=docker.service -After=docker.service -ConditionPathIsDirectory=/srv/guestops/current -ConditionPathIsDirectory=/var/backups/guestops -ConditionPathIsDirectory=/var/lib/guestops-backup/gnupg - -[Service] -Type=oneshot -WorkingDirectory=/srv/guestops/current -EnvironmentFile=/etc/guestops/backup.env -Environment=GNUPGHOME=/var/lib/guestops-backup/gnupg -UMask=0077 -ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/ops.py scheduled-backup --confirm-maintenance -NoNewPrivileges=true -PrivateTmp=true -ProtectHome=true -ProtectSystem=full -ReadWritePaths=/var/backups/guestops /var/lib/guestops-backup/gnupg -TimeoutStartSec=45min diff --git a/deploy/systemd/guestops-backup.timer b/deploy/systemd/guestops-backup.timer deleted file mode 100644 index e9ebfdf..0000000 --- a/deploy/systemd/guestops-backup.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Run the GuestOps encrypted backup each day - -[Timer] -OnCalendar=*-*-* 02:17:00 UTC -RandomizedDelaySec=30min -Persistent=true -Unit=guestops-backup.service - -[Install] -WantedBy=timers.target diff --git a/deploy/systemd/guestops-monitor-status.service b/deploy/systemd/guestops-monitor-status.service deleted file mode 100644 index ce9ddeb..0000000 --- a/deploy/systemd/guestops-monitor-status.service +++ /dev/null @@ -1,21 +0,0 @@ -[Unit] -Description=Write non-sensitive GuestOps monitoring status -Requires=docker.service -After=docker.service network-online.target -ConditionPathIsDirectory=/srv/guestops/current -ConditionPathIsDirectory=/var/backups/guestops - -[Service] -Type=oneshot -WorkingDirectory=/srv/guestops/current -UMask=0022 -ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/monitor_status.py -NoNewPrivileges=true -PrivateTmp=true -ProtectHome=true -ProtectSystem=strict -RuntimeDirectory=guestops-monitor -RuntimeDirectoryMode=0755 -RuntimeDirectoryPreserve=yes -ReadWritePaths=/run/guestops-monitor -TimeoutStartSec=2min diff --git a/deploy/systemd/guestops-monitor-status.timer b/deploy/systemd/guestops-monitor-status.timer deleted file mode 100644 index 24e4b95..0000000 --- a/deploy/systemd/guestops-monitor-status.timer +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Refresh GuestOps monitoring status each minute - -[Timer] -OnBootSec=2min -OnUnitActiveSec=1min -Persistent=true -Unit=guestops-monitor-status.service - -[Install] -WantedBy=timers.target diff --git a/deploy/systemd/zabbix-agent-guestops.conf.example b/deploy/systemd/zabbix-agent-guestops.conf.example deleted file mode 100644 index 265e409..0000000 --- a/deploy/systemd/zabbix-agent-guestops.conf.example +++ /dev/null @@ -1,2 +0,0 @@ -# The privileged systemd probe owns Docker/database access. Zabbix reads only this aggregate JSON. -UserParameter=guestops.status,cat /run/guestops-monitor/status.json diff --git a/deploy/verify_release.py b/deploy/verify_release.py deleted file mode 100644 index b356680..0000000 --- a/deploy/verify_release.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -"""Verify a GuestOps source package and its immutable release record.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import re -import subprocess - - -SHA = re.compile(r"[0-9a-f]{40}") -DIGEST = re.compile(r"sha256:[0-9a-f]{64}") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def loaded_image_id(reference: str) -> str: - result = subprocess.run( - ["docker", "image", "inspect", "--format", "{{.Id}}", reference], - check=True, - capture_output=True, - text=True, - ) - return result.stdout.strip() - - -def validate( - archive: Path, - record_path: Path, - expected_commit: str, - expected_version: str, - verify_loaded_images: bool = False, -) -> dict[str, object]: - require(archive.is_file(), "Release archive does not exist.") - require(record_path.is_file(), "Release record does not exist.") - require(SHA.fullmatch(expected_commit) is not None, "Expected commit must be a full lowercase Git SHA.") - require(re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", expected_version) is not None, - "Expected version must use MAJOR.MINOR.PATCH.") - - record = json.loads(record_path.read_text(encoding="utf-8")) - require(isinstance(record, dict), "Release record must be a JSON object.") - require(record.get("schemaVersion") == 1, "Unsupported release-record schema.") - require(record.get("commit") == expected_commit, "Release-record commit does not match the approved candidate.") - require(record.get("version") == expected_version, "Release-record version does not match the approved version.") - - artifact = record.get("artifact") - require(isinstance(artifact, dict), "Release record has no artifact object.") - require(artifact.get("name") == archive.name, "Release archive filename does not match the record.") - require(artifact.get("size") == archive.stat().st_size, "Release archive size does not match the record.") - archive_sha = sha256(archive) - require(artifact.get("sha256") == archive_sha, "Release archive SHA-256 does not match the record.") - - images = record.get("images") - require(isinstance(images, dict) and set(images) == {"api", "worker"}, - "Release record must contain exactly API and worker images.") - expected_references = { - "api": f"guestops-api:{expected_commit}", - "worker": f"guestops-worker:{expected_commit}", - } - verified_images: dict[str, dict[str, str]] = {} - for name, reference in expected_references.items(): - image = images.get(name) - require(isinstance(image, dict), f"Release record has no {name} image object.") - require(image.get("reference") == reference, f"{name} image reference is not bound to the full candidate SHA.") - image_id = str(image.get("id", "")) - require(DIGEST.fullmatch(image_id) is not None, f"{name} image ID is not an immutable SHA-256 digest.") - if verify_loaded_images: - require(loaded_image_id(reference) == image_id, f"Loaded {name} image ID does not match the release record.") - verified_images[name] = {"reference": reference, "id": image_id} - - return { - "schemaVersion": 1, - "verified": True, - "commit": expected_commit, - "version": expected_version, - "archive": {"name": archive.name, "size": archive.stat().st_size, "sha256": archive_sha}, - "releaseRecord": {"name": record_path.name, "sha256": sha256(record_path)}, - "images": verified_images, - "loadedImageIdsVerified": verify_loaded_images, - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", required=True, type=Path) - parser.add_argument("--record", required=True, type=Path) - parser.add_argument("--commit", required=True) - parser.add_argument("--version", required=True) - parser.add_argument("--verify-loaded-images", action="store_true") - parser.add_argument("--output", type=Path, help="Optional path for the non-sensitive verification summary.") - args = parser.parse_args() - result = validate(args.archive, args.record, args.commit, args.version, args.verify_loaded_images) - rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" - if args.output: - args.output.write_text(rendered, encoding="utf-8") - print(rendered, end="") - - -if __name__ == "__main__": - try: - main() - except (OSError, ValueError, json.JSONDecodeError, subprocess.SubprocessError) as error: - print(f"Release verification failed: {error}", file=__import__("sys").stderr) - raise SystemExit(1) diff --git a/deploy/verify_source_package.py b/deploy/verify_source_package.py deleted file mode 100644 index 08b406f..0000000 --- a/deploy/verify_source_package.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -"""Verify a GuestOps source archive against its deterministic source record.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import re -import sys - - -FULL_SHA = re.compile(r"[0-9a-f]{40}") -SHA256 = re.compile(r"[0-9a-f]{64}") -SEMVER = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") - - -def require(condition: bool, message: str) -> None: - if not condition: - raise ValueError(message) - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def validate( - archive: Path, - record_path: Path, - expected_commit: str | None = None, - expected_version: str | None = None, -) -> dict[str, object]: - require(archive.is_file(), "Source archive does not exist.") - require(record_path.is_file(), "Source record does not exist.") - record = json.loads(record_path.read_text(encoding="utf-8")) - require(isinstance(record, dict), "Source record must be a JSON object.") - require(record.get("schemaVersion") == 1, "Unsupported source-record schema.") - - commit = record.get("commit") - version = record.get("version") - require(isinstance(commit, str) and FULL_SHA.fullmatch(commit) is not None, - "Source record commit must be a full lowercase Git SHA.") - require(isinstance(version, str) and SEMVER.fullmatch(version) is not None, - "Source record version must use MAJOR.MINOR.PATCH.") - if expected_commit is not None: - require(FULL_SHA.fullmatch(expected_commit) is not None, - "Expected commit must be a full lowercase Git SHA.") - require(commit == expected_commit, "Source-record commit does not match the selected release.") - if expected_version is not None: - require(SEMVER.fullmatch(expected_version) is not None, - "Expected version must use MAJOR.MINOR.PATCH.") - require(version == expected_version, "Source-record version does not match the selected release.") - - artifact = record.get("artifact") - require(isinstance(artifact, dict), "Source record has no artifact object.") - require(artifact.get("name") == archive.name, "Source archive filename does not match the record.") - require(artifact.get("size") == archive.stat().st_size, "Source archive size does not match the record.") - recorded_sha = artifact.get("sha256") - require(isinstance(recorded_sha, str) and SHA256.fullmatch(recorded_sha) is not None, - "Source archive record must contain a lowercase SHA-256 digest.") - actual_sha = sha256(archive) - require(recorded_sha == actual_sha, "Source archive SHA-256 does not match the record.") - - return { - "artifact": {"name": archive.name, "sha256": actual_sha, "size": archive.stat().st_size}, - "commit": commit, - "schemaVersion": 1, - "sourceRecord": {"name": record_path.name, "sha256": sha256(record_path)}, - "verified": True, - "version": version, - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--archive", required=True, type=Path) - parser.add_argument("--record", required=True, type=Path) - parser.add_argument("--expected-commit") - parser.add_argument("--expected-version") - parser.add_argument("--output", type=Path) - args = parser.parse_args() - try: - result = validate( - args.archive, - args.record, - args.expected_commit, - args.expected_version, - ) - except (OSError, ValueError, json.JSONDecodeError) as error: - print(f"Source package verification failed: {error}", file=sys.stderr) - raise SystemExit(1) - rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" - if args.output: - args.output.write_text(rendered, encoding="utf-8") - print(rendered, end="") - - -if __name__ == "__main__": - main() diff --git a/docs/accounts.md b/docs/accounts.md index 9dd2a3e..f309923 100644 --- a/docs/accounts.md +++ b/docs/accounts.md @@ -27,7 +27,7 @@ unset RECOVERY_EMAIL The command uses the configured database, generates a private link, prints it to the administrator's terminal and exits. Share it only with the verified owner. Do not paste it into tickets, chat logs, build logs or source control; avoid running this command in a recorded terminal. It expires in 30 minutes and can be consumed once. Running the command again invalidates the earlier link. Bootstrap remains a create-only command and cannot reset passwords. -`PublicUrl` must be the configured HTTPS origin, normally `https://sandbox-guestops.futuresens.co.uk`. Request Host headers never determine recovery-link origins. The Development-only preview uses `http://127.0.0.1:5173`. +`PublicUrl` must be the configured HTTPS origin, such as `https://guestops-sandbox.futuresens.co.uk`. Request Host headers never determine recovery-link origins. The Development-only preview uses `http://127.0.0.1:5173`. ## Storage and session behavior diff --git a/docs/deployment.md b/docs/deployment.md index 8bfa73e..34e999a 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,120 +1,65 @@ -# Debian sandbox deployment +# GuestOps deployment -Target: `https://sandbox-guestops.futuresens.co.uk`. These are reviewable deployment instructions; creating these files does not deploy or change the server. +GuestOps is deployed by the central Futuresens Ansible repository. Sandbox is +`https://guestops-sandbox.futuresens.co.uk` and production is +`https://guestops.futuresens.co.uk`. -## 1. Prepare the host +## Prepare a version -Check the existing Nginx, Docker, MongoDB and firewall configuration before installation. Preserve existing services. The provided Compose file creates a dedicated MongoDB instance for GuestOps with no host port; if MongoDB already exists on the host, use a reviewed private connection and application-scoped user instead of starting a conflicting instance. - -Install Docker Engine/Compose and Nginx using their official Debian instructions. Keep SSH access unchanged. Only HTTPS/HTTP for this hostname need public access; port 8080 is loopback-only and MongoDB has no published port. - -Install the versioned source package through the Futuresens Ansible repository. Do not clone GuestOps from the target server and do not place Gitea credentials on it. The playbook should extract the package into a dedicated application directory. Copy `.env.example` to `.env`, set permissions to 600, and fill two different MongoDB passwords generated with `openssl rand -hex 32`. Use hex values so they are safe in the MongoDB URI. Store real values only on the server or in its secret-management system. - -The MongoDB initialization script runs only on a new volume. Changing `.env` later does not rotate existing database users. Rotate those credentials through MongoDB administration and update application configuration together. - -## 2. Package and install with Ansible - -GuestOps follows the CMS/CMSFront deployment pattern: Gitea stores the application source, a specific committed version is compressed, and a version-selected Ansible playbook installs it. No Gitea Actions runner is required. - -On the trusted packaging machine, fetch the current remote refs, run the automated checks, select the full commit SHA, and create the archive from that committed tree rather than from a working directory: +The .NET version in `Directory.Build.props` and the frontend version in +`web/package.json` must match. Test the committed revision, then package it: ```sh -git fetch origin --prune --tags -git rev-parse origin/main +dotnet run --project tests/GuestOps.Tests +python3 -m unittest discover -s tests -p "test_*.py" +cd web && npm ci && npm run build && cd .. + python3 deploy/package_source.py \ - --ref FULL_40_CHARACTER_SHA \ - --output-directory /secure/release-staging + --ref FULL_COMMIT_OR_TAG \ + --output-directory ../ansible/playbooks/files/guestops ``` -The packaging command resolves the ref to a full commit, reads both application versions from that committed tree, requires them to match, creates deterministic gzip output, and writes a `.source.json` record containing the version, commit, filename, size and SHA-256. Existing package files are never overwritten. Independently confirm that the selected commit is the intended release line before transferring it. +For version `0.2.1`, this creates `guestops_0.2.1.tar.gz`. Published version +files are immutable; change the application version before making another +archive. -Store the archive and source record under a versioned GuestOps files directory in the private Ansible repository. The GuestOps playbook and environment variables should select that version, verify the archive against the source record, extract it into the application directory, preserve the private `.env` and provider configuration, and build images tagged with the full source commit: +## Configure Ansible -The application-owned handoff playbook and required variables are documented in [`deploy/ansible/README.md`](../deploy/ansible/README.md). Import or copy that playbook into the central private Ansible repository, bind its `guestops` inventory group, and keep inventory and secret values there. +The server group needs only the application switch, version and subdomain: -The playbook requires Debian 12 or newer with the approved CPU/RAM baseline, enables Docker and Nginx at boot, installs the reviewed site only after loopback readiness, runs `nginx -t`, rejects a host MongoDB listener or non-loopback API listener, and verifies the public redirect and trusted HTTPS readiness before selecting the release. Firewall reachability, certificate renewal, controlled reboot, durable logs and the persistence exercise still require the supervised checks below. +```yaml +hasguestops: true +guestopsversion: "0.2.1" +guestopssubdomain: "guestops-sandbox" +``` + +Production uses `guestopssubdomain: "guestops"`. Runtime credentials and the +MongoDB connection string belong in encrypted Ansible Vault data. The MongoDB +URI must use a least-privilege `guestops` database user and an address reachable +from Docker containers. + +## Deploy + +Deploy sandbox first: ```sh -docker build --target api -t guestops-api:FULL_40_CHARACTER_SHA . -docker build --target worker -t guestops-worker:FULL_40_CHARACTER_SHA . +ansible-playbook playbooks/guestops.yml -K --ask-vault-pass \ + --extra-vars "thehost=sandbox" ``` -Record the resulting immutable image IDs and bind them to the source archive with `deploy/release_record.py`. Retain the source archive, checksum, release record, its checksum, build output, commit and Ansible run result in the restricted release store. Set `GUESTOPS_API_IMAGE=guestops-api:FULL_40_CHARACTER_SHA` and `GUESTOPS_WORKER_IMAGE=guestops-worker:FULL_40_CHARACTER_SHA` in `.env`. Then the playbook runs: +After acceptance, deploy the same versioned archive to production: ```sh -docker compose config --quiet -docker compose up -d --no-build -curl --fail http://127.0.0.1:8080/health +ansible-playbook playbooks/guestops.yml -K --ask-vault-pass \ + --extra-vars "thehost=dbserver" ``` -Do not run `docker compose config` without `--quiet` in shared logs: expanded configuration contains secrets. Do not store `.env`, provider credentials, host inventory secrets, or restricted evidence in either application repository. Ansible must stop on a checksum, commit, version, build, or health-check mismatch. +The playbook stops the existing Compose project without deleting volumes, +replaces `/srv/guestops`, renders private configuration, builds and starts the +API and worker, configures Nginx, and verifies local and public readiness. Port +8080 remains loopback-only and the `guestops_app-keys` volume survives +redeployment. -## 3. Provision the first hotel owner - -The API has an administrative bootstrap command. It creates one hotel and one owner, refuses an existing email address, and does not reset passwords. Run it with environment variables passed from a private terminal; do not put the password directly in shell history or a repository file. - -```sh -read -r -p 'Owner email: ' BOOTSTRAP_EMAIL -read -r -p 'Hotel name: ' BOOTSTRAP_HOTEL -read -r -s -p 'Owner password (14+ characters): ' BOOTSTRAP_PASSWORD -printf '\n' -export BOOTSTRAP_EMAIL BOOTSTRAP_HOTEL BOOTSTRAP_PASSWORD -docker compose run --rm --no-deps -e BOOTSTRAP_EMAIL -e BOOTSTRAP_HOTEL -e BOOTSTRAP_PASSWORD api --bootstrap -unset BOOTSTRAP_EMAIL BOOTSTRAP_HOTEL BOOTSTRAP_PASSWORD -``` - -There is no development/demo account in production. Owners can issue staff invitation and assisted recovery links through Team; see [account setup](accounts.md). Passwords must be 14–128 characters. Public self-registration is not enabled. - -## 4. Configure HTTPS - -Verify the hostname's DNS resolves to this server. Obtain a valid certificate for it using your existing ACME/Certbot process. The Nginx example expects a Let's Encrypt certificate. For first issuance, configure the port-80 ACME location before enabling the port-443 block; do not point Nginx at missing certificate files. - -Merge `deploy/nginx.conf` into the existing host configuration, check with `nginx -t`, then reload Nginx. Confirm HTTPS serves the login page. API cookies are always Secure outside preview mode; logging in through plain HTTP is intentionally unsupported. - -Compose reserves the private bridge `172.30.87.0/24`, with gateway `172.30.87.1`. The API trusts the host gateway's `X-Forwarded-Proto` header so Nginx's HTTPS connections receive secure session and CSRF cookies. Check for an existing network using that range. If it conflicts, set both `GUESTOPS_SUBNET` and `GUESTOPS_GATEWAY` in `.env` to a free matching subnet and gateway. Do not replace this with unrestricted forwarded-header trust. - -The login rate limiter uses the client address forwarded by the explicitly configured host proxy (10 attempts per client address per minute). ASP.NET accepts one forwarding hop only from `Proxy__KnownAddress`; arbitrary client-supplied forwarding headers are not trusted. Keep the API port loopback-only and update the known address together with any reviewed Compose subnet change. - -## 5. Configure Google - -Create or select your Google Cloud project, enable Gmail API, and configure a Web application OAuth client. Register this exact redirect URI: - -`https://sandbox-guestops.futuresens.co.uk/api/integrations/google/callback` - -Configure `GOOGLE_CLIENT_ID` and `GOOGLE_CLIENT_SECRET` in the server's `.env` and recreate the API/worker services. Sign in as hotel owner, open Settings and connect a dedicated test mailbox. By default the only requested Gmail scope is `gmail.readonly`. Optional staff-approved sending adds `gmail.send` after server configuration and reconnection; see [AI drafts and reply delivery](replies.md). - -OAuth requests expire after ten minutes, are bound to the signed-in user and hotel, and can be consumed once. Refresh tokens are protected with ASP.NET Data Protection. API and worker share the private persistent key volume; back it up securely with the database. Losing it prevents existing mailbox tokens and sessions from being decrypted. Filesystem protection for the key volume is required; it is separate from MongoDB and must not be publicly served or committed. - -A multi-hotel production launch using Gmail restricted scopes requires planning for Google's verification/security-assessment requirements. This implementation does not bypass those requirements. See [Google restricted-scope verification](https://developers.google.com/identity/protocols/oauth2/production-readiness/restricted-scope-verification). - -## 6. Acceptance and rollback - -Verify separate hotels cannot read or edit each other's records; save and reload settings; restart services and confirm persistence; import test messages twice without duplicates; check the worker resumes a paginated import; confirm no mail is sent without explicit staff approval and that default-disabled sending remains blocked. Review the activity log and Google account used by the connection. - -Keep reviewed image IDs and release images for rollback and backups of both MongoDB and the key volume. Do not remove named volumes to fix application errors. The initial release has no automatic schema migration that destroys data. Use the [operational preflight, encrypted backup and isolated restore drill](operations.md), and establish retention and off-server copies before importing real guest data. `/health/ready` checks database reachability; the owner's Workspace health page also reports worker heartbeat and mailbox/reply exceptions. - -Before accepting the host, run the confirmation-gated persistence drill during an announced maintenance window: - -```sh -python3 deploy/ops.py preflight -python3 deploy/ops.py persistence-drill --confirm-restart -python3 deploy/ops.py preflight -``` - -The drill restarts MongoDB, API and worker, then force-recreates the stateless application containers using the already selected images. It verifies that the resolved database and key volumes keep the same identities, MongoDB collection counts and indexes remain unchanged, a value protected before restart can still be decrypted, and loopback readiness recovers. It does not alter provider feature flags, upgrade images, validate public TLS, or replace the separate backup/restore drill. - -Record the host, operator, start/end time, release record checksum, resolved image IDs, preflight output and drill result in the deployment acceptance record. Also verify Docker starts at boot and perform a controlled Debian reboot before Gate A approval. After reboot, run the online preflight and inspect the Workspace health page; do not infer worker health solely from API readiness. - -Keep the `debian-host` and `persistence` records in the restricted evidence store. Start from `deploy/debian-host-acceptance.example.json` and `deploy/persistence-acceptance.example.json`; the examples deliberately fail until every supervised scenario has passed. Bind both records to the expected release identifiers and validate them together: - -```sh -python3 deploy/debian_acceptance.py \ - /secure/acceptance/debian-host.json \ - /secure/acceptance/persistence.json \ - --expected-commit FULL_40_CHARACTER_SHA \ - --expected-release-record-sha256 RELEASE_RECORD_SHA256 -``` - -The validator requires matching archive and image identities, separate operator and reviewer names, the approved host capacity, only ports 80 and 443 recorded as publicly reachable, disabled unaccepted external writes, exact passed scenario sets and no unresolved critical findings. It validates record structure, not the restricted evidence itself. Retain the records, validator output and their checksums outside Git. - -The supplied Docker `json-file` logs are size-capped to protect the small pilot disk, but container recreation removes that container's local log history. Before host acceptance, route GuestOps and Nginx logs to the site's durable restricted logging system, or use a reviewed Docker logging override backed by persistent systemd journal storage. Prove that operators can retrieve pre-recreation logs without exposing request credentials or OAuth callback query strings. Central retention and alerting are completed under milestone 11. +To roll back, select an older retained `guestopsversion` in group vars and run +the same playbook. MongoDB backup, Filebeat, certificates, and host monitoring +are provided by the existing central infrastructure. diff --git a/docs/google-acceptance.md b/docs/google-acceptance.md index 91489c6..3d40f86 100644 --- a/docs/google-acceptance.md +++ b/docs/google-acceptance.md @@ -6,7 +6,7 @@ Keep screenshots, Gmail message source, provider console records and request dia ## Preparation and stop conditions -Record the release commit, release-record SHA-256, image IDs, environment, operator, approver and maintenance window. Use two dedicated Google test accounts so the different-account rejection can be exercised without a personal account. Send only clearly synthetic messages between controlled recipients. +Record the deployed GuestOps version, environment, operator, approver and maintenance window. Use two dedicated Google test accounts so the different-account rejection can be exercised without a personal account. Send only clearly synthetic messages between controlled recipients. Start with AI drafts, staff sending, FAQ live mode, PMS writes and payment creation disabled. Confirm the OAuth redirect URI exactly matches the HTTPS sandbox. Stop immediately if a mailbox appears under the wrong hotel, a recipient differs from the review screen, an unapproved message is sent, a duplicate is observed, provider output exposes credentials, or an uncertain outcome is about to be blindly retried. Preserve evidence and follow the incident process. diff --git a/docs/incident-exercise.md b/docs/incident-exercise.md index 9ddc353..3e6010a 100644 --- a/docs/incident-exercise.md +++ b/docs/incident-exercise.md @@ -17,7 +17,7 @@ For Gate C, also exercise `pms-ambiguous-write` and `payment-ambiguous-create`. ## Record and validation -Copy `deploy/incident-exercise.example.json` into the restricted acceptance store. Bind it to the same full release commit and release-record SHA-256 used by the deployment and pilot decision. Replace the example timestamps, people, targets and evidence references. Change a scenario to `pass` only after its evidence has been independently reviewed. The example is intentionally invalid because its scenarios are `not-run`. +Copy `deploy/incident-exercise.example.json` into the restricted acceptance store and record the deployed GuestOps version and environment. Replace the example timestamps, people, targets and evidence references. Change a scenario to `pass` only after its evidence has been independently reviewed. The example is intentionally invalid because its scenarios are `not-run`. For a Gate C exercise, set `targetGate` to `C` and add the PMS and payment scenario records. Validate the completed record with: diff --git a/docs/migration.md b/docs/migration.md index a0aa827..b06ad95 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -42,11 +42,11 @@ Implemented owner-only disconnect/reconnect and import restart controls, synchro ## Milestone 8: operational readiness -Implemented owner-only workspace health, database readiness, worker heartbeat, Linux deployment preflight, maintenance-window encrypted database/key/configuration backup and an isolated restore drill. The drill verifies collection counts, indexes and actual key decryption. Scheduling, off-server copies and production disaster cutover remain operator tasks; the Debian server rehearsal is still required. See [operations and recovery](operations.md). +The application provides owner-only workspace health, database readiness and worker heartbeat information. Docker, Filebeat, MongoDB backup, host monitoring and disaster recovery are owned by the existing central infrastructure rather than duplicated inside GuestOps. ## Remaining milestones -1. Run dedicated Google test-mailbox acceptance, add full thread aggregation and history repair, and rehearse server backup/restore before the first hotel pilot. +1. Run dedicated Google test-mailbox acceptance, add full thread aggregation and history repair, and confirm the central server backup/restore process before the first hotel pilot. 2. Add verified transactional email for invitations and recovery notifications, MFA, granular roles and user preferences. 3. Run a representative live AI draft evaluation, improve retrieval and evidence presentation, and approve the selected provider data-processing arrangements. 4. Extend the implemented durable reply queue with operator recovery tooling and broaden the controlled FAQ rules only after live acceptance. Preserve the rule that uncertain sends are never blindly replayed. diff --git a/docs/operations.md b/docs/operations.md deleted file mode 100644 index 04949aa..0000000 --- a/docs/operations.md +++ /dev/null @@ -1,208 +0,0 @@ -# Operations and recovery - -The owner-only **Workspace health** page reports database reachability, the worker's last heartbeat, mailbox recovery counts and reply exceptions. A heartbeat older than three minutes is marked stale. It proves that the worker process recently reached MongoDB, not that every provider or job succeeded. Reply totals cover at most the latest 500 conversations in this hotel. The page does not verify backups. `/health/ready` returns only readiness status and HTTP 503 when the database cannot be reached. - -## Release evidence and rollback - -Create the release source archive from an exact committed Gitea tree with `deploy/package_source.py`, then verify its generated source record before Ansible installs it. The tool packages only the selected commit, produces deterministic gzip output, rejects version disagreement, and refuses to overwrite an existing release. Ansible builds the API and worker images under the full Git commit SHA. The post-build `release-record.json` binds the source-archive checksum, application version, commit, image references and immutable Docker image IDs. Retain the archive, source record, release record and Ansible result together in restricted off-host release storage. - -```sh -git fetch origin --prune --tags -python3 deploy/package_source.py \ - --ref FULL_40_CHARACTER_SHA \ - --output-directory /secure/release-staging -``` - -Before deployment, verify the archive against its source record: - -```sh -python3 - <<'PY' -import hashlib, json, pathlib -r = json.load(open('GuestOps-0.2.1-COMMIT.source.json', encoding='utf-8')) -p = pathlib.Path(r['artifact']['name']) -assert hashlib.sha256(p.read_bytes()).hexdigest() == r['artifact']['sha256'] -print(r['commit'], r['version'], r['artifact']) -PY -``` - -After Ansible has built the commit-tagged images, the repository verifier performs the same checks strictly, calculates the release-record checksum used by later acceptance records, and compares the record with the installed Docker images: - -```sh -python3 deploy/verify_release.py \ - --archive GuestOps-0.2.1.tar.gz \ - --record release-record.json \ - --commit FULL_40_CHARACTER_SHA \ - --version 0.2.1 \ - --output release-verification.json - -python3 deploy/verify_release.py \ - --archive GuestOps-0.2.1.tar.gz \ - --record release-record.json \ - --commit FULL_40_CHARACTER_SHA \ - --version 0.2.1 \ - --verify-loaded-images \ - --output loaded-image-verification.json -``` - -Retain both verification summaries with the untouched source archive, its checksum, the release record, its SHA-256, and the exact Ansible run metadata. A package from an uncommitted working tree, a mismatched checksum, or a failed Ansible run is not release evidence. Do not silently replace an approved package or rebuild under the same release identity. - -Verify each installed image ID matches the record, set `GUESTOPS_API_IMAGE` and `GUESTOPS_WORKER_IMAGE` to the recorded full-SHA references, and run the deployment preflight. Record the Ansible run, commit, archive checksum and operator in the change ticket. A release tag is an approval marker; do not move or reuse an existing tag. The application and web versions must match before the package is accepted. - -For rollback, first disable worker-driven external writes and reconcile any sending, payment or PMS operation that may have completed since the prior release. Confirm the previous release archive and record are retained, verify its checksum and image IDs, take an encrypted backup, then select the previous recorded image references in `.env` and recreate only the API and worker. Do not roll back MongoDB or the key volume merely to change application images. Run the online preflight, readiness check and read-only smoke test before re-enabling the worker or provider writes. If a release introduced an incompatible data change, follow its release-specific recovery plan rather than starting an older image against newer data. - -## Deployment preflight - -Run from the dedicated GuestOps checkout on Linux with Python 3.11 or later, Docker with Compose, and GnuPG installed. The tool supports the supplied three-service Compose deployment and the `guestops` database only. Docker access is administrator-equivalent; use the designated server operator account. Load the reviewed API, worker and MongoDB images first, configure `.env` with mode 600, and follow [deployment](deployment.md). - -```sh -python3 deploy/ops.py preflight --offline -# After Nginx, DNS, TLS and services are running: -python3 deploy/ops.py preflight -``` - -Preflight checks production settings, shared persistent keys, unpublished MongoDB/worker ports, a loopback API port, required images and mounts, private secret files, and at least 8 GiB free disk. The online check also verifies the configured HTTPS readiness URL. This is not a firewall, capacity or provider acceptance test. Allow additional disk space for the database, images and temporary backup/restore files; the supplied server initially had 18 GiB free. - -Configured provider files must be readable by the API container's `app` user while inaccessible to other users. Set their ownership to that image's application UID and mode 600, and run the backup as an authorized operator able to read them (for example root with its designated public GPG keyring). Keep their parent directory private. Empty shipped example files contain no credentials and do not need this ownership change. Check the UID in the reviewed image rather than assuming it matches your host login. - -## Encrypted backup - -Keep the recovery private key on an administrator-controlled recovery machine, with a securely stored passphrase and a second protected recovery copy. Import only its public key on the server and verify its full 40-character fingerprint through a trusted channel. The tool selects that exact fingerprint; it does not establish who owns the key. Do not put private keys, decrypted backups or provider secrets in GitHub. - -```sh -gpg --import /secure/path/recovery-public.asc -gpg --fingerprint -install -d -m 700 "$HOME/guestops-backups" -read -r -p 'Verified recovery key fingerprint: ' BACKUP_RECIPIENT -python3 deploy/ops.py backup --recipient "$BACKUP_RECIPIENT" \ - --output "$HOME/guestops-backups/guestops-$(date -u +%Y%m%dT%H%M%SZ).tar.gpg" \ - --confirm-maintenance -unset BACKUP_RECIPIENT -``` - -This command causes a maintenance interruption. It stops the API and worker, temporarily pauses MongoDB's TTL expiry monitor, dumps MongoDB, and saves the shared Data Protection keys and deployed configuration. The configuration includes credentials and provider files, so the entire bundle is encrypted. Services and the previous TTL setting are restored in a `finally` block before encryption finishes. A successful backup message does not prove that restarted services are healthy; run the online preflight afterwards. - -No other application or administrator may write to this database during the snapshot. Standalone MongoDB dumps need coordinated writes for consistency; see the [MongoDB backup guidance](https://www.mongodb.com/docs/v8.0/tutorial/backup-and-restore-tools/). The tool is intentionally limited to the dedicated stack. It requires the hexadecimal MongoDB root password generated in the deployment guide. - -Copy the encrypted file off-server to restricted storage after every successful backup. Retain the exact API, worker and MongoDB images with the release: an image tag alone can change, and the drill requires matching image IDs. Agree the backup schedule, retention and tolerated data loss before a hotel pilot. Scheduling, off-server transfer, deletion and alerting are operator responsibilities in this milestone; none is silently installed. - -### Optional systemd schedule - -The repository includes an opt-in daily systemd service and timer. The service follows the Ansible-selected release at `/srv/guestops/current` and stages encrypted files in `/var/backups/guestops`. The application-owned `deploy/ansible/guestops-operations.yml` installs and verifies these units after the exact release is selected; keep the backup timer disabled until the supervised manual backup passes. - -Create the staging directory and environment file without storing a private key or passphrase on the server: - -```sh -sudo install -d -m 700 /var/backups/guestops /var/lib/guestops-backup/gnupg /etc/guestops -sudo install -m 600 /dev/null /etc/guestops/backup.env -sudoedit /etc/guestops/backup.env -sudo env GNUPGHOME=/var/lib/guestops-backup/gnupg gpg --import /secure/path/recovery-public.asc -sudo env GNUPGHOME=/var/lib/guestops-backup/gnupg gpg --fingerprint -``` - -The file contains only these two settings. `BACKUP_RECIPIENT` is the verified 40-character public-key fingerprint already imported into the service account's GPG keyring. - -```text -BACKUP_RECIPIENT=0123456789ABCDEF0123456789ABCDEF01234567 -BACKUP_DIRECTORY=/var/backups/guestops -``` - -Review the unit sandbox against the host, copy `deploy/systemd/guestops-backup.service` and `.timer` to `/etc/systemd/system`, then validate and test before enabling: - -```sh -sudo systemd-analyze verify /etc/systemd/system/guestops-backup.service /etc/systemd/system/guestops-backup.timer -sudo systemctl daemon-reload -sudo systemctl start guestops-backup.service -sudo systemctl status guestops-backup.service -sudo systemctl enable --now guestops-backup.timer -systemctl list-timers guestops-backup.timer -``` - -The timer deliberately causes the same brief maintenance interruption as a manual backup. `Persistent=true` runs a missed event after downtime, so choose and communicate the maintenance window. A successful unit only stages an encrypted file locally. Configure an independently monitored off-host transfer, verify the destination checksum, alert on both unit and transfer failure, and test the alert route. Do not add automatic deletion until retention, legal hold and recovery requirements have named owners. - -### Restricted off-host transfer - -The optional transfer service uses rsync over pinned-host SSH. Create a dedicated upload-only account at the restricted store, disable interactive login, agent/port forwarding and access outside the GuestOps backup directory, and keep its private key only in `/etc/guestops` with mode 600. Pin the reviewed server host key; never use `StrictHostKeyChecking=no`. - -Start from `deploy/systemd/backup-transfer.env.example` and store the completed file as `/etc/guestops/backup-transfer.env` with mode 600. The local and remote directories must already exist and remain private. Install and verify the service and its 15-minute retry timer: - -```sh -sudo install -m 600 deploy/systemd/backup-transfer.env.example /etc/guestops/backup-transfer.env -sudoedit /etc/guestops/backup-transfer.env -sudo install -m 644 deploy/systemd/guestops-backup-transfer.service /etc/systemd/system/ -sudo install -m 644 deploy/systemd/guestops-backup-transfer.timer /etc/systemd/system/ -sudo systemd-analyze verify /etc/systemd/system/guestops-backup-transfer.service /etc/systemd/system/guestops-backup-transfer.timer -sudo systemctl daemon-reload -sudo systemctl start guestops-backup-transfer.service -sudo systemctl enable --now guestops-backup-transfer.timer -``` - -Only files named `guestops-YYYYMMDDTHHMMSSZ.tar.gpg` are eligible. The transfer writes a unique remote partial file, compares the remote and local SHA-256 values, atomically publishes a previously unused final name, verifies it again, and then writes a non-sensitive local marker. An existing remote name is accepted only when its checksum matches. Failed or mismatched transfers are never marked. The service removes local backups older than seven days only when the marker still matches the local checksum; untransferred or changed files are never pruned. - -The restricted store is the durable copy. Configure its independently reviewed retention policy for 35 daily and 12 monthly recovery points. Legal hold must override expiry. The host tool does not delete remote data or enforce remote retention. - -### Zabbix status boundary - -Do not give the Zabbix agent access to Docker, MongoDB credentials or the application owner session. A root-owned systemd probe reads those local sources and atomically publishes aggregate status under `/run/guestops-monitor/status.json`; the agent reads that file only. - -```sh -sudo install -m 644 deploy/systemd/guestops-monitor-status.service /etc/systemd/system/ -sudo install -m 644 deploy/systemd/guestops-monitor-status.timer /etc/systemd/system/ -sudo install -m 644 deploy/systemd/zabbix-agent-guestops.conf.example /etc/zabbix/zabbix_agentd.d/guestops.conf -sudo systemd-analyze verify /etc/systemd/system/guestops-monitor-status.service /etc/systemd/system/guestops-monitor-status.timer -sudo systemctl daemon-reload -sudo systemctl enable --now guestops-monitor-status.timer -sudo systemctl restart zabbix-agent -``` - -Create dependent Zabbix items from `guestops.status` for HTTPS readiness, certificate days remaining, container state/health, worker heartbeat age, backup age, verified-transfer age, free-disk percentage, persistent journal availability and probe error categories. Alert when the heartbeat is older than three minutes; backup or transfer is older than 30 hours; free disk falls below 25% (warning) or 15% (critical); the certificate has fewer than 30 days (warning) or 14 days (critical); any required container is absent/unhealthy; or the probe/timers fail. Route alerts to the monitoring owner and escalate unacknowledged critical events to the technical owner after 15 minutes. Exercise every trigger and its recovery notification with synthetic conditions. - -Temporary plaintext files are held in private directories and removed on normal completion or exceptions. Process termination or power loss can leave temporary data, stopped services or TTL expiry disabled. After an interrupted run, inspect the dedicated project and remove only its identified abandoned temporary directory after securing any recovery material. Restore the recorded TTL setting (normally true) and restart the services: - -```sh -docker compose exec -T mongo mongosh --quiet --nodb --eval 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD); const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:true}); if(!r.ok)quit(1);' -docker compose start api worker -python3 deploy/ops.py preflight -``` - -Use the previous value instead of true if TTL expiry was deliberately disabled beforehand. Do not remove production volumes to recover from a failed backup. - -## Isolated restore drill - -Use a trusted encrypted backup and the recovery key on a Linux recovery machine. Unlock the key through the local GPG agent before running the batch command; never pass its passphrase on the command line. Load the exact reviewed images from the backed-up release first. - -```sh -python3 deploy/ops.py restore-drill /secure/path/guestops-backup.tar.gpg \ - --api-image guestops-api:REVIEWED_RELEASE \ - --mongo-image mongo:8.0 -``` - -The drill decrypts into a private temporary directory, checks the four expected files and their hashes, rejects unsafe archive entries, and verifies that the restored keys decrypt a protected test value. It then restores MongoDB into a newly created container with no external network or published ports and compares every restored collection's count and indexes. TTL expiry is disabled in this disposable database so expired token records do not disappear during comparison. The exact temporary container and its volumes are removed afterwards. No worker or production application is started. - -The decrypted bundle is capped at 8 GiB for this pilot tool. Leave room for the encrypted file, decrypted archive, extracted files and restored database. GPG's [output limit](https://www.gnupg.org/documentation/manuals/gnupg/GPG-Input-and-Output.html) bounds decrypted output; checksums detect corruption, not the identity of the backup creator. Use backups from the trusted operator only. A drill checks counts, indexes and key decryption, not every document's business meaning or live provider connectivity. - -## Actual disaster recovery - -This milestone implements a rehearsal, not an automatic production restore or cutover command. Before real guest data is accepted, rehearse a separate recovery deployment and document its precise image IDs, volume names, secret locations and operator responsibilities. - -Restore into new isolated MongoDB and key volumes; preserve the damaged original for investigation. Reconstruct reviewed configuration from the encrypted bundle without copying old Docker container IDs or blindly executing archived configuration. Restore the `guestops` database with the compatible MongoDB tools and restore the matching key files with the application's required ownership. Check hotel/account records and saved settings before exposing the recovered API. Keep the worker stopped, provider writes disabled and external network access restricted throughout this process. - -**A restored database can predate emails, invoices and PMS changes that providers already completed.** Review pending, sending and uncertain records against provider evidence before enabling any worker, including automatic FAQ rules. Do not replay an older approval merely because the restored record says it is pending. Reconcile external effects, validate account sessions and mailbox authorization, and explicitly approve the cutover only after these checks. Rotate credentials if compromise prompted the recovery. Keep the old deployment stopped when enabling the replacement. - -Run the synthetic encrypted backup and isolated restore drill in a controlled test environment, including actual key decryption and database comparison. This automated check is separate from the required rehearsal on the Debian server with its actual deployment configuration. - -## Milestone 11 acceptance record - -Keep raw backups, restored data, remote paths, host keys, monitoring recipients, screenshots and logs outside Git. Copy `deploy/backup-restore-acceptance.example.json` to the restricted evidence store and replace every placeholder only after completing the supervised exercises. The example deliberately fails. - -The accepted record uses a 24-hour RPO, four-hour RTO, seven-day verified local staging window, 35 daily and 12 monthly off-host recovery points, and the opaque evidence ID `backup-restore`. Bind it to the same release identifiers as the Debian-host and persistence records: - -```sh -python3 deploy/backup_restore_acceptance.py \ - /secure/acceptance/backup-restore.json \ - --expected-commit FULL_40_CHARACTER_SHA \ - --expected-release-record-sha256 RELEASE_RECORD_SHA256 -sha256sum /secure/acceptance/backup-restore.json -``` - -The validator requires matching local/remote backup checksums, exact immutable image identities, a timed isolated restore, tested retention/legal hold and alert escalation, an image rollback that preserves persistent volumes, return to the approved candidate with external writes disabled, separate independent review, healthy monitoring and no unresolved critical findings. Structural validation does not inspect the restricted evidence or authorize a release by itself. - diff --git a/docs/pilot-release.md b/docs/pilot-release.md deleted file mode 100644 index ffb2201..0000000 --- a/docs/pilot-release.md +++ /dev/null @@ -1,93 +0,0 @@ -# Supervised pilot, capacity and release approval - -Milestone 18 is an evidence exercise against the exact approved release, not a feature toggle. Use synthetic data for capacity work and a separately approved, tightly supervised hotel cohort for the pilot. Keep provider writes and FAQ live mode disabled until their individual acceptance records are approved. - -Before the pilot, complete the [Gate B automation, identity and privacy acceptance](gate-b-prerequisites.md) as well as the Google and desktop exercises. These reviews must use the same release identifiers as the final decision. - -## Read-only capacity probe - -The capacity probe logs in once with a dedicated sandbox staff account and sends bounded concurrent GET requests to readiness, hotel settings and cursor-paginated inbox endpoints. It never calls provider integrations, creates records, edits drafts or retains response bodies. Run it only during an approved sandbox window and monitor CPU, memory, MongoDB latency, disk, Nginx and application errors independently. - -```sh -read -r -p 'Capacity account email: ' CAPACITY_EMAIL -read -r -s -p 'Capacity account password: ' CAPACITY_PASSWORD -export CAPACITY_EMAIL CAPACITY_PASSWORD -python3 deploy/capacity_probe.py \ - --origin https://sandbox-guestops.futuresens.co.uk \ - --requests 500 --concurrency 10 \ - --release-commit FULL_40_CHARACTER_SHA \ - --release-record-sha256 RELEASE_RECORD_SHA256 \ - --output /secure/acceptance/capacity.json \ - --confirm-sandbox -unset CAPACITY_EMAIL CAPACITY_PASSWORD -``` - -For the `0.2.1` Gate B candidate, the approved targets are concurrency 10, p95 latency at or below 500 ms, error rate at or below 1%, and at least 25% CPU and memory headroom on the documented four-core, 7.6 GiB host. The generated result reports HTTP observations, not a pass/fail claim. Repeat after a warm-up, investigate every error, and retain independently captured host metrics with the report. Hash both retained files for the approval record. Do not point the probe at a live hotel or increase its built-in bounds to simulate a denial of service. - -## Five-business-day supervised pilot - -Use one approved hotel and named hotel, technical and rollback owners. Start only after the prerequisite evidence below has passed. Keep PMS writes, payment creation and FAQ live mode disabled. Complete one daily review on each of five business days and stop for tenant leakage, credential exposure, data loss, an unapproved or duplicate send, an unreconciled uncertain send, failed rollback or loss of monitoring. - -Copy `deploy/pilot-run.example.json` to the restricted evidence store and replace all placeholders. Resolve critical and high findings; lower-severity findings may be contained only with an owner, expiry and objective rollback trigger. Validate and hash the final record: - -```sh -python3 deploy/pilot_run.py /secure/acceptance/pilot-run.json -sha256sum /secure/acceptance/pilot-run.json -``` - -The example deliberately fails while daily reviews are `not-run`. A structurally valid record does not substitute for the five elapsed business days or independent evidence review. - -## Pilot exit record - -The go/no-go record must bind all evidence to the same release commit and release-record checksum. Record named owners, dates, evidence locations, findings and explicit dispositions for: - -- exact-commit source package, checksum, recorded image IDs, and successful Ansible installation; -- Debian preflight, HTTPS, persistence and controlled reboot; -- encrypted off-host backup and timed isolated restore; -- Google mailbox and reviewed-send acceptance; -- knowledge, AI and FAQ test-mode evaluation; -- identity, privacy, retention and audit review; -- inbox pagination, draft protection, timezone and desktop-parity acceptance; -- capacity targets, observed host headroom and expected pilot workload; -- support, rollback, provider-reconciliation and incident exercises; -- every pilot usability or security finding. - -Approval requires separate named decisions from the hotel pilot owner and technical release owner. Gate C additionally requires independently accepted PMS and payment-provider evidence. A conditional approval must identify the containment, owner, expiry and rollback trigger; an unresolved finding is not silently converted into acceptance. Retain the signed decision with the release rather than committing guest, credential or incident data to this repository. - -Copy `deploy/pilot-approval.example.json` into the restricted release store and complete it only after reviewing the referenced evidence. Approval schema version 2 binds the five-day pilot record and both the capacity report and independently captured host metrics. The example is intentionally invalid while its decision is `pending`. For a contained finding, record its named owner, future expiry and objective rollback trigger. Validate the completed record with: - -```sh -python3 deploy/pilot_approval.py /secure/acceptance/pilot-approval.json -``` - -The validator requires the exact Gate B evidence set, or that set plus independently accepted PMS and payment-provider evidence for Gate C. It also verifies that observed concurrency meets the pre-agreed target and that p95 latency and error rate remain within their pre-agreed bounds. Structural validation does not inspect evidence or authorize rollout by itself. - -## Integrated Gate B bundle validation - -After every individual record passes independent review, validate the complete bundle together. This prevents records from different commits, release records, archives, image builds or sandbox environments from being combined into one approval. It also binds the final decision to the retained capacity report, host metrics and five-day pilot record by checksum. - -```sh -python3 deploy/gate_b_bundle.py \ - --source-archive /secure/releases/GuestOps-0.2.1-COMMIT.tar.gz \ - --source-record /secure/releases/GuestOps-0.2.1-COMMIT.source.json \ - --release-record /secure/releases/release-record.json \ - --host-metrics /secure/acceptance/capacity-host-metrics.json \ - --debian-host /secure/acceptance/debian-host.json \ - --persistence /secure/acceptance/persistence.json \ - --backup-restore /secure/acceptance/backup-restore.json \ - --google-mailbox /secure/acceptance/google-mailbox.json \ - --automation /secure/acceptance/automation.json \ - --identity-privacy /secure/acceptance/identity-privacy.json \ - --inbox-usability /secure/acceptance/inbox-usability.json \ - --capacity /secure/acceptance/capacity.json \ - --incident-support /secure/acceptance/incident-support.json \ - --pilot-findings /secure/acceptance/pilot-run.json \ - --pilot-approval /secure/acceptance/pilot-approval.json \ - --output /secure/acceptance/gate-b-bundle-summary.json -``` - -The output contains only release identity and file checksums, uses mode `0600`, and refuses to overwrite an existing summary. Retain it with the individual validator outputs. A passing bundle proves structural consistency, not that screenshots, provider activity, approvals, findings or other referenced evidence are genuine. - -Complete the [incident and rollback exercise](incident-exercise.md) before marking `incident-support` as passed. Its record must use the same release identifiers and target gate as this decision. Reference the retained exercise record and validator output; do not substitute a local automated-test result for the supervised exercise. - -Complete the [desktop-parity acceptance exercise](desktop-acceptance.md) before marking `inbox-usability` as passed. Bind it to the same release identifiers and retain its independently reviewed record outside the repository. diff --git a/src/GuestOps.Api/GoogleMailbox.cs b/src/GuestOps.Api/GoogleMailbox.cs index f1accd9..514db2b 100644 --- a/src/GuestOps.Api/GoogleMailbox.cs +++ b/src/GuestOps.Api/GoogleMailbox.cs @@ -16,7 +16,7 @@ public sealed class GoogleMailbox(HttpClient http, IConfiguration config, IStore private readonly IDataProtector protector = protection.CreateProtector("GoogleMailbox.refresh.v1"); private string ClientId => config["Google:ClientId"] ?? ""; private string ClientSecret => config["Google:ClientSecret"] ?? ""; - private string Callback => (config["PublicUrl"] ?? "https://sandbox-guestops.futuresens.co.uk").TrimEnd('/') + "/api/integrations/google/callback"; + private string Callback => (config["PublicUrl"] ?? "https://guestops-sandbox.futuresens.co.uk").TrimEnd('/') + "/api/integrations/google/callback"; public bool Configured => ClientId.Length > 0 && ClientSecret.Length > 0; public bool SendingConfigured => Configured && config.GetValue("Google:EnableSending"); public string AuthorizationUrl(string state) => "https://accounts.google.com/o/oauth2/v2/auth?" + string.Join("&", new Dictionary { diff --git a/tests/production_smoke.py b/tests/production_smoke.py index 39666a0..b2e9bea 100644 --- a/tests/production_smoke.py +++ b/tests/production_smoke.py @@ -72,7 +72,7 @@ else: hotel["signature"] = "Persisted across container restart" request("/api/hotel", "PUT", hotel) invite = request("/api/team/invite", "POST", {"name": "Deployment Staff", "email": "deployment-staff@example.invalid"}) - assert invite["link"].startswith("https://sandbox-guestops.futuresens.co.uk/account#token=") + assert invite["link"].startswith("https://guestops-sandbox.futuresens.co.uk/account#token=") request("/api/auth/logout", "POST") request("/api/hotel", expected=401) print("Production smoke checks passed: built UI, secure cookies, owner login, persisted settings and staff invitation, onboarding and logout.") diff --git a/tests/test_ansible_handoff.py b/tests/test_ansible_handoff.py deleted file mode 100644 index e0cd4d4..0000000 --- a/tests/test_ansible_handoff.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -import unittest - - -ROOT = Path(__file__).resolve().parents[1] -PLAYBOOK = ROOT / "deploy" / "ansible" / "guestops.yml" - - -class AnsibleHandoffTests(unittest.TestCase): - def test_release_is_verified_before_build_and_start(self): - text = PLAYBOOK.read_text(encoding="utf-8") - controller_verify = text.index("Verify source package on the Ansible controller") - target_verify = text.index("Verify transferred package and record on the target") - api_build = text.index("Build API image from the verified source") - preflight = text.index("Run offline deployment preflight") - compose_start = text.index("Start the verified release without rebuilding") - readiness = text.index("Wait for loopback readiness") - public_readiness = text.index("Verify public HTTPS readiness and certificate trust") - current = text.index("Select the current successful release") - self.assertLess(controller_verify, target_verify) - self.assertLess(target_verify, api_build) - self.assertLess(api_build, preflight) - self.assertLess(preflight, compose_start) - self.assertLess(compose_start, readiness) - self.assertLess(readiness, public_readiness) - self.assertLess(public_readiness, current) - - def test_playbook_uses_safe_release_controls(self): - text = PLAYBOOK.read_text(encoding="utf-8") - for required in ( - "guestops_release_commit is match('^[0-9a-f]{40}$')", - "checksum_algorithm: sha256", - "deploy/verify_source_package.py", - "deploy/release_record.py", - "deploy/ops.py", - "--offline", - "--no-build", - "GOOGLE_ENABLE_SENDING", - "AUTO_REPLY_ENABLE_LIVE", - "no_log: true", - "http://127.0.0.1:8080/health/ready", - "Reject public API or MongoDB listeners", - "Validate Nginx configuration", - "validate_certs: true", - "enabled: true", - ): - self.assertIn(required, text) - self.assertNotIn("ansible.builtin.shell", text) - self.assertNotIn("password=", text.lower()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_ansible_operations.py b/tests/test_ansible_operations.py deleted file mode 100644 index 7c0e5cf..0000000 --- a/tests/test_ansible_operations.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -import unittest - - -ROOT = Path(__file__).resolve().parents[1] -PLAYBOOK = ROOT / "deploy" / "ansible" / "guestops-operations.yml" - - -class AnsibleOperationsTests(unittest.TestCase): - def test_operations_require_release_and_private_inputs(self): - text = PLAYBOOK.read_text(encoding="utf-8") - for required in ( - "/srv/guestops/current", - "guestops_release_commit is match('^[0-9a-f]{40}$')", - "guestops_backup_recipient is match('^[A-Fa-f0-9]{40}$')", - "guestops_durable_logs_configured | bool", - "/etc/guestops/backup.env", - "/etc/guestops/backup-transfer.env", - "/etc/guestops/backup-transfer.key", - "/etc/guestops/backup-known-hosts", - "no_log: true", - ): - self.assertIn(required, text) - - def test_public_key_units_monitoring_and_evidence_are_integrated(self): - text = PLAYBOOK.read_text(encoding="utf-8") - ordered = ( - "Import recovery public key only", - "Verify installed systemd units", - "Enable transfer retry and monitoring timers", - "Install restricted Zabbix status item", - "Generate initial non-sensitive monitoring status", - "Retain monitoring status on the Ansible controller", - ) - positions = [text.index(item) for item in ordered] - self.assertEqual(positions, sorted(positions)) - self.assertIn("guestops_enable_backup_schedule: false", text) - self.assertNotIn("ansible.builtin.shell", text) - self.assertNotIn("PRIVATE KEY", text) - - def test_systemd_units_follow_selected_release(self): - for name in ( - "guestops-backup.service", - "guestops-backup-transfer.service", - "guestops-monitor-status.service", - ): - text = (ROOT / "deploy" / "systemd" / name).read_text(encoding="utf-8") - self.assertIn("/srv/guestops/current", text) - self.assertNotIn("WorkingDirectory=/srv/guestops\n", text) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_backup_restore_acceptance.py b/tests/test_backup_restore_acceptance.py deleted file mode 100644 index 0b1be9d..0000000 --- a/tests/test_backup_restore_acceptance.py +++ /dev/null @@ -1,129 +0,0 @@ -import importlib.util -from pathlib import Path -import unittest - - -spec = importlib.util.spec_from_file_location( - "backup_restore_acceptance", - Path(__file__).resolve().parents[1] / "deploy" / "backup_restore_acceptance.py") -acceptance = importlib.util.module_from_spec(spec) -spec.loader.exec_module(acceptance) - -COMMIT = "a" * 40 -RELEASE_SHA = "b" * 64 - - -def valid_record(): - return { - "schemaVersion": 1, - "system": "guestops-backup-recovery", - "evidenceId": "backup-restore", - "dataClassification": "synthetic-only", - "releaseVersion": "0.2.1", - "releaseCommit": COMMIT, - "releaseRecordSha256": RELEASE_SHA, - "archiveSha256": "c" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", - "hostIdentifier": "guestops-sandbox-01", - "images": { - "api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64}, - "worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64}, - "mongo": {"reference": "mongo:8.0", "id": "sha256:" + "f" * 64}, - }, - "owners": { - "backup": "Backup owner", - "monitoring": "Monitoring owner", - "recoveryOperator": "Recovery operator", - "technicalEscalation": "Technical owner", - "retention": "Retention owner", - "independentReviewer": "Independent reviewer", - }, - "startedAt": "2026-10-01T09:00:00Z", - "endedAt": "2026-10-01T12:00:00Z", - "reviewedAt": "2026-10-01T13:00:00Z", - "recoveryObjectives": { - "targetRpoHours": 24, "observedRpoHours": 1, - "targetRtoMinutes": 240, "observedRtoMinutes": 180, - }, - "retention": { - "localVerifiedDays": 7, "offHostDaily": 35, - "offHostMonthly": 12, "legalHoldOverrideTested": True, - }, - "backup": { - "createdAt": "2026-10-01T08:00:00Z", - "sha256": "1" * 64, - "transferredSha256": "1" * 64, - "privateKeyPresentOnHost": False, - }, - "rollback": { - "previousReleaseCommit": "2" * 40, - "previousArchiveSha256": "3" * 64, - "previousImages": { - "api": {"reference": "guestops-api:" + "2" * 40, "id": "sha256:" + "4" * 64}, - "worker": {"reference": "guestops-worker:" + "2" * 40, "id": "sha256:" + "5" * 64}, - }, - "persistentVolumesReplaced": False, - "restoredReleaseCommit": COMMIT, - "unresolvedOperations": 0, - "finalControls": { - "googleSending": "disabled", "faqLiveMode": "disabled", - "pmsWrites": "disabled", "paymentCreation": "disabled", - }, - }, - "monitoringState": "healthy", - "unresolvedCriticalFindings": 0, - "scenarios": [ - {"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]} - for index, scenario in enumerate(sorted(acceptance.SCENARIOS), 1) - ], - } - - -class BackupRestoreAcceptanceTests(unittest.TestCase): - def test_complete_record_passes(self): - acceptance.validate(valid_record(), COMMIT, RELEASE_SHA) - - def test_release_identity_and_images_are_bound(self): - record = valid_record(); record["releaseCommit"] = "9" * 40 - with self.assertRaisesRegex(ValueError, "approved candidate"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["images"]["api"]["reference"] = "guestops-api:latest" - with self.assertRaisesRegex(ValueError, "full approved commit"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["rollback"]["previousImages"]["worker"]["id"] = "mutable" - with self.assertRaisesRegex(ValueError, "retained previous release identity"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - - def test_recovery_objectives_and_checksums_must_pass(self): - record = valid_record(); record["recoveryObjectives"]["observedRtoMinutes"] = 241 - with self.assertRaisesRegex(ValueError, "four-hour"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["backup"]["transferredSha256"] = "9" * 64 - with self.assertRaisesRegex(ValueError, "checksums must match"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["recoveryObjectives"]["observedRpoHours"] = 2 - with self.assertRaisesRegex(ValueError, "match the backup"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - - def test_independent_review_retention_and_final_state_are_required(self): - record = valid_record(); record["owners"]["independentReviewer"] = record["owners"]["backup"] - with self.assertRaisesRegex(ValueError, "different"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["retention"]["legalHoldOverrideTested"] = False - with self.assertRaisesRegex(ValueError, "Retention"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["rollback"]["persistentVolumesReplaced"] = True - with self.assertRaisesRegex(ValueError, "must not replace"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - - def test_exact_passed_scenarios_and_monitoring_are_required(self): - record = valid_record(); record["scenarios"].pop() - with self.assertRaisesRegex(ValueError, "exact backup/recovery scenario"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - record = valid_record(); record["monitoringState"] = "degraded" - with self.assertRaisesRegex(ValueError, "Monitoring must be healthy"): - acceptance.validate(record, COMMIT, RELEASE_SHA) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_backup_transfer.py b/tests/test_backup_transfer.py deleted file mode 100644 index 60c8ca3..0000000 --- a/tests/test_backup_transfer.py +++ /dev/null @@ -1,81 +0,0 @@ -import importlib.util -import json -import os -from pathlib import Path -import tempfile -import time -import unittest -from unittest.mock import patch - - -spec = importlib.util.spec_from_file_location( - "backup_transfer", Path(__file__).resolve().parents[1] / "deploy" / "backup_transfer.py") -transfer = importlib.util.module_from_spec(spec) -spec.loader.exec_module(transfer) - - -class BackupTransferTests(unittest.TestCase): - def backup(self, directory: Path, name="guestops-20261001T021700Z.tar.gpg") -> Path: - path = directory / name - path.write_bytes(b"encrypted-backup") - return path - - def config(self, directory: Path): - return { - "directory": directory, "host": "store.example.invalid", "user": "guestops_upload", - "remote": "/restricted/guestops", "identity": Path("/safe/key"), - "known_hosts": Path("/safe/known_hosts"), - } - - def test_existing_matching_remote_is_marked_without_upload(self): - with tempfile.TemporaryDirectory() as folder: - backup = self.backup(Path(folder)); checksum = transfer.digest(backup) - with patch.object(transfer, "remote_digest", return_value=checksum), \ - patch.object(transfer, "run") as run: - transfer.transfer_one(self.config(Path(folder)), backup) - run.assert_not_called() - marker = json.loads(transfer.marker_path(backup).read_text(encoding="utf-8")) - self.assertEqual(marker["sha256"], checksum) - - def test_existing_mismatched_remote_is_never_overwritten(self): - with tempfile.TemporaryDirectory() as folder: - backup = self.backup(Path(folder)) - with patch.object(transfer, "remote_digest", return_value="9" * 64), \ - patch.object(transfer, "run") as run: - with self.assertRaisesRegex(RuntimeError, "different checksum"): - transfer.transfer_one(self.config(Path(folder)), backup) - run.assert_not_called() - self.assertFalse(transfer.marker_path(backup).exists()) - - def test_new_remote_is_uploaded_verified_and_marked(self): - with tempfile.TemporaryDirectory() as folder: - backup = self.backup(Path(folder)); checksum = transfer.digest(backup) - with patch.object(transfer, "remote_digest", side_effect=[None, checksum, checksum]), \ - patch.object(transfer, "run", return_value=b"") as run, \ - patch.object(transfer.subprocess, "run"): - transfer.transfer_one(self.config(Path(folder)), backup) - self.assertTrue(any(call.args[0][0] == "rsync" for call in run.call_args_list)) - self.assertTrue(transfer.marker_path(backup).exists()) - - def test_prune_removes_only_old_checksum_verified_backups(self): - with tempfile.TemporaryDirectory() as folder: - directory = Path(folder) - verified = self.backup(directory) - checksum = transfer.digest(verified) - transfer.write_marker(verified, checksum) - unverified = self.backup(directory, "guestops-20261002T021700Z.tar.gpg") - old = time.time() - 8 * 86400 - os.utime(verified, (old, old)); os.utime(unverified, (old, old)) - removed = transfer.prune_verified(self.config(directory), 7, now=time.time()) - self.assertEqual(removed, 1) - self.assertFalse(verified.exists()) - self.assertTrue(unverified.exists()) - - def test_retention_bounds_are_enforced(self): - with tempfile.TemporaryDirectory() as folder: - with self.assertRaisesRegex(RuntimeError, "between 1 and 365"): - transfer.prune_verified(self.config(Path(folder)), 0) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_capacity_probe.py b/tests/test_capacity_probe.py deleted file mode 100644 index d15e83b..0000000 --- a/tests/test_capacity_probe.py +++ /dev/null @@ -1,26 +0,0 @@ -import importlib.util -from pathlib import Path -import unittest - - -spec = importlib.util.spec_from_file_location("capacity_probe", Path(__file__).resolve().parents[1] / "deploy" / "capacity_probe.py") -probe = importlib.util.module_from_spec(spec) -spec.loader.exec_module(probe) - - -class CapacityProbeTests(unittest.TestCase): - def test_summary_reports_failures_and_nearest_rank_latency(self): - report = probe.summarize([(True, 10), (True, 20), (False, 30), (True, 40), (True, 100)], 2) - self.assertEqual(report["requests"], 5) - self.assertEqual(report["successes"], 4) - self.assertEqual(report["failures"], 1) - self.assertEqual(report["errorRate"], 0.2) - self.assertEqual(report["latencyMs"], {"median": 30, "p95": 100, "maximum": 100}) - - def test_summary_rejects_empty_results(self): - with self.assertRaisesRegex(ValueError, "At least one"): - probe.summarize([], 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_debian_acceptance.py b/tests/test_debian_acceptance.py deleted file mode 100644 index f59e842..0000000 --- a/tests/test_debian_acceptance.py +++ /dev/null @@ -1,113 +0,0 @@ -import importlib.util -from pathlib import Path -import unittest - - -spec = importlib.util.spec_from_file_location( - "debian_acceptance", Path(__file__).resolve().parents[1] / "deploy" / "debian_acceptance.py") -acceptance = importlib.util.module_from_spec(spec) -spec.loader.exec_module(acceptance) - -COMMIT = "a" * 40 -RELEASE_SHA = "b" * 64 - - -def common(system): - return { - "schemaVersion": 1, - "system": system, - "evidenceId": acceptance.SYSTEMS[system]["evidenceId"], - "releaseVersion": "0.2.1", - "releaseCommit": COMMIT, - "releaseRecordSha256": RELEASE_SHA, - "archiveSha256": "c" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", - "hostIdentifier": "guestops-sandbox-01", - "images": { - "api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64}, - "worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64}, - }, - "operator": "Deployment operator", - "reviewedBy": "Independent reviewer", - "startedAt": "2026-09-30T09:00:00Z", - "endedAt": "2026-09-30T10:00:00Z", - "reviewedAt": "2026-09-30T11:00:00Z", - "unresolvedCriticalFindings": 0, - "scenarios": [ - {"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]} - for index, scenario in enumerate(sorted(acceptance.SYSTEMS[system]["scenarios"]), 1) - ], - } - - -def valid_records(): - host = common("guestops-debian-host") - host["hostFacts"] = { - "debianMajor": 12, - "cpuCores": 4, - "memoryBytes": 8_140_382_208, - "freeDiskBytes": 14_275_686_400, - "publicTcpPorts": [80, 443], - } - host["featureControls"] = { - "googleSending": "disabled", - "faqLiveMode": "disabled", - "pmsWrites": "disabled", - "paymentCreation": "disabled", - } - persistence = common("guestops-persistence") - persistence["drillCommand"] = "python3 deploy/ops.py persistence-drill --confirm-restart" - return host, persistence - - -class DebianAcceptanceTests(unittest.TestCase): - def test_complete_matching_records_pass(self): - acceptance.validate_pair(*valid_records(), COMMIT, RELEASE_SHA) - - def test_expected_release_identity_is_required(self): - host, persistence = valid_records() - host["releaseCommit"] = "f" * 40 - with self.assertRaisesRegex(ValueError, "approved candidate"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - host, persistence = valid_records() - persistence["releaseRecordSha256"] = "f" * 64 - with self.assertRaisesRegex(ValueError, "retained release record"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - - def test_exact_images_and_cross_record_identity_are_required(self): - host, persistence = valid_records() - host["images"]["api"]["reference"] = "guestops-api:latest" - with self.assertRaisesRegex(ValueError, "full approved commit"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - host, persistence = valid_records() - persistence["archiveSha256"] = "f" * 64 - with self.assertRaisesRegex(ValueError, "same archiveSha256"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - - def test_host_capacity_ports_and_disabled_controls_are_required(self): - host, persistence = valid_records() - host["hostFacts"]["publicTcpPorts"] = [80, 443, 8080] - with self.assertRaisesRegex(ValueError, "Only TCP ports"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - host, persistence = valid_records() - host["featureControls"]["googleSending"] = "enabled" - with self.assertRaisesRegex(ValueError, "must remain disabled"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - - def test_independent_review_scenarios_and_findings_are_required(self): - host, persistence = valid_records() - host["reviewedBy"] = host["operator"] - with self.assertRaisesRegex(ValueError, "different people"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - host, persistence = valid_records() - persistence["scenarios"][0]["status"] = "not-run" - with self.assertRaisesRegex(ValueError, "has not passed"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - host, persistence = valid_records() - host["unresolvedCriticalFindings"] = 1 - with self.assertRaisesRegex(ValueError, "critical findings"): - acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_desktop_acceptance.py b/tests/test_desktop_acceptance.py index 86dadc8..80c4f60 100644 --- a/tests/test_desktop_acceptance.py +++ b/tests/test_desktop_acceptance.py @@ -16,7 +16,7 @@ def valid_record(): "desktopBaselineCommit": acceptance.DESKTOP_BASELINE, "releaseCommit": "a" * 40, "releaseRecordSha256": "b" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", + "environment": "https://guestops-sandbox.futuresens.co.uk", "browser": {"name": "Microsoft Edge", "version": "140.0.0.0", "operatingSystem": "Windows 11"}, "viewport": {"width": 1440, "height": 900, "deviceScaleFactor": 1}, "hotelTimeZone": "Europe/London", diff --git a/tests/test_gate_b_acceptance.py b/tests/test_gate_b_acceptance.py index 3cfcbe4..d09045b 100644 --- a/tests/test_gate_b_acceptance.py +++ b/tests/test_gate_b_acceptance.py @@ -21,7 +21,7 @@ def base(system): return { "schemaVersion": 1, "system": system, "targetGate": "B", "releaseCommit": "a" * 40, "releaseRecordSha256": "b" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", + "environment": "https://guestops-sandbox.futuresens.co.uk", "operator": "Acceptance operator", "reviewedBy": "Independent reviewer", "startedAt": "2026-10-01T09:00:00Z", "endedAt": "2026-10-01T10:00:00Z", "reviewedAt": "2026-10-01T11:00:00Z", } diff --git a/tests/test_gate_b_bundle.py b/tests/test_gate_b_bundle.py deleted file mode 100644 index c9742ce..0000000 --- a/tests/test_gate_b_bundle.py +++ /dev/null @@ -1,141 +0,0 @@ -import contextlib -import hashlib -import importlib.util -import json -from pathlib import Path -import tempfile -import unittest -from unittest.mock import patch - - -ROOT = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location("gate_b_bundle", ROOT / "deploy" / "gate_b_bundle.py") -bundle = importlib.util.module_from_spec(spec) -spec.loader.exec_module(bundle) - - -COMMIT = "a" * 40 -API = {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "b" * 64} -WORKER = {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "c" * 64} -ORIGIN = "https://sandbox-guestops.futuresens.co.uk" - - -class GateBBundleTests(unittest.TestCase): - def fixture(self, directory: str): - root = Path(directory) - archive = root / "GuestOps-0.2.1-aaaaaaaaaaaa.tar.gz" - source = root / "GuestOps-0.2.1-aaaaaaaaaaaa.source.json" - release = root / "release-record.json" - metrics = root / "host-metrics.json" - archive.write_bytes(b"approved source") - artifact = { - "name": archive.name, - "size": archive.stat().st_size, - "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), - } - source.write_text(json.dumps({"schemaVersion": 1, "version": "0.2.1", "commit": COMMIT, "artifact": artifact}), encoding="utf-8") - release.write_text(json.dumps({ - "schemaVersion": 1, "version": "0.2.1", "commit": COMMIT, - "artifact": artifact, "images": {"api": API, "worker": WORKER}, - }), encoding="utf-8") - release_sha = hashlib.sha256(release.read_bytes()).hexdigest() - metrics.write_bytes(b'{"cpu":40,"memory":35}') - - values = {} - paths = {} - for name in bundle.RECORD_KEYS: - value = {"releaseCommit": COMMIT, "releaseRecordSha256": release_sha} - if name in { - "debian-host", "persistence", "backup-restore", "google-mailbox", - "automation", "identity-privacy", "inbox-usability", "incident-support", - "pilot-findings", - }: - value["environment"] = ORIGIN - if name in {"debian-host", "persistence", "backup-restore"}: - value["archiveSha256"] = artifact["sha256"] - value["images"] = {"api": API, "worker": WORKER} - values[name] = value - - values["capacity"].update({ - "schemaVersion": 1, "kind": "guestops-read-only-capacity", - "originHost": "sandbox-guestops.futuresens.co.uk", - "paths": ["/health/ready", "/api/hotel", "/api/conversations/page"], - "concurrency": 10, "requests": 100, "successes": 100, "failures": 0, - "errorRate": 0.0, "latencyMs": {"median": 100, "p95": 250, "maximum": 300}, - }) - pilot_path = root / "pilot-findings.json" - pilot_path.write_text(json.dumps(values["pilot-findings"]), encoding="utf-8") - paths["pilot-findings"] = pilot_path - capacity_path = root / "capacity.json" - capacity_path.write_text(json.dumps(values["capacity"]), encoding="utf-8") - paths["capacity"] = capacity_path - values["pilot-approval"].update({ - "capacity": { - "reportSha256": hashlib.sha256(capacity_path.read_bytes()).hexdigest(), - "hostMetricsSha256": hashlib.sha256(metrics.read_bytes()).hexdigest(), - "observedConcurrency": 10, "observedP95Ms": 250, "observedErrorRate": 0.0, - }, - "pilot": {"recordSha256": hashlib.sha256(pilot_path.read_bytes()).hexdigest()}, - }) - for name in bundle.RECORD_KEYS - paths.keys(): - path = root / f"{name}.json" - path.write_text(json.dumps(values[name]), encoding="utf-8") - paths[name] = path - records = {name: (paths[name], values[name]) for name in bundle.RECORD_KEYS} - return archive, source, release, metrics, records - - @contextlib.contextmanager - def mocked_individual_validators(self): - patches = [ - patch.object(bundle.debian_acceptance, "validate_pair"), - patch.object(bundle.backup_restore_acceptance, "validate"), - patch.object(bundle.google_acceptance, "validate"), - patch.object(bundle.automation_acceptance, "validate"), - patch.object(bundle.identity_privacy_acceptance, "validate"), - patch.object(bundle.desktop_acceptance, "validate"), - patch.object(bundle.incident_exercise, "validate"), - patch.object(bundle.pilot_run, "validate"), - patch.object(bundle.pilot_approval, "validate"), - ] - with contextlib.ExitStack() as stack: - for item in patches: - stack.enter_context(item) - yield - - def test_complete_bundle_is_bound_to_one_release(self): - with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): - archive, source, release, metrics, records = self.fixture(directory) - result = bundle.validate_bundle(archive, source, release, records, metrics) - self.assertTrue(result["validated"]) - self.assertEqual(result["releaseCommit"], COMMIT) - self.assertEqual(set(result["records"]), bundle.RECORD_KEYS) - - def test_different_environment_or_checksum_is_rejected(self): - with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): - archive, source, release, metrics, records = self.fixture(directory) - records["google-mailbox"][1]["environment"] = "https://other.example.invalid" - with self.assertRaisesRegex(ValueError, "different environments"): - bundle.validate_bundle(archive, source, release, records, metrics) - - with tempfile.TemporaryDirectory() as directory, self.mocked_individual_validators(): - archive, source, release, metrics, records = self.fixture(directory) - records["pilot-approval"][1]["capacity"]["reportSha256"] = "0" * 64 - with self.assertRaisesRegex(ValueError, "capacity report checksum"): - bundle.validate_bundle(archive, source, release, records, metrics) - - def test_capacity_counts_and_latency_are_checked(self): - report = { - "schemaVersion": 1, "kind": "guestops-read-only-capacity", - "releaseCommit": COMMIT, "releaseRecordSha256": "d" * 64, - "paths": ["/health/ready", "/api/hotel", "/api/conversations/page"], - "concurrency": 10, "requests": 10, "successes": 9, "failures": 1, - "errorRate": 0.1, "latencyMs": {"median": 10, "p95": 20, "maximum": 30}, - } - bundle.validate_capacity(report, COMMIT, "d" * 64) - report["failures"] = 2 - with self.assertRaisesRegex(ValueError, "do not match requests"): - bundle.validate_capacity(report, COMMIT, "d" * 64) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_google_acceptance.py b/tests/test_google_acceptance.py index 5310a1e..39b7a4a 100644 --- a/tests/test_google_acceptance.py +++ b/tests/test_google_acceptance.py @@ -14,7 +14,7 @@ def valid_report(): "system": "google-mailbox", "releaseCommit": "a" * 40, "releaseRecordSha256": "b" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", + "environment": "https://guestops-sandbox.futuresens.co.uk", "mailboxLabel": "sandbox mailbox A", "operator": "Test operator", "startedAt": "2026-09-29T10:00:00Z", diff --git a/tests/test_incident_exercise.py b/tests/test_incident_exercise.py index ef586dc..b6ca6b5 100644 --- a/tests/test_incident_exercise.py +++ b/tests/test_incident_exercise.py @@ -17,7 +17,7 @@ def valid_record(gate="B"): "dataClassification": "synthetic-only", "releaseCommit": "a" * 40, "releaseRecordSha256": "b" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", + "environment": "https://guestops-sandbox.futuresens.co.uk", "operator": "Exercise operator", "incidentCommander": "Incident commander", "reviewedBy": "Independent reviewer", diff --git a/tests/test_monitor_status.py b/tests/test_monitor_status.py deleted file mode 100644 index 3324c9b..0000000 --- a/tests/test_monitor_status.py +++ /dev/null @@ -1,75 +0,0 @@ -import datetime as dt -import importlib.util -import json -import os -from pathlib import Path -import tempfile -import unittest -from unittest.mock import patch - - -spec = importlib.util.spec_from_file_location( - "monitor_status", Path(__file__).resolve().parents[1] / "deploy" / "monitor_status.py") -monitor = importlib.util.module_from_spec(spec) -spec.loader.exec_module(monitor) - - -class MonitorStatusTests(unittest.TestCase): - def test_compose_json_is_reduced_to_safe_state(self): - value = json.dumps([ - {"Service": "api", "State": "running", "Health": "healthy", "Publishers": "secret"}, - {"Service": "worker", "State": "running", "Health": ""}, - {"Service": "mongo", "State": "running", "Health": "healthy"}, - ]).encode() - self.assertEqual(monitor.parse_compose(value)["api"], {"state": "running", "health": "healthy"}) - self.assertNotIn("Publishers", monitor.parse_compose(value)["api"]) - - def test_age_ignores_symlinks_and_unexpected_files(self): - with tempfile.TemporaryDirectory() as folder: - directory = Path(folder) - backup = directory / "guestops-20261001T021700Z.tar.gpg" - backup.write_bytes(b"fixture") - moment = dt.datetime(2026, 10, 1, 3, 17, tzinfo=dt.timezone.utc) - os.utime(backup, (moment.timestamp() - 3600, moment.timestamp() - 3600)) - (directory / "secret.txt").write_text("ignored", encoding="utf-8") - self.assertEqual(monitor.age_seconds(directory, monitor.BACKUP_NAME, moment), 3600) - - def test_build_status_contains_only_aggregate_health(self): - now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc) - compose = json.dumps([ - {"Service": name, "State": "running", "Health": "healthy"} - for name in ("api", "worker", "mongo") - ]).encode() - usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})() - with tempfile.TemporaryDirectory() as folder, \ - patch.object(monitor, "https_status", return_value={"ready": True, "certificateDaysRemaining": 60}), \ - patch.object(monitor, "run", return_value=compose), \ - patch.object(monitor, "worker_heartbeat_age", return_value=30), \ - patch.object(monitor.shutil, "disk_usage", return_value=usage): - status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now) - self.assertEqual(errors, []) - self.assertEqual(status["workerHeartbeatAgeSeconds"], 30) - self.assertEqual(status["diskFreePercent"], 50) - self.assertNotIn("credentials", json.dumps(status).lower()) - - def test_failed_probes_write_categories_not_exception_details(self): - now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc) - usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})() - with tempfile.TemporaryDirectory() as folder, \ - patch.object(monitor, "https_status", side_effect=RuntimeError("secret value")), \ - patch.object(monitor, "run", side_effect=RuntimeError("secret value")), \ - patch.object(monitor, "worker_heartbeat_age", side_effect=RuntimeError("secret value")), \ - patch.object(monitor.shutil, "disk_usage", return_value=usage): - status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now) - self.assertGreaterEqual(len(errors), 3) - self.assertNotIn("secret value", json.dumps(status)) - - def test_status_output_is_atomic_json(self): - with tempfile.TemporaryDirectory() as folder: - path = Path(folder) / "status.json" - monitor.write_status(path, {"schemaVersion": 1, "errors": []}) - self.assertEqual(json.loads(path.read_text(encoding="utf-8"))["schemaVersion"], 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_ops.py b/tests/test_ops.py deleted file mode 100644 index ecac757..0000000 --- a/tests/test_ops.py +++ /dev/null @@ -1,186 +0,0 @@ -import contextlib -import hashlib -import importlib.util -import io -import json -import os -from pathlib import Path -import tarfile -import tempfile -import unittest -from unittest.mock import patch - -spec = importlib.util.spec_from_file_location("ops", Path(__file__).resolve().parents[1] / "deploy" / "ops.py") -ops = importlib.util.module_from_spec(spec) -spec.loader.exec_module(ops) - - -class ArchiveTests(unittest.TestCase): - def test_persistence_layout_resolves_shared_keys_and_database(self): - config = { - "services": { - "api": {"volumes": [{"type": "volume", "source": "app-keys", "target": "/var/lib/guestops/keys"}]}, - "worker": {"volumes": [{"type": "volume", "source": "app-keys", "target": "/var/lib/guestops/keys"}]}, - "mongo": {"volumes": [{"type": "volume", "source": "mongo-data", "target": "/data/db"}]}, - }, - "volumes": { - "app-keys": {"name": "guestops_app-keys"}, - "mongo-data": {"name": "guestops_mongo-data"}, - }, - } - self.assertEqual(ops.persistence_layout(config), {"keys": "guestops_app-keys", "database": "guestops_mongo-data"}) - - def test_persistence_layout_rejects_anonymous_or_split_keys(self): - config = { - "services": { - "api": {"volumes": [{"type": "volume", "source": "api-keys", "target": "/var/lib/guestops/keys"}]}, - "worker": {"volumes": [{"type": "volume", "source": "worker-keys", "target": "/var/lib/guestops/keys"}]}, - "mongo": {"volumes": [{"type": "volume", "source": "mongo-data", "target": "/data/db"}]}, - }, - "volumes": {"api-keys": {}, "worker-keys": {}, "mongo-data": {}}, - } - with self.assertRaisesRegex(RuntimeError, "key volumes differ"): - ops.persistence_layout(config) - - config["services"]["worker"]["volumes"][0] = {"type": "volume", "target": "/var/lib/guestops/keys"} - with self.assertRaisesRegex(RuntimeError, "named persistent volume"): - ops.persistence_layout(config) - - def test_persistence_drill_requires_explicit_restart_confirmation(self): - with self.assertRaisesRegex(RuntimeError, "confirm-restart"): - ops.persistence_drill(type("Args", (), {"confirm_restart": False})()) - - def test_persistence_drill_restarts_then_recreates_stateless_services(self): - compose_calls = [] - inventory = json.dumps({"bytes": 1, "collections": {"hotels": {"count": 1, "indexes": []}}}).encode() - - def compose(*args, **kwargs): - compose_calls.append(args) - return b"a" * 30 if args[-1] == "--backup-probe" else b"" - - def mongo(script): - return inventory if "storageSize" in script else b"" - - with patch.object(ops, "configuration", return_value={}), \ - patch.object(ops, "persistence_layout", return_value={"keys": "keys", "database": "data"}), \ - patch.object(ops, "volume_identity", side_effect=lambda name: {"name": name, "driver": "local", "scope": "local"}), \ - patch.object(ops, "compose", side_effect=compose), \ - patch.object(ops, "mongo", side_effect=mongo), \ - patch.object(ops, "wait_for", side_effect=lambda action, message: action()), \ - patch.object(ops, "readiness"): - ops.persistence_drill(type("Args", (), {"confirm_restart": True})()) - - self.assertIn(("restart", "-t", "150", "mongo"), compose_calls) - self.assertIn(("restart", "-t", "150", "api", "worker"), compose_calls) - self.assertIn(("up", "-d", "--no-build", "--force-recreate", "api", "worker"), compose_calls) - self.assertTrue(any("--verify-backup-probe" in call for call in compose_calls)) - - def test_empty_provider_templates_are_not_secret_configuration(self): - for section, target in (("Pms", "/run/guestops/pms.json"), ("Payments", "/run/guestops/payments.json")): - self.assertFalse(ops.provider_configured({section: {"Hotels": {}}}, target)) - self.assertTrue(ops.provider_configured({section: {"Hotels": {"fixture": {"Key": "fixture"}}}}, target)) - self.assertTrue(ops.provider_configured({section: {"Unknown": "fixture"}}, target)) - - def bundle(self, root, change=None): - files = {name: b"fixture backup data" for name in ops.FILES - {"manifest.json"}} - files["manifest.json"] = json.dumps({"format": 1, "sha256": {name: hashlib.sha256(data).hexdigest() for name, data in files.items()}}).encode() - target = root / "bundle.tar" - with tarfile.open(target, "w") as archive: - for name, data in files.items(): - member = tarfile.TarInfo(name); member.size = len(data) - if change: - change(member) - archive.addfile(member, io.BytesIO(data) if member.isfile() else None) - return target - - def test_checked_archive_roundtrip(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp); dest = root / "dest"; dest.mkdir() - self.assertEqual(ops.unpack(self.bundle(root), dest)["format"], 1) - - def test_path_escape_rejected(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp); dest = root / "dest"; dest.mkdir() - def corrupt(member): - if member.name == "configuration.json": member.name = "../outside" - with self.assertRaisesRegex(RuntimeError, "Unexpected backup members"): - ops.unpack(self.bundle(root, corrupt), dest) - self.assertFalse((root / "outside").exists()) - - def test_symlink_rejected(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp); dest = root / "dest"; dest.mkdir() - def corrupt(member): - if member.name == "configuration.json": member.type = tarfile.SYMTYPE; member.linkname = "/etc/passwd"; member.size = 0 - with self.assertRaisesRegex(RuntimeError, "Invalid or oversized"): - ops.unpack(self.bundle(root, corrupt), dest) - - def test_checksum_mismatch_rejected(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp); dest = root / "dest"; dest.mkdir() - with patch.object(ops, "digest", return_value="changed"): - with self.assertRaisesRegex(RuntimeError, "checksum"): - ops.unpack(self.bundle(root), dest) - - def test_backup_requires_explicit_maintenance(self): - with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"): - ops.backup(type("Args", (), {"confirm_maintenance": False})()) - - def test_scheduled_backup_requires_explicit_maintenance(self): - with self.assertRaisesRegex(RuntimeError, "confirm-maintenance"): - ops.scheduled_backup(type("Args", (), {"confirm_maintenance": False})()) - - @unittest.skipUnless(os.name == "posix", "Linux file permission semantics") - def test_scheduled_backup_uses_private_external_directory_and_environment(self): - with tempfile.TemporaryDirectory() as parent: - directory = Path(parent) / "backups" - directory.mkdir(mode=0o700) - captured = [] - with patch.dict(os.environ, {"BACKUP_RECIPIENT": "A" * 40, "BACKUP_DIRECTORY": str(directory)}), \ - patch.object(ops, "ROOT", Path(parent) / "checkout"), \ - patch.object(ops, "backup", side_effect=lambda args: captured.append(args)), \ - patch.object(ops.time, "strftime", return_value="20260929T020000Z"): - ops.scheduled_backup(type("Args", (), {"confirm_maintenance": True})()) - self.assertEqual(captured[0].recipient, "A" * 40) - self.assertEqual(Path(captured[0].output), directory / "guestops-20260929T020000Z.tar.gpg") - self.assertTrue(captured[0].confirm_maintenance) - - @unittest.skipUnless(os.name == "posix", "Linux file permission semantics") - def test_scheduled_backup_rejects_checkout_directory(self): - with tempfile.TemporaryDirectory() as parent: - checkout = Path(parent) / "checkout" - directory = checkout / "backups" - directory.mkdir(parents=True, mode=0o700) - with patch.dict(os.environ, {"BACKUP_RECIPIENT": "A" * 40, "BACKUP_DIRECTORY": str(directory)}), \ - patch.object(ops, "ROOT", checkout): - with self.assertRaisesRegex(RuntimeError, "outside the application checkout"): - ops.scheduled_backup(type("Args", (), {"confirm_maintenance": True})()) - - @unittest.skipUnless(os.name == "posix", "Linux file permission semantics") - def test_dump_failure_restarts_services_and_ttl(self): - with tempfile.TemporaryDirectory() as temp: - calls = [] - def compose(*args, **kwargs): - calls.append(args) - if args[0] == "ps": return b"a" * 64 - if "sh" in args: raise RuntimeError("Simulated dump failure") - return b"" - def mongo(script): - calls.append((script,)) - if "getParameter" in script: return b"true" - if "storageSize" in script: return b'{"bytes":1,"collections":{}}' - return b"" - def run(args, **kwargs): - return b'[{"Image":"sha256:fixture"}]' if "inspect" in args else b"" - args = type("Args", (), {"confirm_maintenance": True, "recipient": "A" * 40, "output": str(Path(temp) / "backup.gpg")})() - with patch.object(ops, "configuration", return_value={}), patch.object(ops, "run", side_effect=run), patch.object(ops, "compose", side_effect=compose), patch.object(ops, "mongo", side_effect=mongo), patch.object(ops, "maintenance_lock", return_value=contextlib.nullcontext()): - with self.assertRaisesRegex(RuntimeError, "Simulated dump failure"): - ops.backup(args) - self.assertIn(("start", "api", "worker"), calls) - self.assertTrue(any("ttlMonitorEnabled:true" in call[0] for call in calls)) - self.assertFalse(Path(args.output).exists()) - - -if __name__ == "__main__": - unittest.main() - diff --git a/tests/test_package_source.py b/tests/test_package_source.py index 72d3d6e..ec83c78 100644 --- a/tests/test_package_source.py +++ b/tests/test_package_source.py @@ -1,4 +1,3 @@ -import hashlib import importlib.util import json from pathlib import Path @@ -44,20 +43,18 @@ class PackageSourceTests(unittest.TestCase): run(repository, "commit", "-m", "fixture") return repository, run(repository, "rev-parse", "HEAD") - def test_packages_only_the_selected_commit_and_records_identity(self): + def test_packages_only_the_selected_commit_with_version_name(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) repository, commit = self.repository(root) (repository / "application.txt").write_text("uncommitted change\n", encoding="utf-8") - archive, record_path, record = package_source.build_package(repository, commit, root / "output") + archive, details = package_source.build_package(repository, commit, root / "output") - self.assertEqual(record["commit"], commit) - self.assertEqual(record["version"], "1.2.3") - self.assertEqual(record["artifact"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) - self.assertEqual(json.loads(record_path.read_text(encoding="utf-8")), record) + self.assertEqual(archive.name, "guestops_1.2.3.tar.gz") + self.assertEqual(details, {"commit": commit, "version": "1.2.3"}) with tarfile.open(archive, "r:gz") as package: - member = package.extractfile("GuestOps-1.2.3/application.txt") + member = package.extractfile("guestops_1.2.3/application.txt") self.assertIsNotNone(member) self.assertEqual(member.read().decode("utf-8").strip(), "committed application") @@ -65,8 +62,8 @@ class PackageSourceTests(unittest.TestCase): with tempfile.TemporaryDirectory() as directory: root = Path(directory) repository, commit = self.repository(root) - first, _, _ = package_source.build_package(repository, commit, root / "first") - second, _, _ = package_source.build_package(repository, commit, root / "second") + first, _ = package_source.build_package(repository, commit, root / "first") + second, _ = package_source.build_package(repository, commit, root / "second") self.assertEqual(first.read_bytes(), second.read_bytes()) def test_rejects_mismatched_versions_and_existing_output(self): diff --git a/tests/test_pilot_run.py b/tests/test_pilot_run.py index 343ba41..5c43253 100644 --- a/tests/test_pilot_run.py +++ b/tests/test_pilot_run.py @@ -13,7 +13,7 @@ def valid_record(): return { "schemaVersion": 1, "system": "guestops-supervised-pilot", "targetGate": "B", "releaseCommit": "a" * 40, "releaseRecordSha256": "b" * 64, - "environment": "https://sandbox-guestops.futuresens.co.uk", "hotelLabel": "pilot-hotel-a", + "environment": "https://guestops-sandbox.futuresens.co.uk", "hotelLabel": "pilot-hotel-a", "hotelCount": 1, "plannedBusinessDays": 5, "owners": {"hotelOwner": "Hotel owner", "technicalOwner": "Technical owner", "rollbackDecisionMaker": "Technical owner"}, "startedOn": days[0], "endedOn": days[-1], diff --git a/tests/test_release_record.py b/tests/test_release_record.py deleted file mode 100644 index 378d7b7..0000000 --- a/tests/test_release_record.py +++ /dev/null @@ -1,82 +0,0 @@ -import hashlib -import json -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] - - -class ReleaseRecordTests(unittest.TestCase): - def test_writes_versions_checksum_and_immutable_image_ids(self): - with tempfile.TemporaryDirectory() as directory: - artifact = Path(directory) / "GuestOps-0.2.1.tar.gz" - output = Path(directory) / "release-record.json" - artifact.write_bytes(b"reviewed image archive") - - subprocess.run( - [ - sys.executable, - str(ROOT / "deploy" / "release_record.py"), - "--artifact", - str(artifact), - "--commit", - "a" * 40, - "--api-image", - "guestops-api:" + "a" * 40, - "--api-id", - "sha256:api", - "--worker-image", - "guestops-worker:" + "a" * 40, - "--worker-id", - "sha256:worker", - "--output", - str(output), - ], - check=True, - ) - - record = json.loads(output.read_text(encoding="utf-8")) - self.assertEqual(record["version"], "0.2.1") - self.assertEqual(record["commit"], "a" * 40) - self.assertEqual(record["images"]["api"]["id"], "sha256:api") - self.assertEqual( - record["artifact"]["sha256"], - hashlib.sha256(artifact.read_bytes()).hexdigest(), - ) - - def test_rejects_abbreviated_commit(self): - with tempfile.TemporaryDirectory() as directory: - artifact = Path(directory) / "images.tar.gz" - artifact.write_bytes(b"fixture") - result = subprocess.run( - [ - sys.executable, - str(ROOT / "deploy" / "release_record.py"), - "--artifact", - str(artifact), - "--commit", - "abc123", - "--api-image", - "api:test", - "--api-id", - "sha256:api", - "--worker-image", - "worker:test", - "--worker-id", - "sha256:worker", - "--output", - str(Path(directory) / "record.json"), - ], - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("full 40-character Git SHA", result.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_verify_release.py b/tests/test_verify_release.py deleted file mode 100644 index 4b286e9..0000000 --- a/tests/test_verify_release.py +++ /dev/null @@ -1,87 +0,0 @@ -import hashlib -import importlib.util -import json -from pathlib import Path -import tempfile -import unittest -from unittest.mock import patch - - -ROOT = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location("verify_release", ROOT / "deploy" / "verify_release.py") -verify_release = importlib.util.module_from_spec(spec) -spec.loader.exec_module(verify_release) - - -COMMIT = "a" * 40 -API_ID = "sha256:" + "b" * 64 -WORKER_ID = "sha256:" + "c" * 64 - - -class VerifyReleaseTests(unittest.TestCase): - def fixture(self, directory: str): - root = Path(directory) - archive = root / "GuestOps-0.2.1.tar.gz" - record = root / "release-record.json" - archive.write_bytes(b"reviewed image archive") - release = { - "schemaVersion": 1, - "version": "0.2.1", - "commit": COMMIT, - "artifact": { - "name": archive.name, - "size": archive.stat().st_size, - "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), - }, - "images": { - "api": {"reference": f"guestops-api:{COMMIT}", "id": API_ID}, - "worker": {"reference": f"guestops-worker:{COMMIT}", "id": WORKER_ID}, - }, - } - record.write_text(json.dumps(release), encoding="utf-8") - return archive, record, release - - def test_verifies_candidate_archive_record_and_record_checksum(self): - with tempfile.TemporaryDirectory() as directory: - archive, record, _ = self.fixture(directory) - result = verify_release.validate(archive, record, COMMIT, "0.2.1") - self.assertTrue(result["verified"]) - self.assertEqual(result["archive"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) - self.assertEqual(result["releaseRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest()) - self.assertFalse(result["loadedImageIdsVerified"]) - - def test_rejects_changed_archive(self): - with tempfile.TemporaryDirectory() as directory: - archive, record, _ = self.fixture(directory) - archive.write_bytes(b"changed") - with self.assertRaisesRegex(ValueError, "size does not match"): - verify_release.validate(archive, record, COMMIT, "0.2.1") - - def test_rejects_wrong_commit_version_reference_and_mutable_id(self): - cases = [ - (lambda value: value.update(commit="d" * 40), "commit does not match"), - (lambda value: value.update(version="0.3.0"), "version does not match"), - (lambda value: value["images"]["api"].update(reference="guestops-api:latest"), "full candidate SHA"), - (lambda value: value["images"]["worker"].update(id="worker-image"), "immutable SHA-256"), - ] - for mutate, message in cases: - with self.subTest(message=message), tempfile.TemporaryDirectory() as directory: - archive, record, release = self.fixture(directory) - mutate(release) - record.write_text(json.dumps(release), encoding="utf-8") - with self.assertRaisesRegex(ValueError, message): - verify_release.validate(archive, record, COMMIT, "0.2.1") - - def test_loaded_image_ids_must_match(self): - with tempfile.TemporaryDirectory() as directory: - archive, record, _ = self.fixture(directory) - with patch.object(verify_release, "loaded_image_id", side_effect=[API_ID, WORKER_ID]): - result = verify_release.validate(archive, record, COMMIT, "0.2.1", True) - self.assertTrue(result["loadedImageIdsVerified"]) - with patch.object(verify_release, "loaded_image_id", return_value="sha256:" + "d" * 64): - with self.assertRaisesRegex(ValueError, "Loaded api image ID"): - verify_release.validate(archive, record, COMMIT, "0.2.1", True) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_verify_source_package.py b/tests/test_verify_source_package.py deleted file mode 100644 index 5168418..0000000 --- a/tests/test_verify_source_package.py +++ /dev/null @@ -1,72 +0,0 @@ -import hashlib -import importlib.util -import json -from pathlib import Path -import tempfile -import unittest - - -ROOT = Path(__file__).resolve().parents[1] -spec = importlib.util.spec_from_file_location( - "verify_source_package", ROOT / "deploy" / "verify_source_package.py" -) -verify_source_package = importlib.util.module_from_spec(spec) -spec.loader.exec_module(verify_source_package) - - -COMMIT = "a" * 40 -VERSION = "0.2.1" - - -class VerifySourcePackageTests(unittest.TestCase): - def fixture(self, directory: str): - root = Path(directory) - archive = root / "GuestOps-0.2.1-aaaaaaaaaaaa.tar.gz" - record = root / "GuestOps-0.2.1-aaaaaaaaaaaa.source.json" - archive.write_bytes(b"committed source package") - payload = { - "artifact": { - "name": archive.name, - "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), - "size": archive.stat().st_size, - }, - "commit": COMMIT, - "schemaVersion": 1, - "version": VERSION, - } - record.write_text(json.dumps(payload), encoding="utf-8") - return archive, record, payload - - def test_verifies_archive_record_and_expected_identity(self): - with tempfile.TemporaryDirectory() as directory: - archive, record, _ = self.fixture(directory) - result = verify_source_package.validate(archive, record, COMMIT, VERSION) - self.assertTrue(result["verified"]) - self.assertEqual(result["commit"], COMMIT) - self.assertEqual(result["artifact"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest()) - self.assertEqual(result["sourceRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest()) - - def test_rejects_changed_archive(self): - with tempfile.TemporaryDirectory() as directory: - archive, record, _ = self.fixture(directory) - archive.write_bytes(b"changed source package") - with self.assertRaisesRegex(ValueError, "size does not match"): - verify_source_package.validate(archive, record, COMMIT, VERSION) - - def test_rejects_wrong_commit_version_and_filename(self): - cases = [ - (lambda value: value.update(commit="b" * 40), "commit does not match"), - (lambda value: value.update(version="0.3.0"), "version does not match"), - (lambda value: value["artifact"].update(name="other.tar.gz"), "filename does not match"), - ] - for mutate, message in cases: - with self.subTest(message=message), tempfile.TemporaryDirectory() as directory: - archive, record, payload = self.fixture(directory) - mutate(payload) - record.write_text(json.dumps(payload), encoding="utf-8") - with self.assertRaisesRegex(ValueError, message): - verify_source_package.validate(archive, record, COMMIT, VERSION) - - -if __name__ == "__main__": - unittest.main() -- 2.20.1