milestone 9 completed

deploy/verify_release.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Verify a GuestOps release archive and its immutable CI release record."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import subprocess
+
+
+SHA = re.compile(r"[0-9a-f]{40}")
+DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
+
+
+def require(condition: bool, message: str) -> None:
+    if not condition:
+        raise ValueError(message)
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def loaded_image_id(reference: str) -> str:
+    result = subprocess.run(
+        ["docker", "image", "inspect", "--format", "{{.Id}}", reference],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout.strip()
+
+
+def validate(
+    archive: Path,
+    record_path: Path,
+    expected_commit: str,
+    expected_version: str,
+    verify_loaded_images: bool = False,
+) -> dict[str, object]:
+    require(archive.is_file(), "Release archive does not exist.")
+    require(record_path.is_file(), "Release record does not exist.")
+    require(SHA.fullmatch(expected_commit) is not None, "Expected commit must be a full lowercase Git SHA.")
+    require(re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", expected_version) is not None,
+            "Expected version must use MAJOR.MINOR.PATCH.")
+
+    record = json.loads(record_path.read_text(encoding="utf-8"))
+    require(isinstance(record, dict), "Release record must be a JSON object.")
+    require(record.get("schemaVersion") == 1, "Unsupported release-record schema.")
+    require(record.get("commit") == expected_commit, "Release-record commit does not match the approved candidate.")
+    require(record.get("version") == expected_version, "Release-record version does not match the approved version.")
+
+    artifact = record.get("artifact")
+    require(isinstance(artifact, dict), "Release record has no artifact object.")
+    require(artifact.get("name") == archive.name, "Release archive filename does not match the record.")
+    require(artifact.get("size") == archive.stat().st_size, "Release archive size does not match the record.")
+    archive_sha = sha256(archive)
+    require(artifact.get("sha256") == archive_sha, "Release archive SHA-256 does not match the record.")
+
+    images = record.get("images")
+    require(isinstance(images, dict) and set(images) == {"api", "worker"},
+            "Release record must contain exactly API and worker images.")
+    expected_references = {
+        "api": f"guestops-api:{expected_commit}",
+        "worker": f"guestops-worker:{expected_commit}",
+    }
+    verified_images: dict[str, dict[str, str]] = {}
+    for name, reference in expected_references.items():
+        image = images.get(name)
+        require(isinstance(image, dict), f"Release record has no {name} image object.")
+        require(image.get("reference") == reference, f"{name} image reference is not bound to the full candidate SHA.")
+        image_id = str(image.get("id", ""))
+        require(DIGEST.fullmatch(image_id) is not None, f"{name} image ID is not an immutable SHA-256 digest.")
+        if verify_loaded_images:
+            require(loaded_image_id(reference) == image_id, f"Loaded {name} image ID does not match the release record.")
+        verified_images[name] = {"reference": reference, "id": image_id}
+
+    return {
+        "schemaVersion": 1,
+        "verified": True,
+        "commit": expected_commit,
+        "version": expected_version,
+        "archive": {"name": archive.name, "size": archive.stat().st_size, "sha256": archive_sha},
+        "releaseRecord": {"name": record_path.name, "sha256": sha256(record_path)},
+        "images": verified_images,
+        "loadedImageIdsVerified": verify_loaded_images,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--archive", required=True, type=Path)
+    parser.add_argument("--record", required=True, type=Path)
+    parser.add_argument("--commit", required=True)
+    parser.add_argument("--version", required=True)
+    parser.add_argument("--verify-loaded-images", action="store_true")
+    parser.add_argument("--output", type=Path, help="Optional path for the non-sensitive verification summary.")
+    args = parser.parse_args()
+    result = validate(args.archive, args.record, args.commit, args.version, args.verify_loaded_images)
+    rendered = json.dumps(result, indent=2, sort_keys=True) + "\n"
+    if args.output:
+        args.output.write_text(rendered, encoding="utf-8")
+    print(rendered, end="")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (OSError, ValueError, json.JSONDecodeError, subprocess.SubprocessError) as error:
+        print(f"Release verification failed: {error}", file=__import__("sys").stderr)
+        raise SystemExit(1)

docs/operations.md

@@ -18,6 +18,28 @@ print(r['commit'], r['version'], r['images'])
 PY
 ```
 
+The repository verifier performs the same checks strictly, also calculates the release-record checksum used by later acceptance records, and can compare the record with images already loaded on an isolated Docker host:
+
+```sh
+python3 deploy/verify_release.py \
+  --archive guestops-images.tar.gz \
+  --record release-record.json \
+  --commit a3ef408de61d895e69516fa3ba014b43621032bc \
+  --version 0.2.0 \
+  --output release-verification.json
+
+gunzip -c guestops-images.tar.gz | docker load
+python3 deploy/verify_release.py \
+  --archive guestops-images.tar.gz \
+  --record release-record.json \
+  --commit a3ef408de61d895e69516fa3ba014b43621032bc \
+  --version 0.2.0 \
+  --verify-loaded-images \
+  --output loaded-image-verification.json
+```
+
+Retain both verification summaries with the untouched archive, release record, its SHA-256, and the exact default-branch CI metadata. A waiting, cancelled, failed, or branch-only run is not release evidence. Do not substitute a local rebuild for the archived default-branch images.
+
 Load the archive, verify each loaded image ID matches the record, set `GUESTOPS_API_IMAGE` and `GUESTOPS_WORKER_IMAGE` to the recorded full-SHA references, and run the deployment preflight. Record the CI run, commit, checksum and operator in the change ticket. A release tag is an approval marker; do not move or reuse an existing tag. The application and web versions must match before the record can be created.
 
 For rollback, first disable worker-driven external writes and reconcile any sending, payment or PMS operation that may have completed since the prior release. Confirm the previous release archive and record are retained, verify its checksum and image IDs, take an encrypted backup, then select the previous recorded image references in `.env` and recreate only the API and worker. Do not roll back MongoDB or the key volume merely to change application images. Run the online preflight, readiness check and read-only smoke test before re-enabling the worker or provider writes. If a release introduced an incompatible data change, follow its release-specific recovery plan rather than starting an older image against newer data.

tests/test_verify_release.py

@@ -0,0 +1,87 @@
+import hashlib
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("verify_release", ROOT / "deploy" / "verify_release.py")
+verify_release = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(verify_release)
+
+
+COMMIT = "a" * 40
+API_ID = "sha256:" + "b" * 64
+WORKER_ID = "sha256:" + "c" * 64
+
+
+class VerifyReleaseTests(unittest.TestCase):
+    def fixture(self, directory: str):
+        root = Path(directory)
+        archive = root / "guestops-images.tar.gz"
+        record = root / "release-record.json"
+        archive.write_bytes(b"reviewed image archive")
+        release = {
+            "schemaVersion": 1,
+            "version": "0.2.0",
+            "commit": COMMIT,
+            "artifact": {
+                "name": archive.name,
+                "size": archive.stat().st_size,
+                "sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
+            },
+            "images": {
+                "api": {"reference": f"guestops-api:{COMMIT}", "id": API_ID},
+                "worker": {"reference": f"guestops-worker:{COMMIT}", "id": WORKER_ID},
+            },
+        }
+        record.write_text(json.dumps(release), encoding="utf-8")
+        return archive, record, release
+
+    def test_verifies_candidate_archive_record_and_record_checksum(self):
+        with tempfile.TemporaryDirectory() as directory:
+            archive, record, _ = self.fixture(directory)
+            result = verify_release.validate(archive, record, COMMIT, "0.2.0")
+            self.assertTrue(result["verified"])
+            self.assertEqual(result["archive"]["sha256"], hashlib.sha256(archive.read_bytes()).hexdigest())
+            self.assertEqual(result["releaseRecord"]["sha256"], hashlib.sha256(record.read_bytes()).hexdigest())
+            self.assertFalse(result["loadedImageIdsVerified"])
+
+    def test_rejects_changed_archive(self):
+        with tempfile.TemporaryDirectory() as directory:
+            archive, record, _ = self.fixture(directory)
+            archive.write_bytes(b"changed")
+            with self.assertRaisesRegex(ValueError, "size does not match"):
+                verify_release.validate(archive, record, COMMIT, "0.2.0")
+
+    def test_rejects_wrong_commit_version_reference_and_mutable_id(self):
+        cases = [
+            (lambda value: value.update(commit="d" * 40), "commit does not match"),
+            (lambda value: value.update(version="0.3.0"), "version does not match"),
+            (lambda value: value["images"]["api"].update(reference="guestops-api:latest"), "full candidate SHA"),
+            (lambda value: value["images"]["worker"].update(id="worker-image"), "immutable SHA-256"),
+        ]
+        for mutate, message in cases:
+            with self.subTest(message=message), tempfile.TemporaryDirectory() as directory:
+                archive, record, release = self.fixture(directory)
+                mutate(release)
+                record.write_text(json.dumps(release), encoding="utf-8")
+                with self.assertRaisesRegex(ValueError, message):
+                    verify_release.validate(archive, record, COMMIT, "0.2.0")
+
+    def test_loaded_image_ids_must_match(self):
+        with tempfile.TemporaryDirectory() as directory:
+            archive, record, _ = self.fixture(directory)
+            with patch.object(verify_release, "loaded_image_id", side_effect=[API_ID, WORKER_ID]):
+                result = verify_release.validate(archive, record, COMMIT, "0.2.0", True)
+            self.assertTrue(result["loadedImageIdsVerified"])
+            with patch.object(verify_release, "loaded_image_id", return_value="sha256:" + "d" * 64):
+                with self.assertRaisesRegex(ValueError, "Loaded api image ID"):
+                    verify_release.validate(archive, record, COMMIT, "0.2.0", True)
+
+
+if __name__ == "__main__":
+    unittest.main()