Gate B release candidate
Build and verify web migration / verify (push) Has been cancelled

This commit is contained in:
wolf-demon
2026-09-30 15:17:34 +01:00
parent f11a21aae8
commit c41b937acb
21 changed files with 1573 additions and 2 deletions
+129
View File
@@ -0,0 +1,129 @@
import importlib.util
from pathlib import Path
import unittest
spec = importlib.util.spec_from_file_location(
"backup_restore_acceptance",
Path(__file__).resolve().parents[1] / "deploy" / "backup_restore_acceptance.py")
acceptance = importlib.util.module_from_spec(spec)
spec.loader.exec_module(acceptance)
COMMIT = "a" * 40
RELEASE_SHA = "b" * 64
def valid_record():
return {
"schemaVersion": 1,
"system": "guestops-backup-recovery",
"evidenceId": "backup-restore",
"dataClassification": "synthetic-only",
"releaseVersion": "0.2.0",
"releaseCommit": COMMIT,
"releaseRecordSha256": RELEASE_SHA,
"archiveSha256": "c" * 64,
"environment": "https://sandbox-guestops.futuresens.co.uk",
"hostIdentifier": "guestops-sandbox-01",
"images": {
"api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64},
"worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64},
"mongo": {"reference": "mongo:8.0", "id": "sha256:" + "f" * 64},
},
"owners": {
"backup": "Backup owner",
"monitoring": "Monitoring owner",
"recoveryOperator": "Recovery operator",
"technicalEscalation": "Technical owner",
"retention": "Retention owner",
"independentReviewer": "Independent reviewer",
},
"startedAt": "2026-10-01T09:00:00Z",
"endedAt": "2026-10-01T12:00:00Z",
"reviewedAt": "2026-10-01T13:00:00Z",
"recoveryObjectives": {
"targetRpoHours": 24, "observedRpoHours": 1,
"targetRtoMinutes": 240, "observedRtoMinutes": 180,
},
"retention": {
"localVerifiedDays": 7, "offHostDaily": 35,
"offHostMonthly": 12, "legalHoldOverrideTested": True,
},
"backup": {
"createdAt": "2026-10-01T08:00:00Z",
"sha256": "1" * 64,
"transferredSha256": "1" * 64,
"privateKeyPresentOnHost": False,
},
"rollback": {
"previousReleaseCommit": "2" * 40,
"previousArchiveSha256": "3" * 64,
"previousImages": {
"api": {"reference": "guestops-api:" + "2" * 40, "id": "sha256:" + "4" * 64},
"worker": {"reference": "guestops-worker:" + "2" * 40, "id": "sha256:" + "5" * 64},
},
"persistentVolumesReplaced": False,
"restoredReleaseCommit": COMMIT,
"unresolvedOperations": 0,
"finalControls": {
"googleSending": "disabled", "faqLiveMode": "disabled",
"pmsWrites": "disabled", "paymentCreation": "disabled",
},
},
"monitoringState": "healthy",
"unresolvedCriticalFindings": 0,
"scenarios": [
{"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]}
for index, scenario in enumerate(sorted(acceptance.SCENARIOS), 1)
],
}
class BackupRestoreAcceptanceTests(unittest.TestCase):
def test_complete_record_passes(self):
acceptance.validate(valid_record(), COMMIT, RELEASE_SHA)
def test_release_identity_and_images_are_bound(self):
record = valid_record(); record["releaseCommit"] = "9" * 40
with self.assertRaisesRegex(ValueError, "approved candidate"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["images"]["api"]["reference"] = "guestops-api:latest"
with self.assertRaisesRegex(ValueError, "full approved commit"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["rollback"]["previousImages"]["worker"]["id"] = "mutable"
with self.assertRaisesRegex(ValueError, "retained previous release identity"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
def test_recovery_objectives_and_checksums_must_pass(self):
record = valid_record(); record["recoveryObjectives"]["observedRtoMinutes"] = 241
with self.assertRaisesRegex(ValueError, "four-hour"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["backup"]["transferredSha256"] = "9" * 64
with self.assertRaisesRegex(ValueError, "checksums must match"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["recoveryObjectives"]["observedRpoHours"] = 2
with self.assertRaisesRegex(ValueError, "match the backup"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
def test_independent_review_retention_and_final_state_are_required(self):
record = valid_record(); record["owners"]["independentReviewer"] = record["owners"]["backup"]
with self.assertRaisesRegex(ValueError, "different"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["retention"]["legalHoldOverrideTested"] = False
with self.assertRaisesRegex(ValueError, "Retention"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["rollback"]["persistentVolumesReplaced"] = True
with self.assertRaisesRegex(ValueError, "must not replace"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
def test_exact_passed_scenarios_and_monitoring_are_required(self):
record = valid_record(); record["scenarios"].pop()
with self.assertRaisesRegex(ValueError, "exact backup/recovery scenario"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
record = valid_record(); record["monitoringState"] = "degraded"
with self.assertRaisesRegex(ValueError, "Monitoring must be healthy"):
acceptance.validate(record, COMMIT, RELEASE_SHA)
if __name__ == "__main__":
unittest.main()
+81
View File
@@ -0,0 +1,81 @@
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import time
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location(
"backup_transfer", Path(__file__).resolve().parents[1] / "deploy" / "backup_transfer.py")
transfer = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transfer)
class BackupTransferTests(unittest.TestCase):
def backup(self, directory: Path, name="guestops-20261001T021700Z.tar.gpg") -> Path:
path = directory / name
path.write_bytes(b"encrypted-backup")
return path
def config(self, directory: Path):
return {
"directory": directory, "host": "store.example.invalid", "user": "guestops_upload",
"remote": "/restricted/guestops", "identity": Path("/safe/key"),
"known_hosts": Path("/safe/known_hosts"),
}
def test_existing_matching_remote_is_marked_without_upload(self):
with tempfile.TemporaryDirectory() as folder:
backup = self.backup(Path(folder)); checksum = transfer.digest(backup)
with patch.object(transfer, "remote_digest", return_value=checksum), \
patch.object(transfer, "run") as run:
transfer.transfer_one(self.config(Path(folder)), backup)
run.assert_not_called()
marker = json.loads(transfer.marker_path(backup).read_text(encoding="utf-8"))
self.assertEqual(marker["sha256"], checksum)
def test_existing_mismatched_remote_is_never_overwritten(self):
with tempfile.TemporaryDirectory() as folder:
backup = self.backup(Path(folder))
with patch.object(transfer, "remote_digest", return_value="9" * 64), \
patch.object(transfer, "run") as run:
with self.assertRaisesRegex(RuntimeError, "different checksum"):
transfer.transfer_one(self.config(Path(folder)), backup)
run.assert_not_called()
self.assertFalse(transfer.marker_path(backup).exists())
def test_new_remote_is_uploaded_verified_and_marked(self):
with tempfile.TemporaryDirectory() as folder:
backup = self.backup(Path(folder)); checksum = transfer.digest(backup)
with patch.object(transfer, "remote_digest", side_effect=[None, checksum, checksum]), \
patch.object(transfer, "run", return_value=b"") as run, \
patch.object(transfer.subprocess, "run"):
transfer.transfer_one(self.config(Path(folder)), backup)
self.assertTrue(any(call.args[0][0] == "rsync" for call in run.call_args_list))
self.assertTrue(transfer.marker_path(backup).exists())
def test_prune_removes_only_old_checksum_verified_backups(self):
with tempfile.TemporaryDirectory() as folder:
directory = Path(folder)
verified = self.backup(directory)
checksum = transfer.digest(verified)
transfer.write_marker(verified, checksum)
unverified = self.backup(directory, "guestops-20261002T021700Z.tar.gpg")
old = time.time() - 8 * 86400
os.utime(verified, (old, old)); os.utime(unverified, (old, old))
removed = transfer.prune_verified(self.config(directory), 7, now=time.time())
self.assertEqual(removed, 1)
self.assertFalse(verified.exists())
self.assertTrue(unverified.exists())
def test_retention_bounds_are_enforced(self):
with tempfile.TemporaryDirectory() as folder:
with self.assertRaisesRegex(RuntimeError, "between 1 and 365"):
transfer.prune_verified(self.config(Path(folder)), 0)
if __name__ == "__main__":
unittest.main()
+113
View File
@@ -0,0 +1,113 @@
import importlib.util
from pathlib import Path
import unittest
spec = importlib.util.spec_from_file_location(
"debian_acceptance", Path(__file__).resolve().parents[1] / "deploy" / "debian_acceptance.py")
acceptance = importlib.util.module_from_spec(spec)
spec.loader.exec_module(acceptance)
COMMIT = "a" * 40
RELEASE_SHA = "b" * 64
def common(system):
return {
"schemaVersion": 1,
"system": system,
"evidenceId": acceptance.SYSTEMS[system]["evidenceId"],
"releaseVersion": "0.2.0",
"releaseCommit": COMMIT,
"releaseRecordSha256": RELEASE_SHA,
"archiveSha256": "c" * 64,
"environment": "https://sandbox-guestops.futuresens.co.uk",
"hostIdentifier": "guestops-sandbox-01",
"images": {
"api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64},
"worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64},
},
"operator": "Deployment operator",
"reviewedBy": "Independent reviewer",
"startedAt": "2026-09-30T09:00:00Z",
"endedAt": "2026-09-30T10:00:00Z",
"reviewedAt": "2026-09-30T11:00:00Z",
"unresolvedCriticalFindings": 0,
"scenarios": [
{"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]}
for index, scenario in enumerate(sorted(acceptance.SYSTEMS[system]["scenarios"]), 1)
],
}
def valid_records():
host = common("guestops-debian-host")
host["hostFacts"] = {
"debianMajor": 12,
"cpuCores": 4,
"memoryBytes": 8_140_382_208,
"freeDiskBytes": 14_275_686_400,
"publicTcpPorts": [80, 443],
}
host["featureControls"] = {
"googleSending": "disabled",
"faqLiveMode": "disabled",
"pmsWrites": "disabled",
"paymentCreation": "disabled",
}
persistence = common("guestops-persistence")
persistence["drillCommand"] = "python3 deploy/ops.py persistence-drill --confirm-restart"
return host, persistence
class DebianAcceptanceTests(unittest.TestCase):
def test_complete_matching_records_pass(self):
acceptance.validate_pair(*valid_records(), COMMIT, RELEASE_SHA)
def test_expected_release_identity_is_required(self):
host, persistence = valid_records()
host["releaseCommit"] = "f" * 40
with self.assertRaisesRegex(ValueError, "approved candidate"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
host, persistence = valid_records()
persistence["releaseRecordSha256"] = "f" * 64
with self.assertRaisesRegex(ValueError, "retained release record"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
def test_exact_images_and_cross_record_identity_are_required(self):
host, persistence = valid_records()
host["images"]["api"]["reference"] = "guestops-api:latest"
with self.assertRaisesRegex(ValueError, "full approved commit"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
host, persistence = valid_records()
persistence["archiveSha256"] = "f" * 64
with self.assertRaisesRegex(ValueError, "same archiveSha256"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
def test_host_capacity_ports_and_disabled_controls_are_required(self):
host, persistence = valid_records()
host["hostFacts"]["publicTcpPorts"] = [80, 443, 8080]
with self.assertRaisesRegex(ValueError, "Only TCP ports"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
host, persistence = valid_records()
host["featureControls"]["googleSending"] = "enabled"
with self.assertRaisesRegex(ValueError, "must remain disabled"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
def test_independent_review_scenarios_and_findings_are_required(self):
host, persistence = valid_records()
host["reviewedBy"] = host["operator"]
with self.assertRaisesRegex(ValueError, "different people"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
host, persistence = valid_records()
persistence["scenarios"][0]["status"] = "not-run"
with self.assertRaisesRegex(ValueError, "has not passed"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
host, persistence = valid_records()
host["unresolvedCriticalFindings"] = 1
with self.assertRaisesRegex(ValueError, "critical findings"):
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
if __name__ == "__main__":
unittest.main()
+75
View File
@@ -0,0 +1,75 @@
import datetime as dt
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location(
"monitor_status", Path(__file__).resolve().parents[1] / "deploy" / "monitor_status.py")
monitor = importlib.util.module_from_spec(spec)
spec.loader.exec_module(monitor)
class MonitorStatusTests(unittest.TestCase):
def test_compose_json_is_reduced_to_safe_state(self):
value = json.dumps([
{"Service": "api", "State": "running", "Health": "healthy", "Publishers": "secret"},
{"Service": "worker", "State": "running", "Health": ""},
{"Service": "mongo", "State": "running", "Health": "healthy"},
]).encode()
self.assertEqual(monitor.parse_compose(value)["api"], {"state": "running", "health": "healthy"})
self.assertNotIn("Publishers", monitor.parse_compose(value)["api"])
def test_age_ignores_symlinks_and_unexpected_files(self):
with tempfile.TemporaryDirectory() as folder:
directory = Path(folder)
backup = directory / "guestops-20261001T021700Z.tar.gpg"
backup.write_bytes(b"fixture")
moment = dt.datetime(2026, 10, 1, 3, 17, tzinfo=dt.timezone.utc)
os.utime(backup, (moment.timestamp() - 3600, moment.timestamp() - 3600))
(directory / "secret.txt").write_text("ignored", encoding="utf-8")
self.assertEqual(monitor.age_seconds(directory, monitor.BACKUP_NAME, moment), 3600)
def test_build_status_contains_only_aggregate_health(self):
now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc)
compose = json.dumps([
{"Service": name, "State": "running", "Health": "healthy"}
for name in ("api", "worker", "mongo")
]).encode()
usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})()
with tempfile.TemporaryDirectory() as folder, \
patch.object(monitor, "https_status", return_value={"ready": True, "certificateDaysRemaining": 60}), \
patch.object(monitor, "run", return_value=compose), \
patch.object(monitor, "worker_heartbeat_age", return_value=30), \
patch.object(monitor.shutil, "disk_usage", return_value=usage):
status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now)
self.assertEqual(errors, [])
self.assertEqual(status["workerHeartbeatAgeSeconds"], 30)
self.assertEqual(status["diskFreePercent"], 50)
self.assertNotIn("credentials", json.dumps(status).lower())
def test_failed_probes_write_categories_not_exception_details(self):
now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc)
usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})()
with tempfile.TemporaryDirectory() as folder, \
patch.object(monitor, "https_status", side_effect=RuntimeError("secret value")), \
patch.object(monitor, "run", side_effect=RuntimeError("secret value")), \
patch.object(monitor, "worker_heartbeat_age", side_effect=RuntimeError("secret value")), \
patch.object(monitor.shutil, "disk_usage", return_value=usage):
status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now)
self.assertGreaterEqual(len(errors), 3)
self.assertNotIn("secret value", json.dumps(status))
def test_status_output_is_atomic_json(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / "status.json"
monitor.write_status(path, {"schemaVersion": 1, "errors": []})
self.assertEqual(json.loads(path.read_text(encoding="utf-8"))["schemaVersion"], 1)
if __name__ == "__main__":
unittest.main()