This commit is contained in:
@@ -0,0 +1,129 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"backup_restore_acceptance",
|
||||
Path(__file__).resolve().parents[1] / "deploy" / "backup_restore_acceptance.py")
|
||||
acceptance = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(acceptance)
|
||||
|
||||
COMMIT = "a" * 40
|
||||
RELEASE_SHA = "b" * 64
|
||||
|
||||
|
||||
def valid_record():
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"system": "guestops-backup-recovery",
|
||||
"evidenceId": "backup-restore",
|
||||
"dataClassification": "synthetic-only",
|
||||
"releaseVersion": "0.2.0",
|
||||
"releaseCommit": COMMIT,
|
||||
"releaseRecordSha256": RELEASE_SHA,
|
||||
"archiveSha256": "c" * 64,
|
||||
"environment": "https://sandbox-guestops.futuresens.co.uk",
|
||||
"hostIdentifier": "guestops-sandbox-01",
|
||||
"images": {
|
||||
"api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64},
|
||||
"worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64},
|
||||
"mongo": {"reference": "mongo:8.0", "id": "sha256:" + "f" * 64},
|
||||
},
|
||||
"owners": {
|
||||
"backup": "Backup owner",
|
||||
"monitoring": "Monitoring owner",
|
||||
"recoveryOperator": "Recovery operator",
|
||||
"technicalEscalation": "Technical owner",
|
||||
"retention": "Retention owner",
|
||||
"independentReviewer": "Independent reviewer",
|
||||
},
|
||||
"startedAt": "2026-10-01T09:00:00Z",
|
||||
"endedAt": "2026-10-01T12:00:00Z",
|
||||
"reviewedAt": "2026-10-01T13:00:00Z",
|
||||
"recoveryObjectives": {
|
||||
"targetRpoHours": 24, "observedRpoHours": 1,
|
||||
"targetRtoMinutes": 240, "observedRtoMinutes": 180,
|
||||
},
|
||||
"retention": {
|
||||
"localVerifiedDays": 7, "offHostDaily": 35,
|
||||
"offHostMonthly": 12, "legalHoldOverrideTested": True,
|
||||
},
|
||||
"backup": {
|
||||
"createdAt": "2026-10-01T08:00:00Z",
|
||||
"sha256": "1" * 64,
|
||||
"transferredSha256": "1" * 64,
|
||||
"privateKeyPresentOnHost": False,
|
||||
},
|
||||
"rollback": {
|
||||
"previousReleaseCommit": "2" * 40,
|
||||
"previousArchiveSha256": "3" * 64,
|
||||
"previousImages": {
|
||||
"api": {"reference": "guestops-api:" + "2" * 40, "id": "sha256:" + "4" * 64},
|
||||
"worker": {"reference": "guestops-worker:" + "2" * 40, "id": "sha256:" + "5" * 64},
|
||||
},
|
||||
"persistentVolumesReplaced": False,
|
||||
"restoredReleaseCommit": COMMIT,
|
||||
"unresolvedOperations": 0,
|
||||
"finalControls": {
|
||||
"googleSending": "disabled", "faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled", "paymentCreation": "disabled",
|
||||
},
|
||||
},
|
||||
"monitoringState": "healthy",
|
||||
"unresolvedCriticalFindings": 0,
|
||||
"scenarios": [
|
||||
{"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]}
|
||||
for index, scenario in enumerate(sorted(acceptance.SCENARIOS), 1)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class BackupRestoreAcceptanceTests(unittest.TestCase):
|
||||
def test_complete_record_passes(self):
|
||||
acceptance.validate(valid_record(), COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_release_identity_and_images_are_bound(self):
|
||||
record = valid_record(); record["releaseCommit"] = "9" * 40
|
||||
with self.assertRaisesRegex(ValueError, "approved candidate"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["images"]["api"]["reference"] = "guestops-api:latest"
|
||||
with self.assertRaisesRegex(ValueError, "full approved commit"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["rollback"]["previousImages"]["worker"]["id"] = "mutable"
|
||||
with self.assertRaisesRegex(ValueError, "retained previous release identity"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_recovery_objectives_and_checksums_must_pass(self):
|
||||
record = valid_record(); record["recoveryObjectives"]["observedRtoMinutes"] = 241
|
||||
with self.assertRaisesRegex(ValueError, "four-hour"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["backup"]["transferredSha256"] = "9" * 64
|
||||
with self.assertRaisesRegex(ValueError, "checksums must match"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["recoveryObjectives"]["observedRpoHours"] = 2
|
||||
with self.assertRaisesRegex(ValueError, "match the backup"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_independent_review_retention_and_final_state_are_required(self):
|
||||
record = valid_record(); record["owners"]["independentReviewer"] = record["owners"]["backup"]
|
||||
with self.assertRaisesRegex(ValueError, "different"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["retention"]["legalHoldOverrideTested"] = False
|
||||
with self.assertRaisesRegex(ValueError, "Retention"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["rollback"]["persistentVolumesReplaced"] = True
|
||||
with self.assertRaisesRegex(ValueError, "must not replace"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_exact_passed_scenarios_and_monitoring_are_required(self):
|
||||
record = valid_record(); record["scenarios"].pop()
|
||||
with self.assertRaisesRegex(ValueError, "exact backup/recovery scenario"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
record = valid_record(); record["monitoringState"] = "degraded"
|
||||
with self.assertRaisesRegex(ValueError, "Monitoring must be healthy"):
|
||||
acceptance.validate(record, COMMIT, RELEASE_SHA)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,81 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"backup_transfer", Path(__file__).resolve().parents[1] / "deploy" / "backup_transfer.py")
|
||||
transfer = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(transfer)
|
||||
|
||||
|
||||
class BackupTransferTests(unittest.TestCase):
|
||||
def backup(self, directory: Path, name="guestops-20261001T021700Z.tar.gpg") -> Path:
|
||||
path = directory / name
|
||||
path.write_bytes(b"encrypted-backup")
|
||||
return path
|
||||
|
||||
def config(self, directory: Path):
|
||||
return {
|
||||
"directory": directory, "host": "store.example.invalid", "user": "guestops_upload",
|
||||
"remote": "/restricted/guestops", "identity": Path("/safe/key"),
|
||||
"known_hosts": Path("/safe/known_hosts"),
|
||||
}
|
||||
|
||||
def test_existing_matching_remote_is_marked_without_upload(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
backup = self.backup(Path(folder)); checksum = transfer.digest(backup)
|
||||
with patch.object(transfer, "remote_digest", return_value=checksum), \
|
||||
patch.object(transfer, "run") as run:
|
||||
transfer.transfer_one(self.config(Path(folder)), backup)
|
||||
run.assert_not_called()
|
||||
marker = json.loads(transfer.marker_path(backup).read_text(encoding="utf-8"))
|
||||
self.assertEqual(marker["sha256"], checksum)
|
||||
|
||||
def test_existing_mismatched_remote_is_never_overwritten(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
backup = self.backup(Path(folder))
|
||||
with patch.object(transfer, "remote_digest", return_value="9" * 64), \
|
||||
patch.object(transfer, "run") as run:
|
||||
with self.assertRaisesRegex(RuntimeError, "different checksum"):
|
||||
transfer.transfer_one(self.config(Path(folder)), backup)
|
||||
run.assert_not_called()
|
||||
self.assertFalse(transfer.marker_path(backup).exists())
|
||||
|
||||
def test_new_remote_is_uploaded_verified_and_marked(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
backup = self.backup(Path(folder)); checksum = transfer.digest(backup)
|
||||
with patch.object(transfer, "remote_digest", side_effect=[None, checksum, checksum]), \
|
||||
patch.object(transfer, "run", return_value=b"") as run, \
|
||||
patch.object(transfer.subprocess, "run"):
|
||||
transfer.transfer_one(self.config(Path(folder)), backup)
|
||||
self.assertTrue(any(call.args[0][0] == "rsync" for call in run.call_args_list))
|
||||
self.assertTrue(transfer.marker_path(backup).exists())
|
||||
|
||||
def test_prune_removes_only_old_checksum_verified_backups(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
directory = Path(folder)
|
||||
verified = self.backup(directory)
|
||||
checksum = transfer.digest(verified)
|
||||
transfer.write_marker(verified, checksum)
|
||||
unverified = self.backup(directory, "guestops-20261002T021700Z.tar.gpg")
|
||||
old = time.time() - 8 * 86400
|
||||
os.utime(verified, (old, old)); os.utime(unverified, (old, old))
|
||||
removed = transfer.prune_verified(self.config(directory), 7, now=time.time())
|
||||
self.assertEqual(removed, 1)
|
||||
self.assertFalse(verified.exists())
|
||||
self.assertTrue(unverified.exists())
|
||||
|
||||
def test_retention_bounds_are_enforced(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
with self.assertRaisesRegex(RuntimeError, "between 1 and 365"):
|
||||
transfer.prune_verified(self.config(Path(folder)), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,113 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"debian_acceptance", Path(__file__).resolve().parents[1] / "deploy" / "debian_acceptance.py")
|
||||
acceptance = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(acceptance)
|
||||
|
||||
COMMIT = "a" * 40
|
||||
RELEASE_SHA = "b" * 64
|
||||
|
||||
|
||||
def common(system):
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"system": system,
|
||||
"evidenceId": acceptance.SYSTEMS[system]["evidenceId"],
|
||||
"releaseVersion": "0.2.0",
|
||||
"releaseCommit": COMMIT,
|
||||
"releaseRecordSha256": RELEASE_SHA,
|
||||
"archiveSha256": "c" * 64,
|
||||
"environment": "https://sandbox-guestops.futuresens.co.uk",
|
||||
"hostIdentifier": "guestops-sandbox-01",
|
||||
"images": {
|
||||
"api": {"reference": f"guestops-api:{COMMIT}", "id": "sha256:" + "d" * 64},
|
||||
"worker": {"reference": f"guestops-worker:{COMMIT}", "id": "sha256:" + "e" * 64},
|
||||
},
|
||||
"operator": "Deployment operator",
|
||||
"reviewedBy": "Independent reviewer",
|
||||
"startedAt": "2026-09-30T09:00:00Z",
|
||||
"endedAt": "2026-09-30T10:00:00Z",
|
||||
"reviewedAt": "2026-09-30T11:00:00Z",
|
||||
"unresolvedCriticalFindings": 0,
|
||||
"scenarios": [
|
||||
{"id": scenario, "status": "pass", "evidence": [f"restricted-{index}"]}
|
||||
for index, scenario in enumerate(sorted(acceptance.SYSTEMS[system]["scenarios"]), 1)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def valid_records():
|
||||
host = common("guestops-debian-host")
|
||||
host["hostFacts"] = {
|
||||
"debianMajor": 12,
|
||||
"cpuCores": 4,
|
||||
"memoryBytes": 8_140_382_208,
|
||||
"freeDiskBytes": 14_275_686_400,
|
||||
"publicTcpPorts": [80, 443],
|
||||
}
|
||||
host["featureControls"] = {
|
||||
"googleSending": "disabled",
|
||||
"faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled",
|
||||
"paymentCreation": "disabled",
|
||||
}
|
||||
persistence = common("guestops-persistence")
|
||||
persistence["drillCommand"] = "python3 deploy/ops.py persistence-drill --confirm-restart"
|
||||
return host, persistence
|
||||
|
||||
|
||||
class DebianAcceptanceTests(unittest.TestCase):
|
||||
def test_complete_matching_records_pass(self):
|
||||
acceptance.validate_pair(*valid_records(), COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_expected_release_identity_is_required(self):
|
||||
host, persistence = valid_records()
|
||||
host["releaseCommit"] = "f" * 40
|
||||
with self.assertRaisesRegex(ValueError, "approved candidate"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
host, persistence = valid_records()
|
||||
persistence["releaseRecordSha256"] = "f" * 64
|
||||
with self.assertRaisesRegex(ValueError, "retained release record"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_exact_images_and_cross_record_identity_are_required(self):
|
||||
host, persistence = valid_records()
|
||||
host["images"]["api"]["reference"] = "guestops-api:latest"
|
||||
with self.assertRaisesRegex(ValueError, "full approved commit"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
host, persistence = valid_records()
|
||||
persistence["archiveSha256"] = "f" * 64
|
||||
with self.assertRaisesRegex(ValueError, "same archiveSha256"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_host_capacity_ports_and_disabled_controls_are_required(self):
|
||||
host, persistence = valid_records()
|
||||
host["hostFacts"]["publicTcpPorts"] = [80, 443, 8080]
|
||||
with self.assertRaisesRegex(ValueError, "Only TCP ports"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
host, persistence = valid_records()
|
||||
host["featureControls"]["googleSending"] = "enabled"
|
||||
with self.assertRaisesRegex(ValueError, "must remain disabled"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
|
||||
def test_independent_review_scenarios_and_findings_are_required(self):
|
||||
host, persistence = valid_records()
|
||||
host["reviewedBy"] = host["operator"]
|
||||
with self.assertRaisesRegex(ValueError, "different people"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
host, persistence = valid_records()
|
||||
persistence["scenarios"][0]["status"] = "not-run"
|
||||
with self.assertRaisesRegex(ValueError, "has not passed"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
host, persistence = valid_records()
|
||||
host["unresolvedCriticalFindings"] = 1
|
||||
with self.assertRaisesRegex(ValueError, "critical findings"):
|
||||
acceptance.validate_pair(host, persistence, COMMIT, RELEASE_SHA)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,75 @@
|
||||
import datetime as dt
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"monitor_status", Path(__file__).resolve().parents[1] / "deploy" / "monitor_status.py")
|
||||
monitor = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(monitor)
|
||||
|
||||
|
||||
class MonitorStatusTests(unittest.TestCase):
|
||||
def test_compose_json_is_reduced_to_safe_state(self):
|
||||
value = json.dumps([
|
||||
{"Service": "api", "State": "running", "Health": "healthy", "Publishers": "secret"},
|
||||
{"Service": "worker", "State": "running", "Health": ""},
|
||||
{"Service": "mongo", "State": "running", "Health": "healthy"},
|
||||
]).encode()
|
||||
self.assertEqual(monitor.parse_compose(value)["api"], {"state": "running", "health": "healthy"})
|
||||
self.assertNotIn("Publishers", monitor.parse_compose(value)["api"])
|
||||
|
||||
def test_age_ignores_symlinks_and_unexpected_files(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
directory = Path(folder)
|
||||
backup = directory / "guestops-20261001T021700Z.tar.gpg"
|
||||
backup.write_bytes(b"fixture")
|
||||
moment = dt.datetime(2026, 10, 1, 3, 17, tzinfo=dt.timezone.utc)
|
||||
os.utime(backup, (moment.timestamp() - 3600, moment.timestamp() - 3600))
|
||||
(directory / "secret.txt").write_text("ignored", encoding="utf-8")
|
||||
self.assertEqual(monitor.age_seconds(directory, monitor.BACKUP_NAME, moment), 3600)
|
||||
|
||||
def test_build_status_contains_only_aggregate_health(self):
|
||||
now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc)
|
||||
compose = json.dumps([
|
||||
{"Service": name, "State": "running", "Health": "healthy"}
|
||||
for name in ("api", "worker", "mongo")
|
||||
]).encode()
|
||||
usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})()
|
||||
with tempfile.TemporaryDirectory() as folder, \
|
||||
patch.object(monitor, "https_status", return_value={"ready": True, "certificateDaysRemaining": 60}), \
|
||||
patch.object(monitor, "run", return_value=compose), \
|
||||
patch.object(monitor, "worker_heartbeat_age", return_value=30), \
|
||||
patch.object(monitor.shutil, "disk_usage", return_value=usage):
|
||||
status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertEqual(status["workerHeartbeatAgeSeconds"], 30)
|
||||
self.assertEqual(status["diskFreePercent"], 50)
|
||||
self.assertNotIn("credentials", json.dumps(status).lower())
|
||||
|
||||
def test_failed_probes_write_categories_not_exception_details(self):
|
||||
now = dt.datetime(2026, 10, 1, 12, tzinfo=dt.timezone.utc)
|
||||
usage = type("Usage", (), {"total": 1000, "used": 500, "free": 500})()
|
||||
with tempfile.TemporaryDirectory() as folder, \
|
||||
patch.object(monitor, "https_status", side_effect=RuntimeError("secret value")), \
|
||||
patch.object(monitor, "run", side_effect=RuntimeError("secret value")), \
|
||||
patch.object(monitor, "worker_heartbeat_age", side_effect=RuntimeError("secret value")), \
|
||||
patch.object(monitor.shutil, "disk_usage", return_value=usage):
|
||||
status, errors = monitor.build_status(Path(folder), Path(folder), "https://example.invalid", now)
|
||||
self.assertGreaterEqual(len(errors), 3)
|
||||
self.assertNotIn("secret value", json.dumps(status))
|
||||
|
||||
def test_status_output_is_atomic_json(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "status.json"
|
||||
monitor.write_status(path, {"schemaVersion": 1, "errors": []})
|
||||
self.assertEqual(json.loads(path.read_text(encoding="utf-8"))["schemaVersion"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user