This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"system": "guestops-backup-recovery",
|
||||
"evidenceId": "backup-restore",
|
||||
"dataClassification": "synthetic-only",
|
||||
"releaseVersion": "0.2.0",
|
||||
"releaseCommit": "0000000000000000000000000000000000000000",
|
||||
"releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"environment": "https://sandbox-guestops.futuresens.co.uk",
|
||||
"hostIdentifier": "guestops-sandbox-01",
|
||||
"images": {
|
||||
"api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
"worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
"mongo": {"reference": "mongo:8.0", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}
|
||||
},
|
||||
"owners": {
|
||||
"backup": "REPLACE",
|
||||
"monitoring": "REPLACE",
|
||||
"recoveryOperator": "REPLACE",
|
||||
"technicalEscalation": "REPLACE",
|
||||
"retention": "REPLACE",
|
||||
"independentReviewer": "REPLACE"
|
||||
},
|
||||
"startedAt": "2026-10-01T09:00:00Z",
|
||||
"endedAt": "2026-10-01T13:00:00Z",
|
||||
"reviewedAt": "2026-10-01T14:00:00Z",
|
||||
"recoveryObjectives": {
|
||||
"targetRpoHours": 24,
|
||||
"observedRpoHours": 25,
|
||||
"targetRtoMinutes": 240,
|
||||
"observedRtoMinutes": 241
|
||||
},
|
||||
"retention": {
|
||||
"localVerifiedDays": 7,
|
||||
"offHostDaily": 35,
|
||||
"offHostMonthly": 12,
|
||||
"legalHoldOverrideTested": false
|
||||
},
|
||||
"backup": {
|
||||
"createdAt": "2026-10-01T08:00:00Z",
|
||||
"sha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"transferredSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"privateKeyPresentOnHost": false
|
||||
},
|
||||
"rollback": {
|
||||
"previousReleaseCommit": "1111111111111111111111111111111111111111",
|
||||
"previousArchiveSha256": "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"previousImages": {
|
||||
"api": {"reference": "guestops-api:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"},
|
||||
"worker": {"reference": "guestops-worker:1111111111111111111111111111111111111111", "id": "sha256:1111111111111111111111111111111111111111111111111111111111111111"}
|
||||
},
|
||||
"persistentVolumesReplaced": false,
|
||||
"restoredReleaseCommit": "0000000000000000000000000000000000000000",
|
||||
"unresolvedOperations": 1,
|
||||
"finalControls": {
|
||||
"googleSending": "disabled",
|
||||
"faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled",
|
||||
"paymentCreation": "disabled"
|
||||
}
|
||||
},
|
||||
"monitoringState": "not-configured",
|
||||
"unresolvedCriticalFindings": 1,
|
||||
"scenarios": [
|
||||
{"id": "alert-escalation", "status": "not-run", "evidence": []},
|
||||
{"id": "atomic-off-host-transfer", "status": "not-run", "evidence": []},
|
||||
{"id": "controlled-return-to-service", "status": "not-run", "evidence": []},
|
||||
{"id": "data-protection-recovery", "status": "not-run", "evidence": []},
|
||||
{"id": "database-inventory", "status": "not-run", "evidence": []},
|
||||
{"id": "durable-secret-free-logs", "status": "not-run", "evidence": []},
|
||||
{"id": "encrypted-manual-backup", "status": "not-run", "evidence": []},
|
||||
{"id": "image-rollback", "status": "not-run", "evidence": []},
|
||||
{"id": "isolated-restore", "status": "not-run", "evidence": []},
|
||||
{"id": "monitoring-coverage", "status": "not-run", "evidence": []},
|
||||
{"id": "off-host-checksum", "status": "not-run", "evidence": []},
|
||||
{"id": "production-service-recovery", "status": "not-run", "evidence": []},
|
||||
{"id": "retention-and-legal-hold", "status": "not-run", "evidence": []},
|
||||
{"id": "scheduled-backup", "status": "not-run", "evidence": []}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a restricted GuestOps backup, monitoring and recovery record."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
VERSION = "0.2.0"
|
||||
SCENARIOS = {
|
||||
"encrypted-manual-backup",
|
||||
"scheduled-backup",
|
||||
"production-service-recovery",
|
||||
"atomic-off-host-transfer",
|
||||
"off-host-checksum",
|
||||
"retention-and-legal-hold",
|
||||
"monitoring-coverage",
|
||||
"alert-escalation",
|
||||
"durable-secret-free-logs",
|
||||
"isolated-restore",
|
||||
"data-protection-recovery",
|
||||
"database-inventory",
|
||||
"image-rollback",
|
||||
"controlled-return-to-service",
|
||||
}
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}")
|
||||
GIT_SHA = re.compile(r"[0-9a-f]{40}")
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def timestamp(value: object, field: str) -> dt.datetime:
|
||||
require(isinstance(value, str) and value.endswith("Z"),
|
||||
f"{field} must be a UTC timestamp ending in Z.")
|
||||
try:
|
||||
parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{field} is not a valid timestamp.") from error
|
||||
require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.")
|
||||
return parsed
|
||||
|
||||
|
||||
def safe_name(value: object, field: str) -> str:
|
||||
name = str(value or "").strip()
|
||||
require(2 <= len(name) <= 120 and "@" not in name and "/" not in name and "\\" not in name,
|
||||
f"{field} requires a name without an email address or path.")
|
||||
return name
|
||||
|
||||
|
||||
def validate(record: object, expected_commit: str, expected_release_sha256: str) -> None:
|
||||
require(isinstance(record, dict), "Acceptance record must be a JSON object.")
|
||||
require(record.get("schemaVersion") == 1, "Unsupported backup/recovery schema.")
|
||||
require(record.get("system") == "guestops-backup-recovery",
|
||||
"system must be guestops-backup-recovery.")
|
||||
require(record.get("evidenceId") == "backup-restore",
|
||||
"evidenceId must be backup-restore.")
|
||||
require(record.get("dataClassification") == "synthetic-only",
|
||||
"Recovery acceptance must use synthetic data only.")
|
||||
require(record.get("releaseVersion") == VERSION, f"releaseVersion must be {VERSION}.")
|
||||
require(GIT_SHA.fullmatch(str(expected_commit)) is not None,
|
||||
"Expected release commit must be a full lowercase Git SHA.")
|
||||
require(SHA256.fullmatch(str(expected_release_sha256)) is not None,
|
||||
"Expected release-record checksum must be a lowercase SHA-256 digest.")
|
||||
require(record.get("releaseCommit") == expected_commit,
|
||||
"releaseCommit does not match the approved candidate.")
|
||||
require(record.get("releaseRecordSha256") == expected_release_sha256,
|
||||
"releaseRecordSha256 does not match the retained release record.")
|
||||
require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None,
|
||||
"archiveSha256 must be a lowercase SHA-256 digest.")
|
||||
|
||||
origin = urlparse(str(record.get("environment", "")))
|
||||
require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/")
|
||||
and not origin.query and not origin.fragment and origin.username is None
|
||||
and origin.password is None,
|
||||
"environment must be an HTTPS origin without credentials, path, query or fragment.")
|
||||
safe_name(record.get("hostIdentifier"), "hostIdentifier")
|
||||
|
||||
images = record.get("images")
|
||||
require(isinstance(images, dict) and set(images) == {"api", "worker", "mongo"},
|
||||
"images must contain exactly api, worker and mongo.")
|
||||
for name in ("api", "worker"):
|
||||
image = images[name]
|
||||
require(isinstance(image, dict) and set(image) == {"reference", "id"},
|
||||
f"images.{name} must contain exactly reference and id.")
|
||||
require(image["reference"] == f"guestops-{name}:{expected_commit}",
|
||||
f"images.{name}.reference must use the full approved commit.")
|
||||
require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None,
|
||||
f"images.{name}.id must be immutable.")
|
||||
mongo = images["mongo"]
|
||||
require(isinstance(mongo, dict) and set(mongo) == {"reference", "id"}
|
||||
and mongo["reference"] == "mongo:8.0"
|
||||
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(mongo["id"])) is not None,
|
||||
"images.mongo must identify the immutable mongo:8.0 image.")
|
||||
|
||||
owners = record.get("owners")
|
||||
owner_keys = {"backup", "monitoring", "recoveryOperator", "technicalEscalation",
|
||||
"retention", "independentReviewer"}
|
||||
require(isinstance(owners, dict) and set(owners) == owner_keys,
|
||||
"owners must contain the exact operational and review roles.")
|
||||
names = {key: safe_name(value, f"owners.{key}") for key, value in owners.items()}
|
||||
reviewer = names["independentReviewer"].casefold()
|
||||
require(reviewer not in {names[key].casefold() for key in owner_keys - {"independentReviewer"}},
|
||||
"independentReviewer must be different from every operational owner.")
|
||||
|
||||
started = timestamp(record.get("startedAt"), "startedAt")
|
||||
ended = timestamp(record.get("endedAt"), "endedAt")
|
||||
reviewed = timestamp(record.get("reviewedAt"), "reviewedAt")
|
||||
require(started <= ended <= reviewed, "Acceptance timestamps are out of order.")
|
||||
|
||||
recovery = record.get("recoveryObjectives")
|
||||
require(isinstance(recovery, dict) and set(recovery) == {
|
||||
"targetRpoHours", "observedRpoHours", "targetRtoMinutes", "observedRtoMinutes",
|
||||
}, "recoveryObjectives must contain exact target and observed RPO/RTO values.")
|
||||
for field in recovery:
|
||||
require(isinstance(recovery[field], (int, float)) and not isinstance(recovery[field], bool)
|
||||
and recovery[field] >= 0, f"recoveryObjectives.{field} must be non-negative.")
|
||||
require(recovery["targetRpoHours"] == 24 and recovery["observedRpoHours"] <= 24,
|
||||
"Observed RPO must meet the approved 24-hour target.")
|
||||
require(recovery["targetRtoMinutes"] == 240 and recovery["observedRtoMinutes"] <= 240,
|
||||
"Observed RTO must meet the approved four-hour target.")
|
||||
|
||||
retention = record.get("retention")
|
||||
require(retention == {
|
||||
"localVerifiedDays": 7,
|
||||
"offHostDaily": 35,
|
||||
"offHostMonthly": 12,
|
||||
"legalHoldOverrideTested": True,
|
||||
}, "Retention must record seven local days, 35 daily and 12 monthly off-host copies, and legal-hold testing.")
|
||||
|
||||
backup = record.get("backup")
|
||||
require(isinstance(backup, dict) and set(backup) == {
|
||||
"createdAt", "sha256", "transferredSha256", "privateKeyPresentOnHost",
|
||||
}, "backup must contain exact creation, checksum, transfer and private-key fields.")
|
||||
created = timestamp(backup["createdAt"], "backup.createdAt")
|
||||
require(created <= started, "The accepted backup must exist when the timed exercise starts.")
|
||||
require(abs(recovery["observedRpoHours"] - (started - created).total_seconds() / 3600) < 0.01,
|
||||
"Observed RPO must match the backup and exercise timestamps.")
|
||||
require(abs(recovery["observedRtoMinutes"] - (ended - started).total_seconds() / 60) < 0.01,
|
||||
"Observed RTO must match the exercise timestamps.")
|
||||
require(SHA256.fullmatch(str(backup["sha256"])) is not None
|
||||
and backup["transferredSha256"] == backup["sha256"],
|
||||
"Local and transferred backup checksums must match.")
|
||||
require(backup["privateKeyPresentOnHost"] is False,
|
||||
"The recovery private key must not be present on the Debian host.")
|
||||
|
||||
rollback = record.get("rollback")
|
||||
require(isinstance(rollback, dict) and set(rollback) == {
|
||||
"previousReleaseCommit", "previousArchiveSha256", "previousImages",
|
||||
"persistentVolumesReplaced", "restoredReleaseCommit", "unresolvedOperations", "finalControls",
|
||||
}, "rollback must contain the exact rehearsal and final-state fields.")
|
||||
require(GIT_SHA.fullmatch(str(rollback["previousReleaseCommit"])) is not None
|
||||
and rollback["previousReleaseCommit"] != expected_commit,
|
||||
"Rollback must use a different retained previous release.")
|
||||
require(SHA256.fullmatch(str(rollback["previousArchiveSha256"])) is not None,
|
||||
"Rollback requires the previous archive checksum.")
|
||||
previous_images = rollback["previousImages"]
|
||||
require(isinstance(previous_images, dict) and set(previous_images) == {"api", "worker"},
|
||||
"Rollback requires exact previous API and worker images.")
|
||||
for name in ("api", "worker"):
|
||||
image = previous_images[name]
|
||||
require(isinstance(image, dict) and set(image) == {"reference", "id"}
|
||||
and image["reference"] == f"guestops-{name}:{rollback['previousReleaseCommit']}"
|
||||
and re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None,
|
||||
f"rollback.previousImages.{name} must use the retained previous release identity.")
|
||||
require(rollback["persistentVolumesReplaced"] is False,
|
||||
"Image rollback must not replace persistent volumes.")
|
||||
require(rollback["restoredReleaseCommit"] == expected_commit,
|
||||
"The exercise must finish on the approved candidate.")
|
||||
require(rollback["unresolvedOperations"] == 0,
|
||||
"The exercise must finish without unresolved operations.")
|
||||
require(rollback["finalControls"] == {
|
||||
"googleSending": "disabled", "faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled", "paymentCreation": "disabled",
|
||||
}, "The exercise must finish with all unaccepted external writes disabled.")
|
||||
|
||||
scenarios = record.get("scenarios")
|
||||
require(isinstance(scenarios, list), "scenarios must be a list.")
|
||||
ids = [item.get("id") for item in scenarios if isinstance(item, dict)]
|
||||
require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == SCENARIOS,
|
||||
"Acceptance record requires the exact backup/recovery scenario set.")
|
||||
for item in scenarios:
|
||||
scenario_id = item["id"]
|
||||
require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.")
|
||||
evidence = item.get("evidence")
|
||||
require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all(
|
||||
isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value
|
||||
and "\\" not in value and not value.startswith("/") for value in evidence
|
||||
), f"Scenario {scenario_id} requires safe opaque evidence references.")
|
||||
require(record.get("monitoringState") == "healthy",
|
||||
"Monitoring must be healthy at acceptance completion.")
|
||||
require(record.get("unresolvedCriticalFindings") == 0,
|
||||
"Acceptance cannot pass with unresolved critical findings.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("record", type=Path)
|
||||
parser.add_argument("--expected-commit", required=True)
|
||||
parser.add_argument("--expected-release-record-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
validate(json.loads(args.record.read_text(encoding="utf-8")),
|
||||
args.expected_commit, args.expected_release_record_sha256)
|
||||
print("Backup, monitoring and recovery acceptance record is structurally complete and passed. "
|
||||
"This validates the record, not its restricted evidence.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
print(f"Backup/recovery acceptance record rejected: {error}", file=__import__("sys").stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Atomically transfer encrypted GuestOps backups to restricted storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg")
|
||||
SAFE_HOST = re.compile(r"[A-Za-z0-9.-]{1,253}")
|
||||
SAFE_USER = re.compile(r"[A-Za-z_][A-Za-z0-9_-]{0,31}")
|
||||
SAFE_REMOTE_PATH = re.compile(r"/[A-Za-z0-9._/-]{1,500}")
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
with path.open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def private_directory(value: str) -> Path:
|
||||
requested = Path(value)
|
||||
require(requested.is_absolute() and not requested.is_symlink(),
|
||||
"BACKUP_DIRECTORY must be an absolute, non-symlink path.")
|
||||
directory = requested.resolve()
|
||||
require(directory.is_dir(), "BACKUP_DIRECTORY must exist.")
|
||||
require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0,
|
||||
"BACKUP_DIRECTORY must not be accessible to group or other users.")
|
||||
return directory
|
||||
|
||||
|
||||
def regular_file(value: str, field: str, *, private: bool) -> Path:
|
||||
requested = Path(value)
|
||||
require(requested.is_absolute() and not requested.is_symlink(),
|
||||
f"{field} must be an absolute, non-symlink path.")
|
||||
path = requested.resolve()
|
||||
require(path.is_file(), f"{field} must be an existing regular file.")
|
||||
if private:
|
||||
require(stat.S_IMODE(path.stat().st_mode) & 0o077 == 0,
|
||||
f"{field} must not be accessible to group or other users.")
|
||||
return path
|
||||
|
||||
|
||||
def configuration(environment: dict[str, str]) -> dict[str, object]:
|
||||
directory = private_directory(environment.get("BACKUP_DIRECTORY", ""))
|
||||
host = environment.get("BACKUP_REMOTE_HOST", "")
|
||||
user = environment.get("BACKUP_REMOTE_USER", "")
|
||||
remote = environment.get("BACKUP_REMOTE_DIRECTORY", "")
|
||||
require(SAFE_HOST.fullmatch(host) is not None, "BACKUP_REMOTE_HOST is invalid.")
|
||||
require(SAFE_USER.fullmatch(user) is not None, "BACKUP_REMOTE_USER is invalid.")
|
||||
require(SAFE_REMOTE_PATH.fullmatch(remote) is not None and "//" not in remote
|
||||
and "/../" not in remote + "/" and not remote.endswith("/.."),
|
||||
"BACKUP_REMOTE_DIRECTORY must be a safe absolute path.")
|
||||
identity = regular_file(environment.get("BACKUP_SSH_IDENTITY", ""),
|
||||
"BACKUP_SSH_IDENTITY", private=True)
|
||||
known_hosts = regular_file(environment.get("BACKUP_SSH_KNOWN_HOSTS", ""),
|
||||
"BACKUP_SSH_KNOWN_HOSTS", private=False)
|
||||
require(shutil.which("ssh") is not None and shutil.which("rsync") is not None,
|
||||
"ssh and rsync are required.")
|
||||
return {
|
||||
"directory": directory,
|
||||
"host": host,
|
||||
"user": user,
|
||||
"remote": remote.rstrip("/"),
|
||||
"identity": identity,
|
||||
"known_hosts": known_hosts,
|
||||
}
|
||||
|
||||
|
||||
def ssh_base(config: dict[str, object]) -> list[str]:
|
||||
return [
|
||||
"ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
|
||||
"-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15",
|
||||
"-o", f"UserKnownHostsFile={config['known_hosts']}",
|
||||
"-i", str(config["identity"]), f"{config['user']}@{config['host']}",
|
||||
]
|
||||
|
||||
|
||||
def run(args: list[str], *, environment: dict[str, str] | None = None) -> bytes:
|
||||
result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=900, env=environment)
|
||||
require(result.returncode == 0,
|
||||
f"{Path(args[0]).name} step failed; review the restricted operator logs.")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def remote_digest(config: dict[str, object], remote_path: str) -> str | None:
|
||||
command = f"if test -f {remote_path} && test ! -L {remote_path}; then sha256sum -- {remote_path}; fi"
|
||||
output = run([*ssh_base(config), command]).decode("utf-8", "strict").strip()
|
||||
if not output:
|
||||
return None
|
||||
value = output.split()[0]
|
||||
require(SHA256.fullmatch(value) is not None, "Remote checksum response was invalid.")
|
||||
return value
|
||||
|
||||
|
||||
def marker_path(backup: Path) -> Path:
|
||||
return backup.with_name(backup.name + ".transferred.json")
|
||||
|
||||
|
||||
def write_marker(backup: Path, checksum: str) -> None:
|
||||
marker = marker_path(backup)
|
||||
payload = json.dumps({
|
||||
"schemaVersion": 1,
|
||||
"backup": backup.name,
|
||||
"sha256": checksum,
|
||||
"verifiedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
}, sort_keys=True) + "\n"
|
||||
fd, temporary = tempfile.mkstemp(prefix=marker.name + ".", dir=marker.parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
stream.write(payload)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, marker)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def transfer_one(config: dict[str, object], backup: Path) -> None:
|
||||
require(backup.is_file() and not backup.is_symlink()
|
||||
and BACKUP_NAME.fullmatch(backup.name) is not None,
|
||||
"Refusing to transfer an unexpected backup path.")
|
||||
checksum = digest(backup)
|
||||
remote_final = f"{config['remote']}/{backup.name}"
|
||||
existing = remote_digest(config, remote_final)
|
||||
if existing is not None:
|
||||
require(existing == checksum, "A remote backup with this name has a different checksum.")
|
||||
write_marker(backup, checksum)
|
||||
return
|
||||
|
||||
remote_partial = f"{config['remote']}/.{backup.name}.partial-{uuid.uuid4().hex}"
|
||||
rsh = shlex.join([
|
||||
"ssh", "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes",
|
||||
"-o", "StrictHostKeyChecking=yes", "-o", "ConnectTimeout=15",
|
||||
"-o", f"UserKnownHostsFile={config['known_hosts']}",
|
||||
"-i", str(config["identity"]),
|
||||
])
|
||||
rsync_environment = os.environ.copy()
|
||||
rsync_environment["RSYNC_RSH"] = rsh
|
||||
try:
|
||||
run(["rsync", "--archive", "--chmod=F600", "--protect-args", "--",
|
||||
str(backup), f"{config['user']}@{config['host']}:{remote_partial}"],
|
||||
environment=rsync_environment)
|
||||
require(remote_digest(config, remote_partial) == checksum,
|
||||
"Transferred backup checksum does not match the local file.")
|
||||
command = (f"test ! -e {remote_final} && mv -T -- {remote_partial} {remote_final} "
|
||||
f"&& chmod 600 -- {remote_final}")
|
||||
run([*ssh_base(config), command])
|
||||
require(remote_digest(config, remote_final) == checksum,
|
||||
"Final remote backup checksum does not match the local file.")
|
||||
write_marker(backup, checksum)
|
||||
except Exception:
|
||||
# The name contains a fresh random suffix and is the only remote object this run may remove.
|
||||
subprocess.run([*ssh_base(config), f"rm -f -- {remote_partial}"], stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
|
||||
raise
|
||||
|
||||
|
||||
def transfer_pending(config: dict[str, object]) -> int:
|
||||
directory = config["directory"]
|
||||
backups = sorted(path for path in directory.iterdir()
|
||||
if path.is_file() and not path.is_symlink()
|
||||
and BACKUP_NAME.fullmatch(path.name) is not None)
|
||||
for backup in backups:
|
||||
marker = marker_path(backup)
|
||||
if marker.is_file() and not marker.is_symlink():
|
||||
try:
|
||||
recorded = json.loads(marker.read_text(encoding="utf-8"))
|
||||
if recorded.get("sha256") == digest(backup):
|
||||
continue
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
pass
|
||||
transfer_one(config, backup)
|
||||
return len(backups)
|
||||
|
||||
|
||||
def prune_verified(config: dict[str, object], retention_days: int, now: float | None = None) -> int:
|
||||
require(1 <= retention_days <= 365, "Local retention must be between 1 and 365 days.")
|
||||
threshold = (time.time() if now is None else now) - retention_days * 86400
|
||||
removed = 0
|
||||
for backup in config["directory"].iterdir():
|
||||
if not backup.is_file() or backup.is_symlink() or BACKUP_NAME.fullmatch(backup.name) is None:
|
||||
continue
|
||||
marker = marker_path(backup)
|
||||
if backup.stat().st_mtime >= threshold or not marker.is_file() or marker.is_symlink():
|
||||
continue
|
||||
try:
|
||||
recorded = json.loads(marker.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
if recorded.get("backup") != backup.name or recorded.get("sha256") != digest(backup):
|
||||
continue
|
||||
backup.unlink()
|
||||
marker.unlink()
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--prune-verified", action="store_true")
|
||||
parser.add_argument("--retention-days", type=int, default=7)
|
||||
args = parser.parse_args()
|
||||
config = configuration(dict(os.environ))
|
||||
observed = transfer_pending(config)
|
||||
removed = prune_verified(config, args.retention_days) if args.prune_verified else 0
|
||||
print(f"Backup transfer completed: {observed} encrypted backup(s) inspected; "
|
||||
f"{removed} verified local backup(s) expired.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError, json.JSONDecodeError) as error:
|
||||
print(f"Backup transfer failed: {error}", file=__import__("sys").stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"system": "guestops-debian-host",
|
||||
"evidenceId": "debian-host",
|
||||
"releaseVersion": "0.2.0",
|
||||
"releaseCommit": "0000000000000000000000000000000000000000",
|
||||
"releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"environment": "https://sandbox-guestops.futuresens.co.uk",
|
||||
"hostIdentifier": "guestops-sandbox-01",
|
||||
"images": {
|
||||
"api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
"worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}
|
||||
},
|
||||
"hostFacts": {
|
||||
"debianMajor": 12,
|
||||
"cpuCores": 4,
|
||||
"memoryBytes": 8140382208,
|
||||
"freeDiskBytes": 14275686400,
|
||||
"publicTcpPorts": []
|
||||
},
|
||||
"featureControls": {
|
||||
"googleSending": "disabled",
|
||||
"faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled",
|
||||
"paymentCreation": "disabled"
|
||||
},
|
||||
"operator": "REPLACE",
|
||||
"reviewedBy": "REPLACE",
|
||||
"startedAt": "2026-09-30T09:00:00Z",
|
||||
"endedAt": "2026-09-30T10:00:00Z",
|
||||
"reviewedAt": "2026-09-30T11:00:00Z",
|
||||
"unresolvedCriticalFindings": 1,
|
||||
"scenarios": [
|
||||
{"id": "boot-services", "status": "not-run", "evidence": []},
|
||||
{"id": "controlled-reboot", "status": "not-run", "evidence": []},
|
||||
{"id": "durable-log-retrieval", "status": "not-run", "evidence": []},
|
||||
{"id": "host-baseline", "status": "not-run", "evidence": []},
|
||||
{"id": "https-and-redirect", "status": "not-run", "evidence": []},
|
||||
{"id": "network-exposure", "status": "not-run", "evidence": []},
|
||||
{"id": "proxy-trust", "status": "not-run", "evidence": []},
|
||||
{"id": "secret-free-logs", "status": "not-run", "evidence": []},
|
||||
{"id": "workspace-health", "status": "not-run", "evidence": []}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate restricted GuestOps Debian-host and persistence acceptance records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
VERSION = "0.2.0"
|
||||
SYSTEMS = {
|
||||
"guestops-debian-host": {
|
||||
"evidenceId": "debian-host",
|
||||
"scenarios": {
|
||||
"host-baseline",
|
||||
"network-exposure",
|
||||
"https-and-redirect",
|
||||
"proxy-trust",
|
||||
"boot-services",
|
||||
"controlled-reboot",
|
||||
"workspace-health",
|
||||
"durable-log-retrieval",
|
||||
"secret-free-logs",
|
||||
},
|
||||
},
|
||||
"guestops-persistence": {
|
||||
"evidenceId": "persistence",
|
||||
"scenarios": {
|
||||
"separate-volume-layout",
|
||||
"service-restart",
|
||||
"container-recreation",
|
||||
"database-inventory",
|
||||
"data-protection-key",
|
||||
"image-identity",
|
||||
"post-reboot-persistence",
|
||||
},
|
||||
},
|
||||
}
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}")
|
||||
GIT_SHA = re.compile(r"[0-9a-f]{40}")
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def timestamp(value: object, field: str) -> dt.datetime:
|
||||
require(isinstance(value, str) and value.endswith("Z"),
|
||||
f"{field} must be a UTC timestamp ending in Z.")
|
||||
try:
|
||||
parsed = dt.datetime.fromisoformat(value.removesuffix("Z") + "+00:00")
|
||||
except ValueError as error:
|
||||
raise ValueError(f"{field} is not a valid timestamp.") from error
|
||||
require(parsed.tzinfo == dt.timezone.utc, f"{field} must be UTC.")
|
||||
return parsed
|
||||
|
||||
|
||||
def safe_text(value: object, field: str, minimum: int = 2, maximum: int = 160) -> str:
|
||||
text = str(value or "").strip()
|
||||
require(minimum <= len(text) <= maximum and "@" not in text and "\\" not in text,
|
||||
f"{field} must be safe text without an email address or local path.")
|
||||
return text
|
||||
|
||||
|
||||
def validate_common(record: object, expected_commit: str, expected_release_sha256: str) -> str:
|
||||
require(isinstance(record, dict), "Acceptance record must be a JSON object.")
|
||||
require(record.get("schemaVersion") == 1, "Unsupported Debian acceptance schema.")
|
||||
system = record.get("system")
|
||||
require(system in SYSTEMS, "Unknown Debian acceptance record system.")
|
||||
require(record.get("evidenceId") == SYSTEMS[system]["evidenceId"],
|
||||
f"{system} has the wrong evidenceId.")
|
||||
require(record.get("releaseVersion") == VERSION,
|
||||
f"releaseVersion must be {VERSION}.")
|
||||
require(GIT_SHA.fullmatch(str(expected_commit)) is not None,
|
||||
"Expected release commit must be a full lowercase Git SHA.")
|
||||
require(SHA256.fullmatch(str(expected_release_sha256)) is not None,
|
||||
"Expected release-record checksum must be a lowercase SHA-256 digest.")
|
||||
require(record.get("releaseCommit") == expected_commit,
|
||||
"releaseCommit does not match the approved candidate.")
|
||||
require(record.get("releaseRecordSha256") == expected_release_sha256,
|
||||
"releaseRecordSha256 does not match the retained release record.")
|
||||
require(SHA256.fullmatch(str(record.get("archiveSha256", ""))) is not None,
|
||||
"archiveSha256 must be a lowercase SHA-256 digest.")
|
||||
|
||||
origin = urlparse(str(record.get("environment", "")))
|
||||
require(origin.scheme == "https" and origin.hostname and origin.path in ("", "/")
|
||||
and not origin.query and not origin.fragment and origin.username is None
|
||||
and origin.password is None,
|
||||
"environment must be an HTTPS origin without credentials, path, query or fragment.")
|
||||
host = safe_text(record.get("hostIdentifier"), "hostIdentifier")
|
||||
|
||||
images = record.get("images")
|
||||
require(isinstance(images, dict) and set(images) == {"api", "worker"},
|
||||
"images must contain exactly api and worker.")
|
||||
for name in ("api", "worker"):
|
||||
image = images[name]
|
||||
require(isinstance(image, dict) and set(image) == {"reference", "id"},
|
||||
f"images.{name} must contain exactly reference and id.")
|
||||
require(image["reference"] == f"guestops-{name}:{expected_commit}",
|
||||
f"images.{name}.reference must use the full approved commit.")
|
||||
require(re.fullmatch(r"sha256:[0-9a-f]{64}", str(image["id"])) is not None,
|
||||
f"images.{name}.id must be an immutable image ID.")
|
||||
|
||||
operator = safe_text(record.get("operator"), "operator")
|
||||
reviewer = safe_text(record.get("reviewedBy"), "reviewedBy")
|
||||
require(operator.casefold() != reviewer.casefold(),
|
||||
"operator and reviewedBy must be different people.")
|
||||
started = timestamp(record.get("startedAt"), "startedAt")
|
||||
ended = timestamp(record.get("endedAt"), "endedAt")
|
||||
reviewed = timestamp(record.get("reviewedAt"), "reviewedAt")
|
||||
require(started <= ended <= reviewed, "Acceptance timestamps are out of order.")
|
||||
|
||||
scenarios = record.get("scenarios")
|
||||
require(isinstance(scenarios, list), "scenarios must be a list.")
|
||||
ids = [item.get("id") for item in scenarios if isinstance(item, dict)]
|
||||
required = SYSTEMS[system]["scenarios"]
|
||||
require(len(ids) == len(scenarios) and len(ids) == len(set(ids)) and set(ids) == required,
|
||||
f"{system} requires its exact acceptance scenario set.")
|
||||
for item in scenarios:
|
||||
scenario_id = item["id"]
|
||||
require(item.get("status") == "pass", f"Scenario {scenario_id} has not passed.")
|
||||
evidence = item.get("evidence")
|
||||
require(isinstance(evidence, list) and 1 <= len(evidence) <= 10 and all(
|
||||
isinstance(value, str) and 3 <= len(value) <= 200 and "@" not in value
|
||||
and "\\" not in value and not value.startswith("/") for value in evidence
|
||||
), f"Scenario {scenario_id} requires safe opaque evidence references.")
|
||||
|
||||
require(record.get("unresolvedCriticalFindings") == 0,
|
||||
"Acceptance cannot pass with unresolved critical findings.")
|
||||
return host
|
||||
|
||||
|
||||
def validate_host(record: object, expected_commit: str, expected_release_sha256: str) -> str:
|
||||
host = validate_common(record, expected_commit, expected_release_sha256)
|
||||
require(record.get("system") == "guestops-debian-host",
|
||||
"First record must be guestops-debian-host.")
|
||||
facts = record.get("hostFacts")
|
||||
require(isinstance(facts, dict) and set(facts) == {
|
||||
"debianMajor", "cpuCores", "memoryBytes", "freeDiskBytes", "publicTcpPorts",
|
||||
}, "hostFacts must contain the exact reviewed host facts.")
|
||||
require(isinstance(facts["debianMajor"], int) and facts["debianMajor"] >= 12,
|
||||
"Debian 12 or newer is required.")
|
||||
require(isinstance(facts["cpuCores"], int) and facts["cpuCores"] >= 4,
|
||||
"At least four CPU cores are required.")
|
||||
require(isinstance(facts["memoryBytes"], int) and facts["memoryBytes"] >= 7_500_000_000,
|
||||
"At least 7.5 GB of memory is required.")
|
||||
require(isinstance(facts["freeDiskBytes"], int) and facts["freeDiskBytes"] >= 8 * 1024**3,
|
||||
"At least 8 GiB of free disk space is required.")
|
||||
require(facts["publicTcpPorts"] == [80, 443],
|
||||
"Only TCP ports 80 and 443 may be public.")
|
||||
require(record.get("featureControls") == {
|
||||
"googleSending": "disabled",
|
||||
"faqLiveMode": "disabled",
|
||||
"pmsWrites": "disabled",
|
||||
"paymentCreation": "disabled",
|
||||
}, "Unaccepted external writes and FAQ live mode must remain disabled.")
|
||||
return host
|
||||
|
||||
|
||||
def validate_persistence(record: object, expected_commit: str, expected_release_sha256: str) -> str:
|
||||
host = validate_common(record, expected_commit, expected_release_sha256)
|
||||
require(record.get("system") == "guestops-persistence",
|
||||
"Second record must be guestops-persistence.")
|
||||
require(record.get("drillCommand") == "python3 deploy/ops.py persistence-drill --confirm-restart",
|
||||
"drillCommand must identify the confirmation-gated persistence drill.")
|
||||
return host
|
||||
|
||||
|
||||
def validate_pair(host_record: object, persistence_record: object,
|
||||
expected_commit: str, expected_release_sha256: str) -> None:
|
||||
host = validate_host(host_record, expected_commit, expected_release_sha256)
|
||||
persistence_host = validate_persistence(
|
||||
persistence_record, expected_commit, expected_release_sha256)
|
||||
require(host == persistence_host, "Both records must identify the same host.")
|
||||
for field in ("environment", "releaseVersion", "releaseCommit", "releaseRecordSha256",
|
||||
"archiveSha256", "images"):
|
||||
require(host_record.get(field) == persistence_record.get(field),
|
||||
f"Both records must use the same {field}.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("host_record", type=Path)
|
||||
parser.add_argument("persistence_record", type=Path)
|
||||
parser.add_argument("--expected-commit", required=True)
|
||||
parser.add_argument("--expected-release-record-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
host_record = json.loads(args.host_record.read_text(encoding="utf-8"))
|
||||
persistence_record = json.loads(args.persistence_record.read_text(encoding="utf-8"))
|
||||
validate_pair(host_record, persistence_record,
|
||||
args.expected_commit, args.expected_release_record_sha256)
|
||||
print("Debian-host and persistence acceptance records are structurally complete and passed. "
|
||||
"This validates the records, not their restricted evidence.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
print(f"Debian acceptance records rejected: {error}", file=__import__("sys").stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write non-sensitive GuestOps host status for a read-only monitoring agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import ssl
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg")
|
||||
MARKER_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg\.transferred\.json")
|
||||
AUTH = ('const c=new Mongo("mongodb://127.0.0.1");'
|
||||
'c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,'
|
||||
'process.env.MONGO_INITDB_ROOT_PASSWORD);')
|
||||
HEARTBEAT = ('const x=c.getDB("guestops").workerheartbeat.findOne({_id:"worker"});'
|
||||
'print(JSON.stringify(x&&x.At?x.At:null));')
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def run(args: list[str], root: Path, timeout: int = 30) -> bytes:
|
||||
result = subprocess.run(args, cwd=root, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
||||
require(result.returncode == 0, f"{Path(args[0]).name} probe failed.")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def utc_now() -> dt.datetime:
|
||||
return dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
|
||||
def age_seconds(path: Path, pattern: re.Pattern[str], now: dt.datetime) -> int | None:
|
||||
if not path.is_dir() or path.is_symlink():
|
||||
return None
|
||||
times = [item.stat().st_mtime for item in path.iterdir()
|
||||
if item.is_file() and not item.is_symlink() and pattern.fullmatch(item.name)]
|
||||
return max(0, int(now.timestamp() - max(times))) if times else None
|
||||
|
||||
|
||||
def https_status(origin: str, now: dt.datetime) -> dict[str, object]:
|
||||
parsed = urlparse(origin)
|
||||
require(parsed.scheme == "https" and parsed.hostname and parsed.port in (None, 443)
|
||||
and parsed.path in ("", "/") and not parsed.query and not parsed.fragment
|
||||
and parsed.username is None and parsed.password is None,
|
||||
"GUESTOPS_ORIGIN must be an HTTPS origin without credentials or a path.")
|
||||
context = ssl.create_default_context()
|
||||
with socket.create_connection((parsed.hostname, 443), timeout=10) as connection:
|
||||
with context.wrap_socket(connection, server_hostname=parsed.hostname) as secured:
|
||||
certificate = secured.getpeercert()
|
||||
expires = dt.datetime.strptime(certificate["notAfter"], "%b %d %H:%M:%S %Y %Z").replace(
|
||||
tzinfo=dt.timezone.utc)
|
||||
with urllib.request.urlopen(origin.rstrip("/") + "/health/ready", timeout=15,
|
||||
context=context) as response:
|
||||
ready = response.status == 200 and json.load(response) == {"status": "ready"}
|
||||
return {"ready": ready, "certificateDaysRemaining": max(0, int((expires - now).total_seconds() // 86400))}
|
||||
|
||||
|
||||
def parse_compose(value: bytes) -> dict[str, dict[str, str]]:
|
||||
text = value.decode("utf-8", "strict").strip()
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
rows = parsed if isinstance(parsed, list) else [parsed]
|
||||
except json.JSONDecodeError:
|
||||
rows = [json.loads(line) for line in text.splitlines() if line.strip()]
|
||||
result = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
service = str(row.get("Service", ""))
|
||||
if service in {"api", "worker", "mongo"}:
|
||||
result[service] = {
|
||||
"state": str(row.get("State", "unknown")).lower(),
|
||||
"health": str(row.get("Health", "none") or "none").lower(),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def worker_heartbeat_age(root: Path, now: dt.datetime) -> int | None:
|
||||
output = run(["docker", "compose", "exec", "-T", "mongo", "mongosh", "--quiet",
|
||||
"--nodb", "--eval", AUTH + HEARTBEAT], root).decode("utf-8", "strict").strip()
|
||||
value = json.loads(output)
|
||||
if value is None:
|
||||
return None
|
||||
require(isinstance(value, str), "Worker heartbeat response was invalid.")
|
||||
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
require(parsed.tzinfo is not None, "Worker heartbeat must contain a timezone.")
|
||||
return max(0, int((now - parsed.astimezone(dt.timezone.utc)).total_seconds()))
|
||||
|
||||
|
||||
def build_status(root: Path, backup_directory: Path, origin: str,
|
||||
now: dt.datetime | None = None) -> tuple[dict[str, object], list[str]]:
|
||||
moment = now or utc_now()
|
||||
errors: list[str] = []
|
||||
try:
|
||||
https = https_status(origin, moment)
|
||||
except Exception:
|
||||
https = {"ready": False, "certificateDaysRemaining": None}
|
||||
errors.append("https-probe-failed")
|
||||
try:
|
||||
containers = parse_compose(run(["docker", "compose", "ps", "--format", "json"], root))
|
||||
if set(containers) != {"api", "worker", "mongo"}:
|
||||
errors.append("container-set-incomplete")
|
||||
except Exception:
|
||||
containers = {}
|
||||
errors.append("container-probe-failed")
|
||||
try:
|
||||
heartbeat_age = worker_heartbeat_age(root, moment)
|
||||
if heartbeat_age is None:
|
||||
errors.append("worker-heartbeat-missing")
|
||||
except Exception:
|
||||
heartbeat_age = None
|
||||
errors.append("worker-heartbeat-probe-failed")
|
||||
disk = shutil.disk_usage(root)
|
||||
status = {
|
||||
"schemaVersion": 1,
|
||||
"generatedAt": moment.isoformat().replace("+00:00", "Z"),
|
||||
"https": https,
|
||||
"containers": containers,
|
||||
"workerHeartbeatAgeSeconds": heartbeat_age,
|
||||
"backupAgeSeconds": age_seconds(backup_directory, BACKUP_NAME, moment),
|
||||
"verifiedTransferAgeSeconds": age_seconds(backup_directory, MARKER_NAME, moment),
|
||||
"diskFreePercent": round(disk.free * 100 / disk.total, 2),
|
||||
"persistentJournal": Path("/var/log/journal").is_dir(),
|
||||
"errors": sorted(set(errors)),
|
||||
}
|
||||
return status, errors
|
||||
|
||||
|
||||
def write_status(path: Path, status: dict[str, object]) -> None:
|
||||
require(path.is_absolute() and not path.is_symlink(),
|
||||
"Status output must be an absolute, non-symlink path.")
|
||||
parent = path.parent.resolve()
|
||||
require(parent.is_dir() and not path.parent.is_symlink(), "Status output directory must exist.")
|
||||
fd, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o644)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
json.dump(status, stream, sort_keys=True, separators=(",", ":"))
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument("--backup-directory", type=Path, default=Path("/var/backups/guestops"))
|
||||
parser.add_argument("--origin", default="https://sandbox-guestops.futuresens.co.uk")
|
||||
parser.add_argument("--output", type=Path, default=Path("/run/guestops-monitor/status.json"))
|
||||
args = parser.parse_args()
|
||||
root = args.root.resolve()
|
||||
require(root.is_dir(), "GuestOps root must exist.")
|
||||
backup_directory = args.backup_directory.resolve()
|
||||
status, errors = build_status(root, backup_directory, args.origin)
|
||||
write_status(args.output, status)
|
||||
print("GuestOps monitoring status updated." if not errors
|
||||
else "GuestOps monitoring status updated with failed probes.")
|
||||
raise SystemExit(1 if errors else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as error:
|
||||
print(f"GuestOps monitoring probe failed: {error}", file=__import__("sys").stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"system": "guestops-persistence",
|
||||
"evidenceId": "persistence",
|
||||
"releaseVersion": "0.2.0",
|
||||
"releaseCommit": "0000000000000000000000000000000000000000",
|
||||
"releaseRecordSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"archiveSha256": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"environment": "https://sandbox-guestops.futuresens.co.uk",
|
||||
"hostIdentifier": "guestops-sandbox-01",
|
||||
"images": {
|
||||
"api": {"reference": "guestops-api:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"},
|
||||
"worker": {"reference": "guestops-worker:0000000000000000000000000000000000000000", "id": "sha256:0000000000000000000000000000000000000000000000000000000000000000"}
|
||||
},
|
||||
"drillCommand": "python3 deploy/ops.py persistence-drill --confirm-restart",
|
||||
"operator": "REPLACE",
|
||||
"reviewedBy": "REPLACE",
|
||||
"startedAt": "2026-09-30T09:00:00Z",
|
||||
"endedAt": "2026-09-30T10:00:00Z",
|
||||
"reviewedAt": "2026-09-30T11:00:00Z",
|
||||
"unresolvedCriticalFindings": 1,
|
||||
"scenarios": [
|
||||
{"id": "container-recreation", "status": "not-run", "evidence": []},
|
||||
{"id": "data-protection-key", "status": "not-run", "evidence": []},
|
||||
{"id": "database-inventory", "status": "not-run", "evidence": []},
|
||||
{"id": "image-identity", "status": "not-run", "evidence": []},
|
||||
{"id": "post-reboot-persistence", "status": "not-run", "evidence": []},
|
||||
{"id": "separate-volume-layout", "status": "not-run", "evidence": []},
|
||||
{"id": "service-restart", "status": "not-run", "evidence": []}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
BACKUP_DIRECTORY=/var/backups/guestops
|
||||
BACKUP_REMOTE_HOST=restricted-store.example.invalid
|
||||
BACKUP_REMOTE_USER=guestops_upload
|
||||
BACKUP_REMOTE_DIRECTORY=/restricted/guestops/backups
|
||||
BACKUP_SSH_IDENTITY=/etc/guestops/backup-transfer.key
|
||||
BACKUP_SSH_KNOWN_HOSTS=/etc/guestops/backup-known-hosts
|
||||
@@ -0,0 +1,2 @@
|
||||
BACKUP_RECIPIENT=0123456789ABCDEF0123456789ABCDEF01234567
|
||||
BACKUP_DIRECTORY=/var/backups/guestops
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Transfer GuestOps encrypted backups to restricted storage
|
||||
Wants=network-online.target
|
||||
After=network-online.target guestops-backup.service
|
||||
ConditionPathIsDirectory=/srv/guestops
|
||||
ConditionPathIsDirectory=/var/backups/guestops
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/guestops
|
||||
EnvironmentFile=/etc/guestops/backup-transfer.env
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/deploy/backup_transfer.py --prune-verified --retention-days 7
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=full
|
||||
ReadWritePaths=/var/backups/guestops
|
||||
TimeoutStartSec=30min
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Retry GuestOps off-host backup transfer
|
||||
|
||||
[Timer]
|
||||
OnBootSec=10min
|
||||
OnUnitActiveSec=15min
|
||||
Persistent=true
|
||||
Unit=guestops-backup-transfer.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=Write non-sensitive GuestOps monitoring status
|
||||
Requires=docker.service
|
||||
After=docker.service network-online.target
|
||||
ConditionPathIsDirectory=/srv/guestops
|
||||
ConditionPathIsDirectory=/var/backups/guestops
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/guestops
|
||||
UMask=0022
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/deploy/monitor_status.py
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=strict
|
||||
RuntimeDirectory=guestops-monitor
|
||||
RuntimeDirectoryMode=0755
|
||||
RuntimeDirectoryPreserve=yes
|
||||
ReadWritePaths=/run/guestops-monitor
|
||||
TimeoutStartSec=2min
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Refresh GuestOps monitoring status each minute
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=1min
|
||||
Persistent=true
|
||||
Unit=guestops-monitor-status.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,2 @@
|
||||
# The privileged systemd probe owns Docker/database access. Zabbix reads only this aggregate JSON.
|
||||
UserParameter=guestops.status,cat /run/guestops-monitor/status.json
|
||||
Reference in New Issue
Block a user