345 lines
21 KiB
Python
345 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""GuestOps dedicated-Compose operations. Never prints credentials or provider data."""
|
|
import argparse
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FILES = {"mongo.archive.gz", "keys.tar.gz", "configuration.json", "manifest.json"}
|
|
LIMIT = 8 * 1024**3
|
|
|
|
|
|
def require(condition, message):
|
|
if not condition:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def run(args, *, output=None, input_file=None, timeout=900):
|
|
# Provider output may contain secrets; errors identify only the program.
|
|
result = subprocess.run(args, cwd=ROOT, stdin=input_file or subprocess.DEVNULL,
|
|
stdout=output or subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
require(result.returncode == 0, f"{args[0]} step failed. Check the private operator environment; no command output was logged.")
|
|
return result.stdout or b""
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return run(["docker", "compose", *args], **kwargs)
|
|
|
|
|
|
def digest(path):
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def image_id(image):
|
|
require(not image.startswith("-"), "Invalid image reference.")
|
|
return run(["docker", "image", "inspect", "--format", "{{.Id}}", image]).decode().strip()
|
|
|
|
|
|
def provider_configured(config, target):
|
|
section = "Pms" if target == "/run/guestops/pms.json" else "Payments"
|
|
# Only the exact shipped empty template is public; unknown settings fail closed.
|
|
return config not in ({}, {section: {"Hotels": {}}})
|
|
|
|
|
|
def persistence_layout(config):
|
|
"""Return the resolved named volumes required to survive recreation."""
|
|
services = config["services"]
|
|
declared = config.get("volumes", {})
|
|
|
|
def volume_for(service, target):
|
|
matches = [v for v in services[service].get("volumes", []) if v["target"] == target]
|
|
require(len(matches) == 1 and matches[0]["type"] == "volume" and matches[0].get("source"),
|
|
f"{service} requires one named persistent volume at {target}.")
|
|
source = matches[0]["source"]
|
|
require(source in declared, f"{service} volume {source} must be declared at the top level.")
|
|
return declared[source].get("name") or source
|
|
|
|
api_keys = volume_for("api", "/var/lib/guestops/keys")
|
|
worker_keys = volume_for("worker", "/var/lib/guestops/keys")
|
|
require(api_keys == worker_keys, "API and worker key volumes differ.")
|
|
mongo_data = volume_for("mongo", "/data/db")
|
|
require(api_keys != mongo_data, "Database and key data require separate named volumes.")
|
|
return {"keys": api_keys, "database": mongo_data}
|
|
|
|
|
|
AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);'
|
|
INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));'
|
|
|
|
|
|
def mongo(script):
|
|
return compose("exec", "-T", "mongo", "mongosh", "--quiet", "--nodb", "--eval", AUTH + script)
|
|
|
|
|
|
def configuration():
|
|
config = json.loads(compose("config", "--format", "json"))
|
|
services = config["services"]
|
|
require(set(services) == {"api", "worker", "mongo"}, "This tool supports the dedicated three-service GuestOps Compose deployment only.")
|
|
require(not services["mongo"].get("ports"), "MongoDB must not have published ports.")
|
|
ports = services["api"].get("ports", [])
|
|
require(len(ports) == 1 and ports[0].get("host_ip") == "127.0.0.1" and int(ports[0]["target"]) == 8080, "API must publish only port 8080 on loopback.")
|
|
require(not services["worker"].get("ports"), "Worker must not publish ports.")
|
|
for name in ("api", "worker"):
|
|
env = services[name]["environment"]
|
|
require(env.get("Mongo__Database") == "guestops" and "@mongo:27017/guestops?" in env.get("Mongo__ConnectionString", ""), "Backup supports only the dedicated Compose guestops database.")
|
|
require(str(env.get("Preview", "false")).lower() != "true", "Production preview is forbidden.")
|
|
require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.")
|
|
require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.")
|
|
require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.")
|
|
persistence_layout(config)
|
|
return config
|
|
|
|
|
|
def wait_for(action, message, attempts=60):
|
|
for attempt in range(attempts):
|
|
try:
|
|
action()
|
|
return
|
|
except Exception:
|
|
if attempt == attempts - 1:
|
|
raise RuntimeError(message)
|
|
time.sleep(1)
|
|
|
|
|
|
def volume_identity(name):
|
|
require(not name.startswith("-"), "Invalid volume name.")
|
|
details = json.loads(run(["docker", "volume", "inspect", name]))
|
|
require(len(details) == 1 and details[0].get("Name") == name, "Named volume inspection failed.")
|
|
return {"name": name, "driver": details[0].get("Driver"), "scope": details[0].get("Scope")}
|
|
|
|
|
|
def readiness():
|
|
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=10) as response:
|
|
require(json.load(response) == {"status": "ready"}, "Loopback readiness failed.")
|
|
|
|
|
|
def persistence_drill(args):
|
|
require(args.confirm_restart, "Persistence drill requires --confirm-restart: all services will be restarted and application containers recreated.")
|
|
config = configuration()
|
|
layout = persistence_layout(config)
|
|
before_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
|
|
before_inventory = json.loads(mongo(INVENTORY))
|
|
probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
|
|
require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
|
|
|
|
# Restart the database first, then its clients, so recovery is exercised in a known order.
|
|
compose("restart", "-t", "150", "mongo")
|
|
wait_for(lambda: mongo('const r=c.getDB("admin").runCommand({ping:1});if(!r.ok)quit(1);'), "MongoDB did not recover after restart.")
|
|
compose("restart", "-t", "150", "api", "worker")
|
|
wait_for(readiness, "API readiness did not recover after restart.")
|
|
|
|
# Recreate stateless containers as an image upgrade would, without rebuilding or changing data volumes.
|
|
compose("up", "-d", "--no-build", "--force-recreate", "api", "worker")
|
|
wait_for(readiness, "API readiness did not recover after container recreation.")
|
|
compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + probe, "api", "--verify-backup-probe")
|
|
|
|
after_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
|
|
after_inventory = json.loads(mongo(INVENTORY))
|
|
require(after_volumes == before_volumes, "A persistent volume identity changed during the drill.")
|
|
require(after_inventory["collections"] == before_inventory["collections"], "Database collection counts or indexes changed during the drill.")
|
|
print("Persistence drill passed: named volumes, database counts/indexes, data-protection keys and readiness survived restart and container recreation.")
|
|
|
|
|
|
def preflight(args):
|
|
config = configuration()
|
|
env_file = ROOT / ".env"
|
|
require(env_file.is_file() and not env_file.is_symlink(), "Create a private .env file before deployment.")
|
|
require(stat.S_IMODE(env_file.stat().st_mode) & 0o077 == 0, ".env must not be accessible to group or other users.")
|
|
require(shutil.disk_usage(ROOT).free >= 8 * 1024**3, "Keep at least 8 GiB free before deployment; allow extra room for backups.")
|
|
for name, service in config["services"].items():
|
|
image_id(service["image"])
|
|
for volume in service.get("volumes", []):
|
|
if volume["type"] == "bind":
|
|
require(Path(volume["source"]).exists(), f"A required {name} bind mount is missing.")
|
|
if volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"):
|
|
provider = Path(volume["source"])
|
|
if provider_configured(json.loads(provider.read_text()), volume["target"]):
|
|
require(stat.S_IMODE(provider.stat().st_mode) & 0o077 == 0, "Configured provider files must not be accessible to group or other users.")
|
|
if not args.offline:
|
|
url = config["services"]["api"]["environment"]["PublicUrl"]
|
|
require(re.fullmatch(r"https://[A-Za-z0-9.-]+", url), "PublicUrl must be an HTTPS hostname without a path.")
|
|
with urllib.request.urlopen(url + "/health/ready", timeout=15) as response:
|
|
require(response.url == url + "/health/ready" and json.load(response) == {"status": "ready"}, "Public HTTPS readiness failed.")
|
|
print("Preflight passed: private database, loopback API, production settings, images, mounts and disk headroom" + ("; HTTPS not checked." if args.offline else "; public HTTPS and database readiness checked."))
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def maintenance_lock():
|
|
import fcntl
|
|
with (ROOT / ".guestops-maintenance.lock").open("a") as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
yield
|
|
|
|
|
|
def backup(args):
|
|
require(args.confirm_maintenance, "Backup requires --confirm-maintenance: API and workers will briefly stop.")
|
|
require(re.fullmatch(r"[A-Fa-f0-9]{40}", args.recipient), "Use a verified full 40-character GPG recipient fingerprint.")
|
|
run(["gpg", "--batch", "--list-keys", args.recipient])
|
|
destination = Path(args.output).resolve()
|
|
require(not destination.exists(), "Backup output already exists; choose a new filename.")
|
|
require(destination.parent.is_dir(), "Create the private backup directory first.")
|
|
require(stat.S_IMODE(destination.parent.stat().st_mode) & 0o077 == 0, "Backup directory must be private (mode 700).")
|
|
config = configuration()
|
|
runtime = {}
|
|
for service in ("api", "worker", "mongo"):
|
|
cid = compose("ps", "--status", "running", "-q", service).decode().strip()
|
|
require(re.fullmatch(r"[a-f0-9]{12,64}", cid), f"Exactly one running {service} container is required.")
|
|
runtime[service] = json.loads(run(["docker", "inspect", cid]))[0]
|
|
require(shutil.disk_usage(destination.parent).free >= 3 * json.loads(mongo(INVENTORY))["bytes"] + 1024**3, "Insufficient free space for a consistent encrypted backup.")
|
|
partial = destination.with_name(destination.name + ".partial-" + uuid.uuid4().hex)
|
|
with maintenance_lock(), tempfile.TemporaryDirectory(prefix="guestops-backup-", dir=destination.parent) as folder:
|
|
folder = Path(folder)
|
|
stopped = False
|
|
ttl = None
|
|
try:
|
|
# Set before stopping so partial stop failures also attempt recovery.
|
|
stopped = True
|
|
compose("stop", "-t", "150", "api", "worker")
|
|
ttl = json.loads(mongo('print(JSON.stringify(c.getDB("admin").runCommand({getParameter:1,ttlMonitorEnabled:1}).ttlMonitorEnabled));'))
|
|
require(isinstance(ttl, bool), "Cannot determine MongoDB TTL monitor state.")
|
|
mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:false});if(!r.ok)quit(1);')
|
|
inventory = json.loads(mongo(INVENTORY))
|
|
dump = 'set -eu; case "$MONGO_INITDB_ROOT_PASSWORD" in ""|*[!0-9a-fA-F]*) exit 1;; esac; umask 077; cfg=$(mktemp); trap \'rm -f "$cfg"\' EXIT; printf \'password: "%s"\\n\' "$MONGO_INITDB_ROOT_PASSWORD" > "$cfg"; mongodump --config "$cfg" --username "$MONGO_INITDB_ROOT_USERNAME" --authenticationDatabase admin --db guestops --archive --gzip'
|
|
with (folder / "mongo.archive.gz").open("wb") as output:
|
|
compose("exec", "-T", "mongo", "sh", "-c", dump, output=output)
|
|
probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
|
|
require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
|
|
with (folder / "keys.tar.gz").open("wb") as output:
|
|
compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "tar", "api", "-C", "/var/lib/guestops/keys", "-czf", "-", ".", output=output)
|
|
mounted = {}
|
|
for volume in config["services"]["api"].get("volumes", []):
|
|
if volume["type"] == "bind" and volume["target"] in ("/run/guestops/pms.json", "/run/guestops/payments.json"):
|
|
mounted[volume["target"]] = Path(volume["source"]).read_text()
|
|
(folder / "configuration.json").write_text(json.dumps({"compose": config, "runtime": runtime, "providerFiles": mounted}))
|
|
manifest = {"format": 1, "createdAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "inventory": inventory, "probe": probe, "apiImageId": runtime["api"]["Image"], "mongoImageId": runtime["mongo"]["Image"], "sha256": {name: digest(folder / name) for name in FILES - {"manifest.json"}}}
|
|
(folder / "manifest.json").write_text(json.dumps(manifest))
|
|
finally:
|
|
try:
|
|
if ttl is not None:
|
|
mongo('const r=c.getDB("admin").runCommand({setParameter:1,ttlMonitorEnabled:' + str(ttl).lower() + '});if(!r.ok)quit(1);')
|
|
finally:
|
|
if stopped:
|
|
compose("start", "api", "worker")
|
|
try:
|
|
with tarfile.open(folder / "bundle.tar", "w") as archive:
|
|
for name in sorted(FILES):
|
|
archive.add(folder / name, arcname=name, recursive=False)
|
|
run(["gpg", "--batch", "--trust-model", "always", "--recipient", args.recipient, "--output", str(partial), "--encrypt", str(folder / "bundle.tar")])
|
|
os.link(partial, destination) # Atomic publication, never overwrite an existing backup.
|
|
finally:
|
|
partial.unlink(missing_ok=True)
|
|
print("Encrypted backup complete. Copy it off-server and perform a restore drill.")
|
|
|
|
|
|
def scheduled_backup(args):
|
|
require(args.confirm_maintenance, "Scheduled backup requires --confirm-maintenance because API and workers will briefly stop.")
|
|
recipient = os.environ.get("BACKUP_RECIPIENT", "")
|
|
directory_value = os.environ.get("BACKUP_DIRECTORY", "")
|
|
require(re.fullmatch(r"[A-Fa-f0-9]{40}", recipient), "BACKUP_RECIPIENT must be a verified full GPG fingerprint.")
|
|
require(directory_value != "", "BACKUP_DIRECTORY must name the private off-checkout staging directory.")
|
|
requested_directory = Path(directory_value)
|
|
require(requested_directory.is_absolute() and not requested_directory.is_symlink(), "BACKUP_DIRECTORY must be an absolute, non-symlink path.")
|
|
directory = requested_directory.resolve()
|
|
require(directory.is_dir(), "BACKUP_DIRECTORY must be an existing real directory.")
|
|
require(ROOT not in directory.parents and directory != ROOT, "Scheduled backups must be staged outside the application checkout.")
|
|
require(stat.S_IMODE(directory.stat().st_mode) & 0o077 == 0, "BACKUP_DIRECTORY must be private (mode 700).")
|
|
timestamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
|
|
destination = directory / f"guestops-{timestamp}.tar.gpg"
|
|
backup(argparse.Namespace(confirm_maintenance=True, recipient=recipient, output=str(destination)))
|
|
print(f"Scheduled backup staged as {destination.name}. Off-host transfer and alert verification remain required.")
|
|
|
|
|
|
def unpack(bundle, folder):
|
|
with tarfile.open(bundle, "r:") as archive:
|
|
members = archive.getmembers()
|
|
require(len(members) == len(FILES) and {m.name for m in members} == FILES, "Unexpected backup members.")
|
|
require(all(m.isfile() and 0 <= m.size <= LIMIT for m in members) and sum(m.size for m in members) <= LIMIT, "Invalid or oversized backup members.")
|
|
for member in members:
|
|
with archive.extractfile(member) as source, (folder / member.name).open("xb") as output:
|
|
shutil.copyfileobj(source, output)
|
|
manifest = json.loads((folder / "manifest.json").read_text())
|
|
require(manifest.get("format") == 1 and set(manifest.get("sha256", {})) == FILES - {"manifest.json"}, "Unsupported backup format.")
|
|
for name, expected in manifest["sha256"].items():
|
|
require(digest(folder / name) == expected, "Backup checksum verification failed.")
|
|
return manifest
|
|
|
|
|
|
def restore_drill(args):
|
|
backup_file = Path(args.backup).resolve()
|
|
require(backup_file.is_file(), "Backup file is missing.")
|
|
with tempfile.TemporaryDirectory(prefix="guestops-restore-", dir=ROOT) as temp:
|
|
folder = Path(temp)
|
|
run(["gpg", "--batch", "--max-output", str(LIMIT), "--output", str(folder / "bundle.tar"), "--decrypt", str(backup_file)])
|
|
require((folder / "bundle.tar").stat().st_size <= LIMIT, "Decrypted bundle exceeds the 8 GiB pilot limit.")
|
|
manifest = unpack(folder / "bundle.tar", folder)
|
|
require(image_id(args.api_image) == manifest["apiImageId"] and image_id(args.mongo_image) == manifest["mongoImageId"], "Load the exact trusted API and MongoDB images used for this backup.")
|
|
require(shutil.disk_usage(ROOT).free > 3 * manifest["inventory"]["bytes"] + 1024**3, "Insufficient restore drill space.")
|
|
keys = folder / "keys"
|
|
keys.mkdir(mode=0o700)
|
|
with tarfile.open(folder / "keys.tar.gz", "r:gz") as archive:
|
|
total = 0
|
|
for member in archive:
|
|
if member.name in (".", "./") and member.isdir():
|
|
continue
|
|
name = member.name.removeprefix("./")
|
|
total += member.size
|
|
require(member.isfile() and re.fullmatch(r"[A-Za-z0-9_-]+\.xml", name) and member.size < 1024**2 and total < 16 * 1024**2, "Invalid key archive.")
|
|
with archive.extractfile(member) as source, (keys / name).open("xb") as output:
|
|
shutil.copyfileobj(source, output)
|
|
run(["docker", "run", "--rm", "--pull", "never", "--network", "none", "--user", "0:0", "--read-only", "--mount", f"type=bind,src={keys},dst=/var/lib/guestops/keys,readonly", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + manifest["probe"], args.api_image, "--verify-backup-probe"])
|
|
cid = run(["docker", "run", "-d", "--pull", "never", "--network", "none", "--memory", "1g", "--label", "guestops.restore-drill=true", args.mongo_image, "--bind_ip", "127.0.0.1", "--setParameter", "ttlMonitorEnabled=false"]).decode().strip()
|
|
require(re.fullmatch(r"[a-f0-9]{64}", cid), "Unexpected restore container identifier.")
|
|
try:
|
|
for attempt in range(30):
|
|
try:
|
|
run(["docker", "exec", cid, "mongosh", "--quiet", "--eval", "db.adminCommand({ping:1})"], timeout=10)
|
|
break
|
|
except RuntimeError:
|
|
if attempt == 29:
|
|
raise
|
|
time.sleep(1)
|
|
with (folder / "mongo.archive.gz").open("rb") as source:
|
|
run(["docker", "exec", "-i", cid, "mongorestore", "--archive", "--gzip", "--nsInclude", "guestops.*"], input_file=source)
|
|
restored = json.loads(run(["docker", "exec", cid, "mongosh", "--quiet", "--nodb", "--eval", 'const c=new Mongo("mongodb://127.0.0.1");' + INVENTORY]))
|
|
require(restored["collections"] == manifest["inventory"]["collections"], "Restored collection counts or indexes differ.")
|
|
finally:
|
|
run(["docker", "rm", "-f", "-v", cid]) # Only the exact container created above and its anonymous volumes.
|
|
print("Restore drill passed: collection counts, indexes and actual key decryption verified in isolated containers. Production was not restored or modified.")
|
|
|
|
|
|
def main():
|
|
require(os.name == "posix", "Run deployment operations on Linux.")
|
|
os.umask(0o077)
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
subs = parser.add_subparsers(dest="command", required=True)
|
|
check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
|
|
persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true")
|
|
save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
|
|
scheduled = subs.add_parser("scheduled-backup"); scheduled.add_argument("--confirm-maintenance", action="store_true")
|
|
drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
|
|
args = parser.parse_args()
|
|
{"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "scheduled-backup": scheduled_backup, "restore-drill": restore_drill}[args.command](args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
# Never include subprocess output, config values or parsed guest data.
|
|
print(str(error) if isinstance(error, RuntimeError) else "Operation failed (" + type(error).__name__ + "). No sensitive details were logged.", file=sys.stderr)
|
|
sys.exit(1)
|
|
|