Add Debian persistence acceptance drill

This commit is contained in:
wolf-demon
2026-09-29 18:51:56 +01:00
parent e843d7f292
commit ede39ec892
5 changed files with 150 additions and 9 deletions
+74 -7
View File
@@ -55,6 +55,27 @@ def provider_configured(config, target):
return config not in ({}, {section: {"Hotels": {}}})
def persistence_layout(config):
"""Return the resolved named volumes required to survive recreation."""
services = config["services"]
declared = config.get("volumes", {})
def volume_for(service, target):
matches = [v for v in services[service].get("volumes", []) if v["target"] == target]
require(len(matches) == 1 and matches[0]["type"] == "volume" and matches[0].get("source"),
f"{service} requires one named persistent volume at {target}.")
source = matches[0]["source"]
require(source in declared, f"{service} volume {source} must be declared at the top level.")
return declared[source].get("name") or source
api_keys = volume_for("api", "/var/lib/guestops/keys")
worker_keys = volume_for("worker", "/var/lib/guestops/keys")
require(api_keys == worker_keys, "API and worker key volumes differ.")
mongo_data = volume_for("mongo", "/data/db")
require(api_keys != mongo_data, "Database and key data require separate named volumes.")
return {"keys": api_keys, "database": mongo_data}
AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);'
INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));'
@@ -78,15 +99,60 @@ def configuration():
require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.")
require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.")
require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.")
key_sources = []
for name in ("api", "worker"):
keys = [v for v in services[name].get("volumes", []) if v["target"] == "/var/lib/guestops/keys"]
require(len(keys) == 1 and keys[0]["type"] == "volume", "API and worker require a shared persistent key volume.")
key_sources.append(keys[0]["source"])
require(key_sources[0] == key_sources[1], "API and worker key volumes differ.")
persistence_layout(config)
return config
def wait_for(action, message, attempts=60):
for attempt in range(attempts):
try:
action()
return
except Exception:
if attempt == attempts - 1:
raise RuntimeError(message)
time.sleep(1)
def volume_identity(name):
require(not name.startswith("-"), "Invalid volume name.")
details = json.loads(run(["docker", "volume", "inspect", name]))
require(len(details) == 1 and details[0].get("Name") == name, "Named volume inspection failed.")
return {"name": name, "driver": details[0].get("Driver"), "scope": details[0].get("Scope")}
def readiness():
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=10) as response:
require(json.load(response) == {"status": "ready"}, "Loopback readiness failed.")
def persistence_drill(args):
require(args.confirm_restart, "Persistence drill requires --confirm-restart: all services will be restarted and application containers recreated.")
config = configuration()
layout = persistence_layout(config)
before_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
before_inventory = json.loads(mongo(INVENTORY))
probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
# Restart the database first, then its clients, so recovery is exercised in a known order.
compose("restart", "-t", "150", "mongo")
wait_for(lambda: mongo('const r=c.getDB("admin").runCommand({ping:1});if(!r.ok)quit(1);'), "MongoDB did not recover after restart.")
compose("restart", "-t", "150", "api", "worker")
wait_for(readiness, "API readiness did not recover after restart.")
# Recreate stateless containers as an image upgrade would, without rebuilding or changing data volumes.
compose("up", "-d", "--no-build", "--force-recreate", "api", "worker")
wait_for(readiness, "API readiness did not recover after container recreation.")
compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + probe, "api", "--verify-backup-probe")
after_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
after_inventory = json.loads(mongo(INVENTORY))
require(after_volumes == before_volumes, "A persistent volume identity changed during the drill.")
require(after_inventory["collections"] == before_inventory["collections"], "Database collection counts or indexes changed during the drill.")
print("Persistence drill passed: named volumes, database counts/indexes, data-protection keys and readiness survived restart and container recreation.")
def preflight(args):
config = configuration()
env_file = ROOT / ".env"
@@ -242,10 +308,11 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
subs = parser.add_subparsers(dest="command", required=True)
check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true")
save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
args = parser.parse_args()
{"preflight": preflight, "backup": backup, "restore-drill": restore_drill}[args.command](args)
{"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "restore-drill": restore_drill}[args.command](args)
if __name__ == "__main__":