Add Debian persistence acceptance drill
This commit is contained in:
+74
-7
@@ -55,6 +55,27 @@ def provider_configured(config, target):
|
||||
return config not in ({}, {section: {"Hotels": {}}})
|
||||
|
||||
|
||||
def persistence_layout(config):
|
||||
"""Return the resolved named volumes required to survive recreation."""
|
||||
services = config["services"]
|
||||
declared = config.get("volumes", {})
|
||||
|
||||
def volume_for(service, target):
|
||||
matches = [v for v in services[service].get("volumes", []) if v["target"] == target]
|
||||
require(len(matches) == 1 and matches[0]["type"] == "volume" and matches[0].get("source"),
|
||||
f"{service} requires one named persistent volume at {target}.")
|
||||
source = matches[0]["source"]
|
||||
require(source in declared, f"{service} volume {source} must be declared at the top level.")
|
||||
return declared[source].get("name") or source
|
||||
|
||||
api_keys = volume_for("api", "/var/lib/guestops/keys")
|
||||
worker_keys = volume_for("worker", "/var/lib/guestops/keys")
|
||||
require(api_keys == worker_keys, "API and worker key volumes differ.")
|
||||
mongo_data = volume_for("mongo", "/data/db")
|
||||
require(api_keys != mongo_data, "Database and key data require separate named volumes.")
|
||||
return {"keys": api_keys, "database": mongo_data}
|
||||
|
||||
|
||||
AUTH = 'const c=new Mongo("mongodb://127.0.0.1"); c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,process.env.MONGO_INITDB_ROOT_PASSWORD);'
|
||||
INVENTORY = 'const d=c.getDB("guestops"); print(JSON.stringify({bytes:d.stats().storageSize+d.stats().indexSize,collections:Object.fromEntries(d.getCollectionNames().filter(n=>!n.startsWith("system.")).sort().map(n=>[n,{count:d.getCollection(n).countDocuments({}),indexes:d.getCollection(n).getIndexes().map(i=>{delete i.ns;return i;}).sort((a,b)=>a.name.localeCompare(b.name))}]))}));'
|
||||
|
||||
@@ -78,15 +99,60 @@ def configuration():
|
||||
require(env.get("Keys__Path") == "/var/lib/guestops/keys", "Unexpected key directory.")
|
||||
require(services["api"]["environment"].get("ASPNETCORE_ENVIRONMENT") == "Production", "API must use Production environment.")
|
||||
require(re.fullmatch(r"https://[A-Za-z0-9.-]+", services["api"]["environment"].get("PublicUrl", "")), "PublicUrl must be an HTTPS hostname without a path.")
|
||||
key_sources = []
|
||||
for name in ("api", "worker"):
|
||||
keys = [v for v in services[name].get("volumes", []) if v["target"] == "/var/lib/guestops/keys"]
|
||||
require(len(keys) == 1 and keys[0]["type"] == "volume", "API and worker require a shared persistent key volume.")
|
||||
key_sources.append(keys[0]["source"])
|
||||
require(key_sources[0] == key_sources[1], "API and worker key volumes differ.")
|
||||
persistence_layout(config)
|
||||
return config
|
||||
|
||||
|
||||
def wait_for(action, message, attempts=60):
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
action()
|
||||
return
|
||||
except Exception:
|
||||
if attempt == attempts - 1:
|
||||
raise RuntimeError(message)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def volume_identity(name):
|
||||
require(not name.startswith("-"), "Invalid volume name.")
|
||||
details = json.loads(run(["docker", "volume", "inspect", name]))
|
||||
require(len(details) == 1 and details[0].get("Name") == name, "Named volume inspection failed.")
|
||||
return {"name": name, "driver": details[0].get("Driver"), "scope": details[0].get("Scope")}
|
||||
|
||||
|
||||
def readiness():
|
||||
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=10) as response:
|
||||
require(json.load(response) == {"status": "ready"}, "Loopback readiness failed.")
|
||||
|
||||
|
||||
def persistence_drill(args):
|
||||
require(args.confirm_restart, "Persistence drill requires --confirm-restart: all services will be restarted and application containers recreated.")
|
||||
config = configuration()
|
||||
layout = persistence_layout(config)
|
||||
before_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
|
||||
before_inventory = json.loads(mongo(INVENTORY))
|
||||
probe = compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "api", "--backup-probe").decode().strip()
|
||||
require(re.fullmatch(r"[A-Za-z0-9_-]{20,4096}", probe), "Key probe did not return a valid protected value.")
|
||||
|
||||
# Restart the database first, then its clients, so recovery is exercised in a known order.
|
||||
compose("restart", "-t", "150", "mongo")
|
||||
wait_for(lambda: mongo('const r=c.getDB("admin").runCommand({ping:1});if(!r.ok)quit(1);'), "MongoDB did not recover after restart.")
|
||||
compose("restart", "-t", "150", "api", "worker")
|
||||
wait_for(readiness, "API readiness did not recover after restart.")
|
||||
|
||||
# Recreate stateless containers as an image upgrade would, without rebuilding or changing data volumes.
|
||||
compose("up", "-d", "--no-build", "--force-recreate", "api", "worker")
|
||||
wait_for(readiness, "API readiness did not recover after container recreation.")
|
||||
compose("run", "--rm", "--no-deps", "-T", "-e", "Logging__LogLevel__Default=None", "-e", "BACKUP_PROBE=" + probe, "api", "--verify-backup-probe")
|
||||
|
||||
after_volumes = {purpose: volume_identity(name) for purpose, name in layout.items()}
|
||||
after_inventory = json.loads(mongo(INVENTORY))
|
||||
require(after_volumes == before_volumes, "A persistent volume identity changed during the drill.")
|
||||
require(after_inventory["collections"] == before_inventory["collections"], "Database collection counts or indexes changed during the drill.")
|
||||
print("Persistence drill passed: named volumes, database counts/indexes, data-protection keys and readiness survived restart and container recreation.")
|
||||
|
||||
|
||||
def preflight(args):
|
||||
config = configuration()
|
||||
env_file = ROOT / ".env"
|
||||
@@ -242,10 +308,11 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subs = parser.add_subparsers(dest="command", required=True)
|
||||
check = subs.add_parser("preflight"); check.add_argument("--offline", action="store_true")
|
||||
persistence = subs.add_parser("persistence-drill"); persistence.add_argument("--confirm-restart", action="store_true")
|
||||
save = subs.add_parser("backup"); save.add_argument("--recipient", required=True); save.add_argument("--output", required=True); save.add_argument("--confirm-maintenance", action="store_true")
|
||||
drill = subs.add_parser("restore-drill"); drill.add_argument("backup"); drill.add_argument("--api-image", required=True); drill.add_argument("--mongo-image", default="mongo:8.0")
|
||||
args = parser.parse_args()
|
||||
{"preflight": preflight, "backup": backup, "restore-drill": restore_drill}[args.command](args)
|
||||
{"preflight": preflight, "persistence-drill": persistence_drill, "backup": backup, "restore-drill": restore_drill}[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user