190 lines
7.7 KiB
Python
190 lines
7.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Write non-sensitive GuestOps host status for a read-only monitoring agent."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import urllib.request
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
BACKUP_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg")
|
|
MARKER_NAME = re.compile(r"guestops-[0-9]{8}T[0-9]{6}Z\.tar\.gpg\.transferred\.json")
|
|
AUTH = ('const c=new Mongo("mongodb://127.0.0.1");'
|
|
'c.getDB("admin").auth(process.env.MONGO_INITDB_ROOT_USERNAME,'
|
|
'process.env.MONGO_INITDB_ROOT_PASSWORD);')
|
|
HEARTBEAT = ('const x=c.getDB("guestops").workerheartbeat.findOne({_id:"worker"});'
|
|
'print(JSON.stringify(x&&x.At?x.At:null));')
|
|
|
|
|
|
def require(condition: bool, message: str) -> None:
|
|
if not condition:
|
|
raise RuntimeError(message)
|
|
|
|
|
|
def run(args: list[str], root: Path, timeout: int = 30) -> bytes:
|
|
result = subprocess.run(args, cwd=root, stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
require(result.returncode == 0, f"{Path(args[0]).name} probe failed.")
|
|
return result.stdout
|
|
|
|
|
|
def utc_now() -> dt.datetime:
|
|
return dt.datetime.now(dt.timezone.utc)
|
|
|
|
|
|
def age_seconds(path: Path, pattern: re.Pattern[str], now: dt.datetime) -> int | None:
|
|
if not path.is_dir() or path.is_symlink():
|
|
return None
|
|
times = [item.stat().st_mtime for item in path.iterdir()
|
|
if item.is_file() and not item.is_symlink() and pattern.fullmatch(item.name)]
|
|
return max(0, int(now.timestamp() - max(times))) if times else None
|
|
|
|
|
|
def https_status(origin: str, now: dt.datetime) -> dict[str, object]:
|
|
parsed = urlparse(origin)
|
|
require(parsed.scheme == "https" and parsed.hostname and parsed.port in (None, 443)
|
|
and parsed.path in ("", "/") and not parsed.query and not parsed.fragment
|
|
and parsed.username is None and parsed.password is None,
|
|
"GUESTOPS_ORIGIN must be an HTTPS origin without credentials or a path.")
|
|
context = ssl.create_default_context()
|
|
with socket.create_connection((parsed.hostname, 443), timeout=10) as connection:
|
|
with context.wrap_socket(connection, server_hostname=parsed.hostname) as secured:
|
|
certificate = secured.getpeercert()
|
|
expires = dt.datetime.strptime(certificate["notAfter"], "%b %d %H:%M:%S %Y %Z").replace(
|
|
tzinfo=dt.timezone.utc)
|
|
with urllib.request.urlopen(origin.rstrip("/") + "/health/ready", timeout=15,
|
|
context=context) as response:
|
|
ready = response.status == 200 and json.load(response) == {"status": "ready"}
|
|
return {"ready": ready, "certificateDaysRemaining": max(0, int((expires - now).total_seconds() // 86400))}
|
|
|
|
|
|
def parse_compose(value: bytes) -> dict[str, dict[str, str]]:
|
|
text = value.decode("utf-8", "strict").strip()
|
|
if not text:
|
|
return {}
|
|
try:
|
|
parsed = json.loads(text)
|
|
rows = parsed if isinstance(parsed, list) else [parsed]
|
|
except json.JSONDecodeError:
|
|
rows = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
result = {}
|
|
for row in rows:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
service = str(row.get("Service", ""))
|
|
if service in {"api", "worker", "mongo"}:
|
|
result[service] = {
|
|
"state": str(row.get("State", "unknown")).lower(),
|
|
"health": str(row.get("Health", "none") or "none").lower(),
|
|
}
|
|
return result
|
|
|
|
|
|
def worker_heartbeat_age(root: Path, now: dt.datetime) -> int | None:
|
|
output = run(["docker", "compose", "exec", "-T", "mongo", "mongosh", "--quiet",
|
|
"--nodb", "--eval", AUTH + HEARTBEAT], root).decode("utf-8", "strict").strip()
|
|
value = json.loads(output)
|
|
if value is None:
|
|
return None
|
|
require(isinstance(value, str), "Worker heartbeat response was invalid.")
|
|
parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
require(parsed.tzinfo is not None, "Worker heartbeat must contain a timezone.")
|
|
return max(0, int((now - parsed.astimezone(dt.timezone.utc)).total_seconds()))
|
|
|
|
|
|
def build_status(root: Path, backup_directory: Path, origin: str,
|
|
now: dt.datetime | None = None) -> tuple[dict[str, object], list[str]]:
|
|
moment = now or utc_now()
|
|
errors: list[str] = []
|
|
try:
|
|
https = https_status(origin, moment)
|
|
except Exception:
|
|
https = {"ready": False, "certificateDaysRemaining": None}
|
|
errors.append("https-probe-failed")
|
|
try:
|
|
containers = parse_compose(run(["docker", "compose", "ps", "--format", "json"], root))
|
|
if set(containers) != {"api", "worker", "mongo"}:
|
|
errors.append("container-set-incomplete")
|
|
except Exception:
|
|
containers = {}
|
|
errors.append("container-probe-failed")
|
|
try:
|
|
heartbeat_age = worker_heartbeat_age(root, moment)
|
|
if heartbeat_age is None:
|
|
errors.append("worker-heartbeat-missing")
|
|
except Exception:
|
|
heartbeat_age = None
|
|
errors.append("worker-heartbeat-probe-failed")
|
|
disk = shutil.disk_usage(root)
|
|
status = {
|
|
"schemaVersion": 1,
|
|
"generatedAt": moment.isoformat().replace("+00:00", "Z"),
|
|
"https": https,
|
|
"containers": containers,
|
|
"workerHeartbeatAgeSeconds": heartbeat_age,
|
|
"backupAgeSeconds": age_seconds(backup_directory, BACKUP_NAME, moment),
|
|
"verifiedTransferAgeSeconds": age_seconds(backup_directory, MARKER_NAME, moment),
|
|
"diskFreePercent": round(disk.free * 100 / disk.total, 2),
|
|
"persistentJournal": Path("/var/log/journal").is_dir(),
|
|
"errors": sorted(set(errors)),
|
|
}
|
|
return status, errors
|
|
|
|
|
|
def write_status(path: Path, status: dict[str, object]) -> None:
|
|
require(path.is_absolute() and not path.is_symlink(),
|
|
"Status output must be an absolute, non-symlink path.")
|
|
parent = path.parent.resolve()
|
|
require(parent.is_dir() and not path.parent.is_symlink(), "Status output directory must exist.")
|
|
fd, temporary = tempfile.mkstemp(prefix=path.name + ".", dir=parent)
|
|
try:
|
|
os.fchmod(fd, 0o644)
|
|
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
|
json.dump(status, stream, sort_keys=True, separators=(",", ":"))
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, path)
|
|
finally:
|
|
try:
|
|
os.unlink(temporary)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1])
|
|
parser.add_argument("--backup-directory", type=Path, default=Path("/var/backups/guestops"))
|
|
parser.add_argument("--origin", default="https://sandbox-guestops.futuresens.co.uk")
|
|
parser.add_argument("--output", type=Path, default=Path("/run/guestops-monitor/status.json"))
|
|
args = parser.parse_args()
|
|
root = args.root.resolve()
|
|
require(root.is_dir(), "GuestOps root must exist.")
|
|
backup_directory = args.backup_directory.resolve()
|
|
status, errors = build_status(root, backup_directory, args.origin)
|
|
write_status(args.output, status)
|
|
print("GuestOps monitoring status updated." if not errors
|
|
else "GuestOps monitoring status updated with failed probes.")
|
|
raise SystemExit(1 if errors else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, json.JSONDecodeError) as error:
|
|
print(f"GuestOps monitoring probe failed: {error}", file=__import__("sys").stderr)
|
|
raise SystemExit(1)
|