Files
engine/engine-guard/hanzo-engine-guard

383 lines
14 KiB
Python

#!/usr/bin/env python3
"""hanzo-engine-guard - keep the local Hanzo engine alive, safely, anywhere.
Two bounded jobs (neither can DoS the machine):
* circuit breaker - kill the legacy `mistralrs-server` recursion on sight;
if `hanzo serve` procs exceed RUNAWAY_CEILING, kill all.
* keepalive - respawn an engine port that was healthy and then died,
but ONLY while a desktop/node is running, with per-port
exponential BACKOFF and a GIVE-UP cap, replaying the
exact command last seen (never guessing).
Decomplected: policy is pure and side-effect-free (`parse_port`, `is_bomb`,
`backoff_seconds`, `over_ceiling`, `keepalive_decision`); I/O is a thin,
swappable layer; the daemon is one OS-agnostic loop (`run` -> `_tick`);
installation is a thin per-OS adapter (`install`/`uninstall`) -> systemd-user
(Linux), launchd LaunchAgent (macOS), Task Scheduler (Windows). All user-level;
no admin.
Usage: hanzo-engine-guard run | install | uninstall | status
"""
import os
import re
import subprocess
import sys
import time
import urllib.request
try:
import psutil # cross-platform proc ops; optional (Unix falls back to pgrep/pkill)
except Exception: # pragma: no cover
psutil = None
APP = "hanzo-engine-guard"
LABEL = "ai.hanzo.engine-guard" # launchd label / windows task name base
CYCLE = 5
RUNAWAY_CEILING = int(os.environ.get("HANZO_ENGINE_RUNAWAY_CEILING", "256"))
BACKOFF = [15, 30, 60, 120, 300]
MAX_FAILS = 8
ENGINE_PAT = "release/hanzo serve" # native engine; never matches mistralrs
BOMB_PAT = "mistralrs-server"
DESKTOP_PATS = ("hanzo-desktop", "zoo-desktop", "lux-desktop", "hanzo-node")
IS_WIN = os.name == "nt"
# ---------------------------------------------------------------- pure policy
# These functions have no I/O and no state beyond the config constants above;
# every decision the daemon makes routes through them, so they are the unit-test
# surface and the single place each rule lives.
def parse_port(cmdline):
"""The `-p <port>` an engine was launched with, or None."""
m = re.search(r"-p\s+(\d+)", cmdline)
return int(m.group(1)) if m else None
def is_bomb(name, exe, argv0):
"""True iff a process IS the legacy mistralrs-server binary - matched
PRECISELY by exe/argv0 basename or comm (`name`), NEVER by a cmdline
substring (which would wrongly flag editors, greps, or scripts that merely
mention the name). `name` also matches the 15-char-truncated Linux comm."""
return (os.path.basename(exe or "") == BOMB_PAT
or os.path.basename(argv0 or "") == BOMB_PAT
or (name or "") in (BOMB_PAT, BOMB_PAT[:15]))
def backoff_seconds(fails):
"""Seconds to wait before the next respawn after `fails` failures (clamped)."""
return BACKOFF[min(fails, len(BACKOFF) - 1)]
def over_ceiling(count, ceiling):
"""True iff a runaway proc count breaches the kill-all ceiling."""
return count > ceiling
def keepalive_decision(st, now, desktop_up, healthy, has_proc):
"""Pure per-port keepalive verdict. Returns exactly one of:
'healthy' - a live proc is answering /health; caller resets failures.
'no-owner' - no desktop/node is running; do nothing.
'gaveup' - past MAX_FAILS; do nothing (caller logs once).
'in-backoff' - too soon since the last attempt; wait.
'respawn' - down long enough with an owner present; caller respawns.
Order matters: a healthy live proc short-circuits regardless of owner, and
the no-owner gate precedes every respawn/give-up rule so a quit desktop is
never hammered.
"""
if has_proc and healthy:
return "healthy"
if not desktop_up:
return "no-owner"
if st.get("gaveup") or st.get("fails", 0) >= MAX_FAILS:
return "gaveup"
if now - st.get("last", 0) < backoff_seconds(st.get("fails", 0)):
return "in-backoff"
return "respawn"
# ---------------------------------------------------------------- thin I/O layer
def _log(msg):
if IS_WIN:
print(f"[{APP}] {msg}", flush=True)
else:
subprocess.run(["logger", "-t", APP, msg], check=False)
def _procs_matching(pat):
"""[(pid, cmdline)] of processes whose command line contains pat."""
if psutil:
out = []
for p in psutil.process_iter(["pid", "cmdline"]):
cl = " ".join(p.info.get("cmdline") or [])
if pat in cl:
out.append((p.info["pid"], cl))
return out
res = subprocess.run(["pgrep", "-af", pat], capture_output=True, text=True).stdout
out = []
for line in res.splitlines():
parts = line.split(" ", 1)
if len(parts) == 2 and "pgrep" not in parts[1]:
out.append((int(parts[0]), parts[1]))
return out
def _kill_matching(pat):
if psutil:
for pid, _ in _procs_matching(pat):
try:
psutil.Process(pid).kill()
except Exception:
pass
else:
subprocess.run(["pkill", "-9", "-f", pat], check=False)
def _bomb_procs():
"""Processes that ARE the legacy mistralrs-server binary (precise: see
is_bomb). Never matched by a cmdline substring."""
if psutil:
out = []
for p in psutil.process_iter(["pid", "name", "exe", "cmdline"]):
cl = p.info.get("cmdline") or []
a0 = cl[0] if cl else ""
if is_bomb(p.info.get("name"), p.info.get("exe"), a0):
out.append((p.info["pid"], " ".join(cl)))
return out
res = subprocess.run(["pgrep", "-x", BOMB_PAT[:15]], capture_output=True, text=True).stdout
return [(int(x), "") for x in res.split()]
def _kill_bomb():
if psutil:
for pid, _ in _bomb_procs():
try:
psutil.Process(pid).kill()
except Exception:
pass
else:
subprocess.run(["pkill", "-9", "-x", BOMB_PAT[:15]], check=False)
def _any_running(pats):
for pat in pats:
if _procs_matching(pat):
return True
return False
def _proc_context(pid):
"""Capture (cwd, environ) of a live engine so a respawn replays its exact
runtime context (HF cache, CUDA libs, PATH, etc.). The command alone is not
enough - the daemon's own env is minimal and the engine would die on launch."""
try:
pr = psutil.Process(pid)
return pr.cwd(), dict(pr.environ())
except Exception:
return None, None
def _spawn_detached(argv, cwd=None, env=None):
kw = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
if IS_WIN:
kw["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
else:
kw["start_new_session"] = True
if cwd:
kw["cwd"] = cwd
if env:
kw["env"] = env
subprocess.Popen(argv, **kw)
def _health(port):
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) as r:
return r.status == 200
except Exception:
return False
# ---------------------------------------------------------------- daemon (run)
def _tick(state, gaveup_logged):
"""One policy cycle (no sleep). Mutates `state` (port -> {cmd,fails,last,
gaveup,cwd,env}) and `gaveup_logged`; every branch routes through a pure
policy fn, so this stays thin orchestration over mockable I/O."""
_kill_bomb()
procs = _procs_matching(ENGINE_PAT)
if over_ceiling(len(procs), RUNAWAY_CEILING):
_log(f"RUNAWAY {len(procs)} > ceiling {RUNAWAY_CEILING}; killing all engines")
_kill_matching(ENGINE_PAT)
return
live = {}
for pid, cmd in procs:
p = parse_port(cmd)
if p is not None:
live[p] = (pid, cmd)
for p, (pid, cmd) in live.items():
if _health(p):
st = state.setdefault(p, {})
st.update(cmd=cmd.split(), fails=0, gaveup=False)
cwd, env = _proc_context(pid)
if cwd:
st["cwd"] = cwd
if env:
st["env"] = env
gaveup_logged.discard(p)
now = time.time()
desktop_up = _any_running(DESKTOP_PATS)
for p, st in state.items():
if "cmd" not in st:
continue
action = keepalive_decision(st, now, desktop_up, (p in live) and _health(p), p in live)
if action == "respawn":
fails = st.get("fails", 0)
_log(f"respawning :{p} (attempt {fails + 1}/{MAX_FAILS})")
_spawn_detached(st["cmd"], st.get("cwd"), st.get("env"))
st["last"], st["fails"] = now, fails + 1
elif action == "gaveup":
if st.get("gaveup"):
if p not in gaveup_logged:
_log(f"gave up on :{p} after {MAX_FAILS} fails; manual fix needed")
gaveup_logged.add(p)
else:
st["gaveup"] = True
def run():
_log(f"started (ceiling={RUNAWAY_CEILING} backoff={BACKOFF} max_fails={MAX_FAILS} psutil={'y' if psutil else 'n'})")
state, gaveup_logged = {}, set()
while True:
_tick(state, gaveup_logged)
time.sleep(CYCLE)
# ---------------------------------------------------------------- install (per-OS)
def _self_cmd():
return [os.path.abspath(sys.executable), os.path.abspath(__file__), "run"]
def install_linux():
d = os.path.expanduser("~/.config/systemd/user")
os.makedirs(d, exist_ok=True)
unit = os.path.join(d, f"{APP}.service")
exec_start = " ".join(_self_cmd())
with open(unit, "w") as f:
f.write(f"""[Unit]
Description=Hanzo engine guard (circuit breaker + safe keepalive)
After=default.target
[Service]
ExecStart={exec_start}
Restart=always
RestartSec=3
# KillMode=process: a guard restart/stop must NOT kill the engines it spawned
# (they run in this cgroup as detached children). No MemoryMax here -- it would
# count a respawned engine's multi-GB RAM and cgroup-OOM it.
KillMode=process
OOMScoreAdjust=-700
[Install]
WantedBy=default.target
""")
subprocess.run(["loginctl", "enable-linger", os.environ.get("USER", "")], check=False)
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
subprocess.run(["systemctl", "--user", "enable", "--now", f"{APP}.service"], check=False)
print(f"installed systemd --user service: {unit}\n systemctl --user status {APP}")
def uninstall_linux():
subprocess.run(["systemctl", "--user", "disable", "--now", f"{APP}.service"], check=False)
unit = os.path.expanduser(f"~/.config/systemd/user/{APP}.service")
if os.path.exists(unit):
os.remove(unit)
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
print("uninstalled systemd --user service")
def install_macos():
d = os.path.expanduser("~/Library/LaunchAgents")
os.makedirs(d, exist_ok=True)
plist = os.path.join(d, f"{LABEL}.plist")
args = "".join(f" <string>{a}</string>\n" for a in _self_cmd())
with open(plist, "w") as f:
f.write(f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>Label</key><string>{LABEL}</string>
<key>ProgramArguments</key><array>
{args} </array>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
<key>ProcessType</key><string>Background</string>
</dict></plist>
""")
subprocess.run(["launchctl", "unload", plist], check=False)
subprocess.run(["launchctl", "load", "-w", plist], check=False)
print(f"installed launchd LaunchAgent: {plist}\n launchctl list | grep {LABEL}")
def uninstall_macos():
plist = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist")
subprocess.run(["launchctl", "unload", "-w", plist], check=False)
if os.path.exists(plist):
os.remove(plist)
print("uninstalled launchd LaunchAgent")
def install_windows():
task = "HanzoEngineGuard"
pyw = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), "pythonw.exe")
runner = pyw if os.path.exists(pyw) else os.path.abspath(sys.executable)
tr = f'"{runner}" "{os.path.abspath(__file__)}" run'
subprocess.run(["schtasks", "/Create", "/F", "/SC", "ONLOGON", "/RL", "LIMITED",
"/TN", task, "/TR", tr], check=False)
subprocess.run(["schtasks", "/Run", "/TN", task], check=False)
print(f"installed Task Scheduler task: {task} (runs at logon)\n schtasks /Query /TN {task}")
def uninstall_windows():
subprocess.run(["schtasks", "/End", "/TN", "HanzoEngineGuard"], check=False)
subprocess.run(["schtasks", "/Delete", "/F", "/TN", "HanzoEngineGuard"], check=False)
print("uninstalled Task Scheduler task")
def _dispatch(action):
plat = "windows" if IS_WIN else ("macos" if sys.platform == "darwin" else "linux")
fn = globals().get(f"{action}_{plat}")
if not fn:
sys.exit(f"{action} not supported on {plat}")
fn()
def status():
procs = _procs_matching(ENGINE_PAT)
ports = sorted({parse_port(c) for _, c in procs} - {None})
print(f"{APP}: {len(procs)} engine proc(s); ports {ports or '-'}")
for p in ports:
print(f" :{p} health={'UP' if _health(p) else 'down'}")
print(f" desktop/node running: {_any_running(DESKTOP_PATS)}")
bombs = _bomb_procs()
if bombs:
print(f" WARNING {len(bombs)} '{BOMB_PAT}' procs present (will be killed)")
def main():
action = sys.argv[1] if len(sys.argv) > 1 else "run"
if action == "run":
run()
elif action in ("install", "uninstall"):
_dispatch(action)
elif action == "status":
status()
else:
sys.exit(__doc__)
if __name__ == "__main__": # pragma: no cover
main()