mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
383 lines
14 KiB
Python
383 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""hanzo-engine-guard - keep the local Hanzo engine alive, safely, anywhere.
|
|
|
|
Two bounded jobs (neither can DoS the machine):
|
|
|
|
* circuit breaker - kill the legacy `mistralrs-server` recursion on sight;
|
|
if `hanzo serve` procs exceed RUNAWAY_CEILING, kill all.
|
|
* keepalive - respawn an engine port that was healthy and then died,
|
|
but ONLY while a desktop/node is running, with per-port
|
|
exponential BACKOFF and a GIVE-UP cap, replaying the
|
|
exact command last seen (never guessing).
|
|
|
|
Decomplected: policy is pure and side-effect-free (`parse_port`, `is_bomb`,
|
|
`backoff_seconds`, `over_ceiling`, `keepalive_decision`); I/O is a thin,
|
|
swappable layer; the daemon is one OS-agnostic loop (`run` -> `_tick`);
|
|
installation is a thin per-OS adapter (`install`/`uninstall`) -> systemd-user
|
|
(Linux), launchd LaunchAgent (macOS), Task Scheduler (Windows). All user-level;
|
|
no admin.
|
|
|
|
Usage: hanzo-engine-guard run | install | uninstall | status
|
|
"""
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
try:
|
|
import psutil # cross-platform proc ops; optional (Unix falls back to pgrep/pkill)
|
|
except Exception: # pragma: no cover
|
|
psutil = None
|
|
|
|
APP = "hanzo-engine-guard"
|
|
LABEL = "ai.hanzo.engine-guard" # launchd label / windows task name base
|
|
CYCLE = 5
|
|
RUNAWAY_CEILING = int(os.environ.get("HANZO_ENGINE_RUNAWAY_CEILING", "256"))
|
|
BACKOFF = [15, 30, 60, 120, 300]
|
|
MAX_FAILS = 8
|
|
ENGINE_PAT = "release/hanzo serve" # native engine; never matches mistralrs
|
|
BOMB_PAT = "mistralrs-server"
|
|
DESKTOP_PATS = ("hanzo-desktop", "zoo-desktop", "lux-desktop", "hanzo-node")
|
|
|
|
IS_WIN = os.name == "nt"
|
|
|
|
|
|
# ---------------------------------------------------------------- pure policy
|
|
# These functions have no I/O and no state beyond the config constants above;
|
|
# every decision the daemon makes routes through them, so they are the unit-test
|
|
# surface and the single place each rule lives.
|
|
def parse_port(cmdline):
|
|
"""The `-p <port>` an engine was launched with, or None."""
|
|
m = re.search(r"-p\s+(\d+)", cmdline)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def is_bomb(name, exe, argv0):
|
|
"""True iff a process IS the legacy mistralrs-server binary - matched
|
|
PRECISELY by exe/argv0 basename or comm (`name`), NEVER by a cmdline
|
|
substring (which would wrongly flag editors, greps, or scripts that merely
|
|
mention the name). `name` also matches the 15-char-truncated Linux comm."""
|
|
return (os.path.basename(exe or "") == BOMB_PAT
|
|
or os.path.basename(argv0 or "") == BOMB_PAT
|
|
or (name or "") in (BOMB_PAT, BOMB_PAT[:15]))
|
|
|
|
|
|
def backoff_seconds(fails):
|
|
"""Seconds to wait before the next respawn after `fails` failures (clamped)."""
|
|
return BACKOFF[min(fails, len(BACKOFF) - 1)]
|
|
|
|
|
|
def over_ceiling(count, ceiling):
|
|
"""True iff a runaway proc count breaches the kill-all ceiling."""
|
|
return count > ceiling
|
|
|
|
|
|
def keepalive_decision(st, now, desktop_up, healthy, has_proc):
|
|
"""Pure per-port keepalive verdict. Returns exactly one of:
|
|
|
|
'healthy' - a live proc is answering /health; caller resets failures.
|
|
'no-owner' - no desktop/node is running; do nothing.
|
|
'gaveup' - past MAX_FAILS; do nothing (caller logs once).
|
|
'in-backoff' - too soon since the last attempt; wait.
|
|
'respawn' - down long enough with an owner present; caller respawns.
|
|
|
|
Order matters: a healthy live proc short-circuits regardless of owner, and
|
|
the no-owner gate precedes every respawn/give-up rule so a quit desktop is
|
|
never hammered.
|
|
"""
|
|
if has_proc and healthy:
|
|
return "healthy"
|
|
if not desktop_up:
|
|
return "no-owner"
|
|
if st.get("gaveup") or st.get("fails", 0) >= MAX_FAILS:
|
|
return "gaveup"
|
|
if now - st.get("last", 0) < backoff_seconds(st.get("fails", 0)):
|
|
return "in-backoff"
|
|
return "respawn"
|
|
|
|
|
|
# ---------------------------------------------------------------- thin I/O layer
|
|
def _log(msg):
|
|
if IS_WIN:
|
|
print(f"[{APP}] {msg}", flush=True)
|
|
else:
|
|
subprocess.run(["logger", "-t", APP, msg], check=False)
|
|
|
|
|
|
def _procs_matching(pat):
|
|
"""[(pid, cmdline)] of processes whose command line contains pat."""
|
|
if psutil:
|
|
out = []
|
|
for p in psutil.process_iter(["pid", "cmdline"]):
|
|
cl = " ".join(p.info.get("cmdline") or [])
|
|
if pat in cl:
|
|
out.append((p.info["pid"], cl))
|
|
return out
|
|
res = subprocess.run(["pgrep", "-af", pat], capture_output=True, text=True).stdout
|
|
out = []
|
|
for line in res.splitlines():
|
|
parts = line.split(" ", 1)
|
|
if len(parts) == 2 and "pgrep" not in parts[1]:
|
|
out.append((int(parts[0]), parts[1]))
|
|
return out
|
|
|
|
|
|
def _kill_matching(pat):
|
|
if psutil:
|
|
for pid, _ in _procs_matching(pat):
|
|
try:
|
|
psutil.Process(pid).kill()
|
|
except Exception:
|
|
pass
|
|
else:
|
|
subprocess.run(["pkill", "-9", "-f", pat], check=False)
|
|
|
|
|
|
def _bomb_procs():
|
|
"""Processes that ARE the legacy mistralrs-server binary (precise: see
|
|
is_bomb). Never matched by a cmdline substring."""
|
|
if psutil:
|
|
out = []
|
|
for p in psutil.process_iter(["pid", "name", "exe", "cmdline"]):
|
|
cl = p.info.get("cmdline") or []
|
|
a0 = cl[0] if cl else ""
|
|
if is_bomb(p.info.get("name"), p.info.get("exe"), a0):
|
|
out.append((p.info["pid"], " ".join(cl)))
|
|
return out
|
|
res = subprocess.run(["pgrep", "-x", BOMB_PAT[:15]], capture_output=True, text=True).stdout
|
|
return [(int(x), "") for x in res.split()]
|
|
|
|
|
|
def _kill_bomb():
|
|
if psutil:
|
|
for pid, _ in _bomb_procs():
|
|
try:
|
|
psutil.Process(pid).kill()
|
|
except Exception:
|
|
pass
|
|
else:
|
|
subprocess.run(["pkill", "-9", "-x", BOMB_PAT[:15]], check=False)
|
|
|
|
|
|
def _any_running(pats):
|
|
for pat in pats:
|
|
if _procs_matching(pat):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _proc_context(pid):
|
|
"""Capture (cwd, environ) of a live engine so a respawn replays its exact
|
|
runtime context (HF cache, CUDA libs, PATH, etc.). The command alone is not
|
|
enough - the daemon's own env is minimal and the engine would die on launch."""
|
|
try:
|
|
pr = psutil.Process(pid)
|
|
return pr.cwd(), dict(pr.environ())
|
|
except Exception:
|
|
return None, None
|
|
|
|
|
|
def _spawn_detached(argv, cwd=None, env=None):
|
|
kw = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
|
|
if IS_WIN:
|
|
kw["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
|
else:
|
|
kw["start_new_session"] = True
|
|
if cwd:
|
|
kw["cwd"] = cwd
|
|
if env:
|
|
kw["env"] = env
|
|
subprocess.Popen(argv, **kw)
|
|
|
|
|
|
def _health(port):
|
|
try:
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) as r:
|
|
return r.status == 200
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------- daemon (run)
|
|
def _tick(state, gaveup_logged):
|
|
"""One policy cycle (no sleep). Mutates `state` (port -> {cmd,fails,last,
|
|
gaveup,cwd,env}) and `gaveup_logged`; every branch routes through a pure
|
|
policy fn, so this stays thin orchestration over mockable I/O."""
|
|
_kill_bomb()
|
|
procs = _procs_matching(ENGINE_PAT)
|
|
if over_ceiling(len(procs), RUNAWAY_CEILING):
|
|
_log(f"RUNAWAY {len(procs)} > ceiling {RUNAWAY_CEILING}; killing all engines")
|
|
_kill_matching(ENGINE_PAT)
|
|
return
|
|
|
|
live = {}
|
|
for pid, cmd in procs:
|
|
p = parse_port(cmd)
|
|
if p is not None:
|
|
live[p] = (pid, cmd)
|
|
for p, (pid, cmd) in live.items():
|
|
if _health(p):
|
|
st = state.setdefault(p, {})
|
|
st.update(cmd=cmd.split(), fails=0, gaveup=False)
|
|
cwd, env = _proc_context(pid)
|
|
if cwd:
|
|
st["cwd"] = cwd
|
|
if env:
|
|
st["env"] = env
|
|
gaveup_logged.discard(p)
|
|
|
|
now = time.time()
|
|
desktop_up = _any_running(DESKTOP_PATS)
|
|
for p, st in state.items():
|
|
if "cmd" not in st:
|
|
continue
|
|
action = keepalive_decision(st, now, desktop_up, (p in live) and _health(p), p in live)
|
|
if action == "respawn":
|
|
fails = st.get("fails", 0)
|
|
_log(f"respawning :{p} (attempt {fails + 1}/{MAX_FAILS})")
|
|
_spawn_detached(st["cmd"], st.get("cwd"), st.get("env"))
|
|
st["last"], st["fails"] = now, fails + 1
|
|
elif action == "gaveup":
|
|
if st.get("gaveup"):
|
|
if p not in gaveup_logged:
|
|
_log(f"gave up on :{p} after {MAX_FAILS} fails; manual fix needed")
|
|
gaveup_logged.add(p)
|
|
else:
|
|
st["gaveup"] = True
|
|
|
|
|
|
def run():
|
|
_log(f"started (ceiling={RUNAWAY_CEILING} backoff={BACKOFF} max_fails={MAX_FAILS} psutil={'y' if psutil else 'n'})")
|
|
state, gaveup_logged = {}, set()
|
|
while True:
|
|
_tick(state, gaveup_logged)
|
|
time.sleep(CYCLE)
|
|
|
|
|
|
# ---------------------------------------------------------------- install (per-OS)
|
|
def _self_cmd():
|
|
return [os.path.abspath(sys.executable), os.path.abspath(__file__), "run"]
|
|
|
|
|
|
def install_linux():
|
|
d = os.path.expanduser("~/.config/systemd/user")
|
|
os.makedirs(d, exist_ok=True)
|
|
unit = os.path.join(d, f"{APP}.service")
|
|
exec_start = " ".join(_self_cmd())
|
|
with open(unit, "w") as f:
|
|
f.write(f"""[Unit]
|
|
Description=Hanzo engine guard (circuit breaker + safe keepalive)
|
|
After=default.target
|
|
|
|
[Service]
|
|
ExecStart={exec_start}
|
|
Restart=always
|
|
RestartSec=3
|
|
# KillMode=process: a guard restart/stop must NOT kill the engines it spawned
|
|
# (they run in this cgroup as detached children). No MemoryMax here -- it would
|
|
# count a respawned engine's multi-GB RAM and cgroup-OOM it.
|
|
KillMode=process
|
|
OOMScoreAdjust=-700
|
|
|
|
[Install]
|
|
WantedBy=default.target
|
|
""")
|
|
subprocess.run(["loginctl", "enable-linger", os.environ.get("USER", "")], check=False)
|
|
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
|
subprocess.run(["systemctl", "--user", "enable", "--now", f"{APP}.service"], check=False)
|
|
print(f"installed systemd --user service: {unit}\n systemctl --user status {APP}")
|
|
|
|
|
|
def uninstall_linux():
|
|
subprocess.run(["systemctl", "--user", "disable", "--now", f"{APP}.service"], check=False)
|
|
unit = os.path.expanduser(f"~/.config/systemd/user/{APP}.service")
|
|
if os.path.exists(unit):
|
|
os.remove(unit)
|
|
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
|
print("uninstalled systemd --user service")
|
|
|
|
|
|
def install_macos():
|
|
d = os.path.expanduser("~/Library/LaunchAgents")
|
|
os.makedirs(d, exist_ok=True)
|
|
plist = os.path.join(d, f"{LABEL}.plist")
|
|
args = "".join(f" <string>{a}</string>\n" for a in _self_cmd())
|
|
with open(plist, "w") as f:
|
|
f.write(f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0"><dict>
|
|
<key>Label</key><string>{LABEL}</string>
|
|
<key>ProgramArguments</key><array>
|
|
{args} </array>
|
|
<key>RunAtLoad</key><true/>
|
|
<key>KeepAlive</key><true/>
|
|
<key>ProcessType</key><string>Background</string>
|
|
</dict></plist>
|
|
""")
|
|
subprocess.run(["launchctl", "unload", plist], check=False)
|
|
subprocess.run(["launchctl", "load", "-w", plist], check=False)
|
|
print(f"installed launchd LaunchAgent: {plist}\n launchctl list | grep {LABEL}")
|
|
|
|
|
|
def uninstall_macos():
|
|
plist = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist")
|
|
subprocess.run(["launchctl", "unload", "-w", plist], check=False)
|
|
if os.path.exists(plist):
|
|
os.remove(plist)
|
|
print("uninstalled launchd LaunchAgent")
|
|
|
|
|
|
def install_windows():
|
|
task = "HanzoEngineGuard"
|
|
pyw = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), "pythonw.exe")
|
|
runner = pyw if os.path.exists(pyw) else os.path.abspath(sys.executable)
|
|
tr = f'"{runner}" "{os.path.abspath(__file__)}" run'
|
|
subprocess.run(["schtasks", "/Create", "/F", "/SC", "ONLOGON", "/RL", "LIMITED",
|
|
"/TN", task, "/TR", tr], check=False)
|
|
subprocess.run(["schtasks", "/Run", "/TN", task], check=False)
|
|
print(f"installed Task Scheduler task: {task} (runs at logon)\n schtasks /Query /TN {task}")
|
|
|
|
|
|
def uninstall_windows():
|
|
subprocess.run(["schtasks", "/End", "/TN", "HanzoEngineGuard"], check=False)
|
|
subprocess.run(["schtasks", "/Delete", "/F", "/TN", "HanzoEngineGuard"], check=False)
|
|
print("uninstalled Task Scheduler task")
|
|
|
|
|
|
def _dispatch(action):
|
|
plat = "windows" if IS_WIN else ("macos" if sys.platform == "darwin" else "linux")
|
|
fn = globals().get(f"{action}_{plat}")
|
|
if not fn:
|
|
sys.exit(f"{action} not supported on {plat}")
|
|
fn()
|
|
|
|
|
|
def status():
|
|
procs = _procs_matching(ENGINE_PAT)
|
|
ports = sorted({parse_port(c) for _, c in procs} - {None})
|
|
print(f"{APP}: {len(procs)} engine proc(s); ports {ports or '-'}")
|
|
for p in ports:
|
|
print(f" :{p} health={'UP' if _health(p) else 'down'}")
|
|
print(f" desktop/node running: {_any_running(DESKTOP_PATS)}")
|
|
bombs = _bomb_procs()
|
|
if bombs:
|
|
print(f" WARNING {len(bombs)} '{BOMB_PAT}' procs present (will be killed)")
|
|
|
|
|
|
def main():
|
|
action = sys.argv[1] if len(sys.argv) > 1 else "run"
|
|
if action == "run":
|
|
run()
|
|
elif action in ("install", "uninstall"):
|
|
_dispatch(action)
|
|
elif action == "status":
|
|
status()
|
|
else:
|
|
sys.exit(__doc__)
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
main()
|