mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
383 lines
14 KiB
Python
383 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""hanzo-engine-guard - keep the local Hanzo engine alive, safely, anywhere.
|
|
|
|
Two bounded jobs (neither can DoS the machine):
|
|
|
|
* circuit breaker - kill the legacy server binary's recursion on sight;
|
|
if `hanzo serve` procs exceed RUNAWAY_CEILING, kill all.
|
|
* keepalive - respawn an engine port that was healthy and then died,
|
|
but ONLY while a desktop/node is running, with per-port
|
|
exponential BACKOFF and a GIVE-UP cap, replaying the
|
|
exact command last seen (never guessing).
|
|
|
|
Decomplected: policy is pure and side-effect-free (`parse_port`, `is_bomb`,
|
|
`backoff_seconds`, `over_ceiling`, `keepalive_decision`); I/O is a thin,
|
|
swappable layer; the daemon is one OS-agnostic loop (`run` -> `_tick`);
|
|
installation is a thin per-OS adapter (`install`/`uninstall`) -> systemd-user
|
|
(Linux), launchd LaunchAgent (macOS), Task Scheduler (Windows). All user-level;
|
|
no admin.
|
|
|
|
Usage: hanzo-engine-guard run | install | uninstall | status
|
|
"""
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
try:
|
|
import psutil # cross-platform proc ops; optional (Unix falls back to pgrep/pkill)
|
|
except Exception: # pragma: no cover
|
|
psutil = None
|
|
|
|
APP = "hanzo-engine-guard"
|
|
LABEL = "ai.hanzo.engine-guard" # launchd label / windows task name base
|
|
CYCLE = 5
|
|
RUNAWAY_CEILING = int(os.environ.get("HANZO_ENGINE_RUNAWAY_CEILING", "256"))
|
|
BACKOFF = [15, 30, 60, 120, 300]
|
|
MAX_FAILS = 8
|
|
ENGINE_PAT = "release/hanzo serve" # native engine; never matches BOMB_PAT
|
|
BOMB_PAT = "mistralrs-server" # the legacy server binary's executable name on disk
|
|
DESKTOP_PATS = ("hanzo-desktop", "zoo-desktop", "lux-desktop", "hanzo-node")
|
|
|
|
IS_WIN = os.name == "nt"
|
|
|
|
|
|
# ---------------------------------------------------------------- pure policy
|
|
# These functions have no I/O and no state beyond the config constants above;
|
|
# every decision the daemon makes routes through them, so they are the unit-test
|
|
# surface and the single place each rule lives.
|
|
def parse_port(cmdline):
|
|
"""The `-p <port>` an engine was launched with, or None."""
|
|
m = re.search(r"-p\s+(\d+)", cmdline)
|
|
return int(m.group(1)) if m else None
|
|
|
|
|
|
def is_bomb(name, exe, argv0):
|
|
"""True iff a process IS the legacy server binary (BOMB_PAT) - matched
|
|
PRECISELY by exe/argv0 basename or comm (`name`), NEVER by a cmdline
|
|
substring (which would wrongly flag editors, greps, or scripts that merely
|
|
mention the name). `name` also matches the 15-char-truncated Linux comm."""
|
|
return (os.path.basename(exe or "") == BOMB_PAT
|
|
or os.path.basename(argv0 or "") == BOMB_PAT
|
|
or (name or "") in (BOMB_PAT, BOMB_PAT[:15]))
|
|
|
|
|
|
def backoff_seconds(fails):
|
|
"""Seconds to wait before the next respawn after `fails` failures (clamped)."""
|
|
return BACKOFF[min(fails, len(BACKOFF) - 1)]
|
|
|
|
|
|
def over_ceiling(count, ceiling):
|
|
"""True iff a runaway proc count breaches the kill-all ceiling."""
|
|
return count > ceiling
|
|
|
|
|
|
def keepalive_decision(st, now, desktop_up, healthy, has_proc):
|
|
"""Pure per-port keepalive verdict. Returns exactly one of:
|
|
|
|
'healthy' - a live proc is answering /health; caller resets failures.
|
|
'no-owner' - no desktop/node is running; do nothing.
|
|
'gaveup' - past MAX_FAILS; do nothing (caller logs once).
|
|
'in-backoff' - too soon since the last attempt; wait.
|
|
'respawn' - down long enough with an owner present; caller respawns.
|
|
|
|
Order matters: a healthy live proc short-circuits regardless of owner, and
|
|
the no-owner gate precedes every respawn/give-up rule so a quit desktop is
|
|
never hammered.
|
|
"""
|
|
if has_proc and healthy:
|
|
return "healthy"
|
|
if not desktop_up:
|
|
return "no-owner"
|
|
if st.get("gaveup") or st.get("fails", 0) >= MAX_FAILS:
|
|
return "gaveup"
|
|
if now - st.get("last", 0) < backoff_seconds(st.get("fails", 0)):
|
|
return "in-backoff"
|
|
return "respawn"
|
|
|
|
|
|
# ---------------------------------------------------------------- thin I/O layer
|
|
def _log(msg):
|
|
if IS_WIN:
|
|
print(f"[{APP}] {msg}", flush=True)
|
|
else:
|
|
subprocess.run(["logger", "-t", APP, msg], check=False)
|
|
|
|
|
|
def _procs_matching(pat):
|
|
"""[(pid, cmdline)] of processes whose command line contains pat."""
|
|
if psutil:
|
|
out = []
|
|
for p in psutil.process_iter(["pid", "cmdline"]):
|
|
cl = " ".join(p.info.get("cmdline") or [])
|
|
if pat in cl:
|
|
out.append((p.info["pid"], cl))
|
|
return out
|
|
res = subprocess.run(["pgrep", "-af", pat], capture_output=True, text=True).stdout
|
|
out = []
|
|
for line in res.splitlines():
|
|
parts = line.split(" ", 1)
|
|
if len(parts) == 2 and "pgrep" not in parts[1]:
|
|
out.append((int(parts[0]), parts[1]))
|
|
return out
|
|
|
|
|
|
def _kill_matching(pat):
|
|
if psutil:
|
|
for pid, _ in _procs_matching(pat):
|
|
try:
|
|
psutil.Process(pid).kill()
|
|
except Exception:
|
|
pass
|
|
else:
|
|
subprocess.run(["pkill", "-9", "-f", pat], check=False)
|
|
|
|
|
|
def _bomb_procs():
|
|
"""Processes that ARE the legacy server binary, BOMB_PAT (precise: see
|
|
is_bomb). Never matched by a cmdline substring."""
|
|
if psutil:
|
|
out = []
|
|
for p in psutil.process_iter(["pid", "name", "exe", "cmdline"]):
|
|
cl = p.info.get("cmdline") or []
|
|
a0 = cl[0] if cl else ""
|
|
if is_bomb(p.info.get("name"), p.info.get("exe"), a0):
|
|
out.append((p.info["pid"], " ".join(cl)))
|
|
return out
|
|
res = subprocess.run(["pgrep", "-x", BOMB_PAT[:15]], capture_output=True, text=True).stdout
|
|
return [(int(x), "") for x in res.split()]
|
|
|
|
|
|
def _kill_bomb():
|
|
if psutil:
|
|
for pid, _ in _bomb_procs():
|
|
try:
|
|
psutil.Process(pid).kill()
|
|
except Exception:
|
|
pass
|
|
else:
|
|
subprocess.run(["pkill", "-9", "-x", BOMB_PAT[:15]], check=False)
|
|
|
|
|
|
def _any_running(pats):
|
|
for pat in pats:
|
|
if _procs_matching(pat):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _proc_context(pid):
|
|
"""Capture (cwd, environ) of a live engine so a respawn replays its exact
|
|
runtime context (HF cache, CUDA libs, PATH, etc.). The command alone is not
|
|
enough - the daemon's own env is minimal and the engine would die on launch."""
|
|
try:
|
|
pr = psutil.Process(pid)
|
|
return pr.cwd(), dict(pr.environ())
|
|
except Exception:
|
|
return None, None
|
|
|
|
|
|
def _spawn_detached(argv, cwd=None, env=None):
|
|
kw = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
|
|
if IS_WIN:
|
|
kw["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
|
|
else:
|
|
kw["start_new_session"] = True
|
|
if cwd:
|
|
kw["cwd"] = cwd
|
|
if env:
|
|
kw["env"] = env
|
|
subprocess.Popen(argv, **kw)
|
|
|
|
|
|
def _health(port):
|
|
try:
|
|
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) as r:
|
|
return r.status == 200
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# ---------------------------------------------------------------- daemon (run)
|
|
def _tick(state, gaveup_logged):
|
|
"""One policy cycle (no sleep). Mutates `state` (port -> {cmd,fails,last,
|
|
gaveup,cwd,env}) and `gaveup_logged`; every branch routes through a pure
|
|
policy fn, so this stays thin orchestration over mockable I/O."""
|
|
_kill_bomb()
|
|
procs = _procs_matching(ENGINE_PAT)
|
|
if over_ceiling(len(procs), RUNAWAY_CEILING):
|
|
_log(f"RUNAWAY {len(procs)} > ceiling {RUNAWAY_CEILING}; killing all engines")
|
|
_kill_matching(ENGINE_PAT)
|
|
return
|
|
|
|
live = {}
|
|
for pid, cmd in procs:
|
|
p = parse_port(cmd)
|
|
if p is not None:
|
|
live[p] = (pid, cmd)
|
|
for p, (pid, cmd) in live.items():
|
|
if _health(p):
|
|
st = state.setdefault(p, {})
|
|
st.update(cmd=cmd.split(), fails=0, gaveup=False)
|
|
cwd, env = _proc_context(pid)
|
|
if cwd:
|
|
st["cwd"] = cwd
|
|
if env:
|
|
st["env"] = env
|
|
gaveup_logged.discard(p)
|
|
|
|
now = time.time()
|
|
desktop_up = _any_running(DESKTOP_PATS)
|
|
for p, st in state.items():
|
|
if "cmd" not in st:
|
|
continue
|
|
action = keepalive_decision(st, now, desktop_up, (p in live) and _health(p), p in live)
|
|
if action == "respawn":
|
|
fails = st.get("fails", 0)
|
|
_log(f"respawning :{p} (attempt {fails + 1}/{MAX_FAILS})")
|
|
_spawn_detached(st["cmd"], st.get("cwd"), st.get("env"))
|
|
st["last"], st["fails"] = now, fails + 1
|
|
elif action == "gaveup":
|
|
if st.get("gaveup"):
|
|
if p not in gaveup_logged:
|
|
_log(f"gave up on :{p} after {MAX_FAILS} fails; manual fix needed")
|
|
gaveup_logged.add(p)
|
|
else:
|
|
st["gaveup"] = True
|
|
|
|
|
|
def run():
|
|
_log(f"started (ceiling={RUNAWAY_CEILING} backoff={BACKOFF} max_fails={MAX_FAILS} psutil={'y' if psutil else 'n'})")
|
|
state, gaveup_logged = {}, set()
|
|
while True:
|
|
_tick(state, gaveup_logged)
|
|
time.sleep(CYCLE)
|
|
|
|
|
|
# ---------------------------------------------------------------- install (per-OS)
|
|
def _self_cmd():
|
|
return [os.path.abspath(sys.executable), os.path.abspath(__file__), "run"]
|
|
|
|
|
|
def install_linux():
|
|
d = os.path.expanduser("~/.config/systemd/user")
|
|
os.makedirs(d, exist_ok=True)
|
|
unit = os.path.join(d, f"{APP}.service")
|
|
exec_start = " ".join(_self_cmd())
|
|
with open(unit, "w") as f:
|
|
f.write(f"""[Unit]
|
|
Description=Hanzo engine guard (circuit breaker + safe keepalive)
|
|
After=default.target
|
|
|
|
[Service]
|
|
ExecStart={exec_start}
|
|
Restart=always
|
|
RestartSec=3
|
|
# KillMode=process: a guard restart/stop must NOT kill the engines it spawned
|
|
# (they run in this cgroup as detached children). No MemoryMax here -- it would
|
|
# count a respawned engine's multi-GB RAM and cgroup-OOM it.
|
|
KillMode=process
|
|
OOMScoreAdjust=-700
|
|
|
|
[Install]
|
|
WantedBy=default.target
|
|
""")
|
|
subprocess.run(["loginctl", "enable-linger", os.environ.get("USER", "")], check=False)
|
|
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
|
subprocess.run(["systemctl", "--user", "enable", "--now", f"{APP}.service"], check=False)
|
|
print(f"installed systemd --user service: {unit}\n systemctl --user status {APP}")
|
|
|
|
|
|
def uninstall_linux():
|
|
subprocess.run(["systemctl", "--user", "disable", "--now", f"{APP}.service"], check=False)
|
|
unit = os.path.expanduser(f"~/.config/systemd/user/{APP}.service")
|
|
if os.path.exists(unit):
|
|
os.remove(unit)
|
|
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
|
|
print("uninstalled systemd --user service")
|
|
|
|
|
|
def install_macos():
|
|
d = os.path.expanduser("~/Library/LaunchAgents")
|
|
os.makedirs(d, exist_ok=True)
|
|
plist = os.path.join(d, f"{LABEL}.plist")
|
|
args = "".join(f" <string>{a}</string>\n" for a in _self_cmd())
|
|
with open(plist, "w") as f:
|
|
f.write(f"""<?xml version="1.0" encoding="UTF-8"?>
|
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
<plist version="1.0"><dict>
|
|
<key>Label</key><string>{LABEL}</string>
|
|
<key>ProgramArguments</key><array>
|
|
{args} </array>
|
|
<key>RunAtLoad</key><true/>
|
|
<key>KeepAlive</key><true/>
|
|
<key>ProcessType</key><string>Background</string>
|
|
</dict></plist>
|
|
""")
|
|
subprocess.run(["launchctl", "unload", plist], check=False)
|
|
subprocess.run(["launchctl", "load", "-w", plist], check=False)
|
|
print(f"installed launchd LaunchAgent: {plist}\n launchctl list | grep {LABEL}")
|
|
|
|
|
|
def uninstall_macos():
|
|
plist = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist")
|
|
subprocess.run(["launchctl", "unload", "-w", plist], check=False)
|
|
if os.path.exists(plist):
|
|
os.remove(plist)
|
|
print("uninstalled launchd LaunchAgent")
|
|
|
|
|
|
def install_windows():
|
|
task = "HanzoEngineGuard"
|
|
pyw = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), "pythonw.exe")
|
|
runner = pyw if os.path.exists(pyw) else os.path.abspath(sys.executable)
|
|
tr = f'"{runner}" "{os.path.abspath(__file__)}" run'
|
|
subprocess.run(["schtasks", "/Create", "/F", "/SC", "ONLOGON", "/RL", "LIMITED",
|
|
"/TN", task, "/TR", tr], check=False)
|
|
subprocess.run(["schtasks", "/Run", "/TN", task], check=False)
|
|
print(f"installed Task Scheduler task: {task} (runs at logon)\n schtasks /Query /TN {task}")
|
|
|
|
|
|
def uninstall_windows():
|
|
subprocess.run(["schtasks", "/End", "/TN", "HanzoEngineGuard"], check=False)
|
|
subprocess.run(["schtasks", "/Delete", "/F", "/TN", "HanzoEngineGuard"], check=False)
|
|
print("uninstalled Task Scheduler task")
|
|
|
|
|
|
def _dispatch(action):
|
|
plat = "windows" if IS_WIN else ("macos" if sys.platform == "darwin" else "linux")
|
|
fn = globals().get(f"{action}_{plat}")
|
|
if not fn:
|
|
sys.exit(f"{action} not supported on {plat}")
|
|
fn()
|
|
|
|
|
|
def status():
|
|
procs = _procs_matching(ENGINE_PAT)
|
|
ports = sorted({parse_port(c) for _, c in procs} - {None})
|
|
print(f"{APP}: {len(procs)} engine proc(s); ports {ports or '-'}")
|
|
for p in ports:
|
|
print(f" :{p} health={'UP' if _health(p) else 'down'}")
|
|
print(f" desktop/node running: {_any_running(DESKTOP_PATS)}")
|
|
bombs = _bomb_procs()
|
|
if bombs:
|
|
print(f" WARNING {len(bombs)} '{BOMB_PAT}' procs present (will be killed)")
|
|
|
|
|
|
def main():
|
|
action = sys.argv[1] if len(sys.argv) > 1 else "run"
|
|
if action == "run":
|
|
run()
|
|
elif action in ("install", "uninstall"):
|
|
_dispatch(action)
|
|
elif action == "status":
|
|
status()
|
|
else:
|
|
sys.exit(__doc__)
|
|
|
|
|
|
if __name__ == "__main__": # pragma: no cover
|
|
main()
|