Files
zeekayandHanzo Dev 42e34c2d41 refactor: name only Hanzo Engine in code, comments and docs
Every mention of another inference engine is rewritten to say what our code
does, or the math it computes. Reference values are "the reference value
from the published config" or "the recorded reference output", line
citations into another engine's source become the rule itself, comparisons
with other engines are dropped, and asides about other engines' bugs go.

Identifiers follow:
- CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's
  MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels'
  `hanzo_rotary`; the shuffle/ldg macros are HANZO_*.
- GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is
  `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side.
- Tests: `*_matches_vllm` and friends name what they compare against (the
  reference fixture, the tile rule).
- MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op
  is `rotary-embedding`.
- hanzo-bench's board scores ours against a "rival" whose name comes from
  the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s`
  columns; dossier.sh and spec_decode.sh take `--rival-bench`.
- install.ps1 installs this repository's hanzo-cli, as install.sh does,
  with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL.
- bench_http.py reads only the hanzo: metric families.

Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and
tools/upstream-map.sh; each ported file's copyright/licence/provenance
header; Cargo.toml dependency names; data identifiers (Hugging Face repo
IDs, fixture file names and fixture metadata keys the tests read, on-disk
paths, executable names the GPU and engine guards match on a host); model
and format names.

LLM.md says the rule once, under "One name: Hanzo Engine".


Co-authored-by: Hanzo Dev <[email protected]>
2026-10-16 20:59:50 -07:00

383 lines
14 KiB
Python

#!/usr/bin/env python3
"""hanzo-engine-guard - keep the local Hanzo engine alive, safely, anywhere.
Two bounded jobs (neither can DoS the machine):
* circuit breaker - kill the legacy server binary's recursion on sight;
if `hanzo serve` procs exceed RUNAWAY_CEILING, kill all.
* keepalive - respawn an engine port that was healthy and then died,
but ONLY while a desktop/node is running, with per-port
exponential BACKOFF and a GIVE-UP cap, replaying the
exact command last seen (never guessing).
Decomplected: policy is pure and side-effect-free (`parse_port`, `is_bomb`,
`backoff_seconds`, `over_ceiling`, `keepalive_decision`); I/O is a thin,
swappable layer; the daemon is one OS-agnostic loop (`run` -> `_tick`);
installation is a thin per-OS adapter (`install`/`uninstall`) -> systemd-user
(Linux), launchd LaunchAgent (macOS), Task Scheduler (Windows). All user-level;
no admin.
Usage: hanzo-engine-guard run | install | uninstall | status
"""
import os
import re
import subprocess
import sys
import time
import urllib.request
try:
import psutil # cross-platform proc ops; optional (Unix falls back to pgrep/pkill)
except Exception: # pragma: no cover
psutil = None
APP = "hanzo-engine-guard"
LABEL = "ai.hanzo.engine-guard" # launchd label / windows task name base
CYCLE = 5
RUNAWAY_CEILING = int(os.environ.get("HANZO_ENGINE_RUNAWAY_CEILING", "256"))
BACKOFF = [15, 30, 60, 120, 300]
MAX_FAILS = 8
ENGINE_PAT = "release/hanzo serve" # native engine; never matches BOMB_PAT
BOMB_PAT = "mistralrs-server" # the legacy server binary's executable name on disk
DESKTOP_PATS = ("hanzo-desktop", "zoo-desktop", "lux-desktop", "hanzo-node")
IS_WIN = os.name == "nt"
# ---------------------------------------------------------------- pure policy
# These functions have no I/O and no state beyond the config constants above;
# every decision the daemon makes routes through them, so they are the unit-test
# surface and the single place each rule lives.
def parse_port(cmdline):
"""The `-p <port>` an engine was launched with, or None."""
m = re.search(r"-p\s+(\d+)", cmdline)
return int(m.group(1)) if m else None
def is_bomb(name, exe, argv0):
"""True iff a process IS the legacy server binary (BOMB_PAT) - matched
PRECISELY by exe/argv0 basename or comm (`name`), NEVER by a cmdline
substring (which would wrongly flag editors, greps, or scripts that merely
mention the name). `name` also matches the 15-char-truncated Linux comm."""
return (os.path.basename(exe or "") == BOMB_PAT
or os.path.basename(argv0 or "") == BOMB_PAT
or (name or "") in (BOMB_PAT, BOMB_PAT[:15]))
def backoff_seconds(fails):
"""Seconds to wait before the next respawn after `fails` failures (clamped)."""
return BACKOFF[min(fails, len(BACKOFF) - 1)]
def over_ceiling(count, ceiling):
"""True iff a runaway proc count breaches the kill-all ceiling."""
return count > ceiling
def keepalive_decision(st, now, desktop_up, healthy, has_proc):
"""Pure per-port keepalive verdict. Returns exactly one of:
'healthy' - a live proc is answering /health; caller resets failures.
'no-owner' - no desktop/node is running; do nothing.
'gaveup' - past MAX_FAILS; do nothing (caller logs once).
'in-backoff' - too soon since the last attempt; wait.
'respawn' - down long enough with an owner present; caller respawns.
Order matters: a healthy live proc short-circuits regardless of owner, and
the no-owner gate precedes every respawn/give-up rule so a quit desktop is
never hammered.
"""
if has_proc and healthy:
return "healthy"
if not desktop_up:
return "no-owner"
if st.get("gaveup") or st.get("fails", 0) >= MAX_FAILS:
return "gaveup"
if now - st.get("last", 0) < backoff_seconds(st.get("fails", 0)):
return "in-backoff"
return "respawn"
# ---------------------------------------------------------------- thin I/O layer
def _log(msg):
if IS_WIN:
print(f"[{APP}] {msg}", flush=True)
else:
subprocess.run(["logger", "-t", APP, msg], check=False)
def _procs_matching(pat):
"""[(pid, cmdline)] of processes whose command line contains pat."""
if psutil:
out = []
for p in psutil.process_iter(["pid", "cmdline"]):
cl = " ".join(p.info.get("cmdline") or [])
if pat in cl:
out.append((p.info["pid"], cl))
return out
res = subprocess.run(["pgrep", "-af", pat], capture_output=True, text=True).stdout
out = []
for line in res.splitlines():
parts = line.split(" ", 1)
if len(parts) == 2 and "pgrep" not in parts[1]:
out.append((int(parts[0]), parts[1]))
return out
def _kill_matching(pat):
if psutil:
for pid, _ in _procs_matching(pat):
try:
psutil.Process(pid).kill()
except Exception:
pass
else:
subprocess.run(["pkill", "-9", "-f", pat], check=False)
def _bomb_procs():
"""Processes that ARE the legacy server binary, BOMB_PAT (precise: see
is_bomb). Never matched by a cmdline substring."""
if psutil:
out = []
for p in psutil.process_iter(["pid", "name", "exe", "cmdline"]):
cl = p.info.get("cmdline") or []
a0 = cl[0] if cl else ""
if is_bomb(p.info.get("name"), p.info.get("exe"), a0):
out.append((p.info["pid"], " ".join(cl)))
return out
res = subprocess.run(["pgrep", "-x", BOMB_PAT[:15]], capture_output=True, text=True).stdout
return [(int(x), "") for x in res.split()]
def _kill_bomb():
if psutil:
for pid, _ in _bomb_procs():
try:
psutil.Process(pid).kill()
except Exception:
pass
else:
subprocess.run(["pkill", "-9", "-x", BOMB_PAT[:15]], check=False)
def _any_running(pats):
for pat in pats:
if _procs_matching(pat):
return True
return False
def _proc_context(pid):
"""Capture (cwd, environ) of a live engine so a respawn replays its exact
runtime context (HF cache, CUDA libs, PATH, etc.). The command alone is not
enough - the daemon's own env is minimal and the engine would die on launch."""
try:
pr = psutil.Process(pid)
return pr.cwd(), dict(pr.environ())
except Exception:
return None, None
def _spawn_detached(argv, cwd=None, env=None):
kw = dict(stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL)
if IS_WIN:
kw["creationflags"] = subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
else:
kw["start_new_session"] = True
if cwd:
kw["cwd"] = cwd
if env:
kw["env"] = env
subprocess.Popen(argv, **kw)
def _health(port):
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) as r:
return r.status == 200
except Exception:
return False
# ---------------------------------------------------------------- daemon (run)
def _tick(state, gaveup_logged):
"""One policy cycle (no sleep). Mutates `state` (port -> {cmd,fails,last,
gaveup,cwd,env}) and `gaveup_logged`; every branch routes through a pure
policy fn, so this stays thin orchestration over mockable I/O."""
_kill_bomb()
procs = _procs_matching(ENGINE_PAT)
if over_ceiling(len(procs), RUNAWAY_CEILING):
_log(f"RUNAWAY {len(procs)} > ceiling {RUNAWAY_CEILING}; killing all engines")
_kill_matching(ENGINE_PAT)
return
live = {}
for pid, cmd in procs:
p = parse_port(cmd)
if p is not None:
live[p] = (pid, cmd)
for p, (pid, cmd) in live.items():
if _health(p):
st = state.setdefault(p, {})
st.update(cmd=cmd.split(), fails=0, gaveup=False)
cwd, env = _proc_context(pid)
if cwd:
st["cwd"] = cwd
if env:
st["env"] = env
gaveup_logged.discard(p)
now = time.time()
desktop_up = _any_running(DESKTOP_PATS)
for p, st in state.items():
if "cmd" not in st:
continue
action = keepalive_decision(st, now, desktop_up, (p in live) and _health(p), p in live)
if action == "respawn":
fails = st.get("fails", 0)
_log(f"respawning :{p} (attempt {fails + 1}/{MAX_FAILS})")
_spawn_detached(st["cmd"], st.get("cwd"), st.get("env"))
st["last"], st["fails"] = now, fails + 1
elif action == "gaveup":
if st.get("gaveup"):
if p not in gaveup_logged:
_log(f"gave up on :{p} after {MAX_FAILS} fails; manual fix needed")
gaveup_logged.add(p)
else:
st["gaveup"] = True
def run():
_log(f"started (ceiling={RUNAWAY_CEILING} backoff={BACKOFF} max_fails={MAX_FAILS} psutil={'y' if psutil else 'n'})")
state, gaveup_logged = {}, set()
while True:
_tick(state, gaveup_logged)
time.sleep(CYCLE)
# ---------------------------------------------------------------- install (per-OS)
def _self_cmd():
return [os.path.abspath(sys.executable), os.path.abspath(__file__), "run"]
def install_linux():
d = os.path.expanduser("~/.config/systemd/user")
os.makedirs(d, exist_ok=True)
unit = os.path.join(d, f"{APP}.service")
exec_start = " ".join(_self_cmd())
with open(unit, "w") as f:
f.write(f"""[Unit]
Description=Hanzo engine guard (circuit breaker + safe keepalive)
After=default.target
[Service]
ExecStart={exec_start}
Restart=always
RestartSec=3
# KillMode=process: a guard restart/stop must NOT kill the engines it spawned
# (they run in this cgroup as detached children). No MemoryMax here -- it would
# count a respawned engine's multi-GB RAM and cgroup-OOM it.
KillMode=process
OOMScoreAdjust=-700
[Install]
WantedBy=default.target
""")
subprocess.run(["loginctl", "enable-linger", os.environ.get("USER", "")], check=False)
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
subprocess.run(["systemctl", "--user", "enable", "--now", f"{APP}.service"], check=False)
print(f"installed systemd --user service: {unit}\n systemctl --user status {APP}")
def uninstall_linux():
subprocess.run(["systemctl", "--user", "disable", "--now", f"{APP}.service"], check=False)
unit = os.path.expanduser(f"~/.config/systemd/user/{APP}.service")
if os.path.exists(unit):
os.remove(unit)
subprocess.run(["systemctl", "--user", "daemon-reload"], check=False)
print("uninstalled systemd --user service")
def install_macos():
d = os.path.expanduser("~/Library/LaunchAgents")
os.makedirs(d, exist_ok=True)
plist = os.path.join(d, f"{LABEL}.plist")
args = "".join(f" <string>{a}</string>\n" for a in _self_cmd())
with open(plist, "w") as f:
f.write(f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>Label</key><string>{LABEL}</string>
<key>ProgramArguments</key><array>
{args} </array>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
<key>ProcessType</key><string>Background</string>
</dict></plist>
""")
subprocess.run(["launchctl", "unload", plist], check=False)
subprocess.run(["launchctl", "load", "-w", plist], check=False)
print(f"installed launchd LaunchAgent: {plist}\n launchctl list | grep {LABEL}")
def uninstall_macos():
plist = os.path.expanduser(f"~/Library/LaunchAgents/{LABEL}.plist")
subprocess.run(["launchctl", "unload", "-w", plist], check=False)
if os.path.exists(plist):
os.remove(plist)
print("uninstalled launchd LaunchAgent")
def install_windows():
task = "HanzoEngineGuard"
pyw = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), "pythonw.exe")
runner = pyw if os.path.exists(pyw) else os.path.abspath(sys.executable)
tr = f'"{runner}" "{os.path.abspath(__file__)}" run'
subprocess.run(["schtasks", "/Create", "/F", "/SC", "ONLOGON", "/RL", "LIMITED",
"/TN", task, "/TR", tr], check=False)
subprocess.run(["schtasks", "/Run", "/TN", task], check=False)
print(f"installed Task Scheduler task: {task} (runs at logon)\n schtasks /Query /TN {task}")
def uninstall_windows():
subprocess.run(["schtasks", "/End", "/TN", "HanzoEngineGuard"], check=False)
subprocess.run(["schtasks", "/Delete", "/F", "/TN", "HanzoEngineGuard"], check=False)
print("uninstalled Task Scheduler task")
def _dispatch(action):
plat = "windows" if IS_WIN else ("macos" if sys.platform == "darwin" else "linux")
fn = globals().get(f"{action}_{plat}")
if not fn:
sys.exit(f"{action} not supported on {plat}")
fn()
def status():
procs = _procs_matching(ENGINE_PAT)
ports = sorted({parse_port(c) for _, c in procs} - {None})
print(f"{APP}: {len(procs)} engine proc(s); ports {ports or '-'}")
for p in ports:
print(f" :{p} health={'UP' if _health(p) else 'down'}")
print(f" desktop/node running: {_any_running(DESKTOP_PATS)}")
bombs = _bomb_procs()
if bombs:
print(f" WARNING {len(bombs)} '{BOMB_PAT}' procs present (will be killed)")
def main():
action = sys.argv[1] if len(sys.argv) > 1 else "run"
if action == "run":
run()
elif action in ("install", "uninstall"):
_dispatch(action)
elif action == "status":
status()
else:
sys.exit(__doc__)
if __name__ == "__main__": # pragma: no cover
main()