mirror of
https://github.com/zap-proto/zip.git
synced 2026-10-19 15:38:57 +00:00
zeekayandHanzo Dev
3d7fc4e43c
A plugin that panicked was invisible and permanent. Nothing watched the child,
so when it died cur still pointed at the corpse: every request became a
connection error dressed as a 502, Plugins() kept reporting Running, and only a
manual Reload recovered. A plugin is a separate process precisely so a panic
cannot take the host down — which is only true if the host notices.
It notices now. The supervisor sees the exit, stops serving from the dead
instance (503 "deployed but down", the truth, rather than 502 blaming an
upstream that no longer exists), and restarts with capped backoff. An instance
retired by Reload or Unload also exits; a compare-and-swap tells the two apart,
so only a death that was still current counts as a crash. Restarts is reported
separately from Reloads — deliberate swaps and crashes should never be one
number, and a climbing Restarts is a crash loop.
Proven, not argued: TestPlugins_SurvivesPanic panics the plugin on a goroutine,
where no recover() can catch it, and asserts the host survives, the pid changes,
Restarts increments, and the same routes serve again with no re-registration.
Child exit is now a closed channel rather than a single-value one, because the
supervisor, a drain and Shutdown all need to observe the same exit and the first
reader would otherwise starve the rest.
Also adds per-plugin resource accounting, read from /proc:
Usage{CPU, RSS, Threads, FDs} // measured, not self-reported
This is the observability a monolith cannot offer at any price. One process has
one RSS and one CPU clock, and nothing inside it can be attributed to a
subsystem — "which app is eating the memory" is an argument. Running a service
as its own process makes it a question with an exact answer. Measured on a live
plugin: rss=12.7MB threads=5 fds=7.
And plugin output is tagged with the plugin name. An untagged line in a merged
stream is worse than useless: you can see something is wrong and not what.
Co-authored-by: Hanzo Dev <[email protected]>
63 lines
1.9 KiB
Go
63 lines
1.9 KiB
Go
//go:build linux
|
|
|
|
package zip
|
|
|
|
import (
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// readUsage reports what the kernel already knows about a running plugin.
|
|
//
|
|
// This is the observability a monolith cannot give you. In one process you can
|
|
// measure the process; you cannot say which subsystem is holding the RSS or
|
|
// burning the CPU, because there is no boundary to measure at. A plugin IS a
|
|
// process, so per-plugin CPU, memory, thread and descriptor counts come from
|
|
// /proc for free — exact, continuous, and with no instrumentation in the
|
|
// plugin at all. It cannot drift from reality because it is not self-reported.
|
|
func readUsage(pid int) Usage {
|
|
var u Usage
|
|
if pid <= 0 {
|
|
return u
|
|
}
|
|
dir := "/proc/" + strconv.Itoa(pid)
|
|
|
|
// utime+stime are fields 14 and 15, and num_threads is 20 — but comm
|
|
// (field 2) is parenthesised and may itself contain spaces and parens, so
|
|
// fields are counted from the LAST ')' rather than by splitting the line.
|
|
if b, err := os.ReadFile(dir + "/stat"); err == nil {
|
|
if i := strings.LastIndexByte(string(b), ')'); i > 0 && i+2 < len(b) {
|
|
f := strings.Fields(string(b)[i+2:])
|
|
// After comm, field 3 is state, so utime/stime/num_threads land at
|
|
// offsets 11, 12 and 17 of this slice.
|
|
if len(f) > 17 {
|
|
ut, _ := strconv.ParseInt(f[11], 10, 64)
|
|
st, _ := strconv.ParseInt(f[12], 10, 64)
|
|
// USER_HZ is 100 on every Linux target we ship to.
|
|
u.CPU = time.Duration(ut+st) * 10 * time.Millisecond
|
|
u.Threads, _ = strconv.Atoi(f[17])
|
|
}
|
|
}
|
|
}
|
|
|
|
// VmRSS is the resident set — what this plugin actually costs in memory.
|
|
if b, err := os.ReadFile(dir + "/status"); err == nil {
|
|
for line := range strings.SplitSeq(string(b), "\n") {
|
|
if kb, ok := strings.CutPrefix(line, "VmRSS:"); ok {
|
|
if f := strings.Fields(kb); len(f) > 0 {
|
|
n, _ := strconv.ParseInt(f[0], 10, 64)
|
|
u.RSS = n * 1024
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if ents, err := os.ReadDir(dir + "/fd"); err == nil {
|
|
u.FDs = len(ents)
|
|
}
|
|
return u
|
|
}
|