mirror of
https://github.com/zap-proto/zip.git
synced 2026-10-19 15:38:57 +00:00
hold the plugin ceiling at the start, not on a ticker
Warm was an argument to Evict, so the count was TRIMMED once a minute rather than held. That is a bound only while starts arrive slower than the sweep, and the thing that starts fastest is exactly the thing that needs bounding: a host whose fan-out door asks every subsystem at once started ~100 children inside 90 seconds, held all of them until the next sweep, stopped answering its own liveness probe, and was killed. The ceiling appeared only in the logs of a container that was already gone. So Warm is Config now and makeRoom applies it in startOnDemand, before another process exists: if the host is at its ceiling, the least recently used evictable plugin is stopped to make room. Age keeps its own sweep — steady state and burst are different bounds and both belong to the same policy. Adoption is what makes it possible. A plugin's `app` is the definition that Load'ed it, which for a composed service is a sub-app holding one plugin and no ceiling, so asking it "how many are running" answers 1 and "what is the budget" answers 0 — measured, and the reason a first attempt at this was inert. The root of a composition is known in exactly one place, where the generation computes its hosts, so that is where every plugin learns its owner. Both halves are proven necessary by mutation: dropping the start-path call, or dropping the adoption, each turns the burst test from peak 4 into peak 12 against a ceiling of 4. Race-clean over two runs; full suite green. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
@@ -95,6 +95,17 @@ func (a *App) build() (*generation, error) {
|
||||
g.serve = g.router.Handler()
|
||||
g.ops = composeOps(occ)
|
||||
g.hosts = computeHosts(a, occ)
|
||||
// Adopt: every plugin in this composition learns which app is the ROOT of it.
|
||||
// This is the only place that knows, and a plugin needs it to ask the two
|
||||
// questions its own definition cannot answer — how many processes are running
|
||||
// across the whole host, and what the host's ceiling is.
|
||||
for _, h := range g.hosts {
|
||||
h.plugMu.Lock()
|
||||
for _, pl := range h.plugins {
|
||||
pl.owner.Store(a)
|
||||
}
|
||||
h.plugMu.Unlock()
|
||||
}
|
||||
g.routes = len(g.router.GetRoutes(false))
|
||||
return g, nil
|
||||
}
|
||||
|
||||
@@ -45,13 +45,21 @@ func (p Plugin) idleAfter() time.Duration { return p.IdleAfter }
|
||||
//
|
||||
// One pass, no goroutine — the caller decides the cadence, so a host under test
|
||||
// can drive it directly and a host in production can put it on the ticker it
|
||||
// already has. warm is an ARGUMENT rather than a field because the number of
|
||||
// processes a host may hold is a fact about the host's memory budget, which the
|
||||
// host knows and a library cannot.
|
||||
// already has.
|
||||
//
|
||||
// THE COUNT IS NOT THIS FUNCTION'S TO ENFORCE ALONE, and an earlier version of
|
||||
// this comment claimed otherwise: warm was an ARGUMENT here, on the reasoning
|
||||
// that the number is a fact about the host and not the library. The reasoning
|
||||
// held; the placement did not. A bound consulted once a minute is outrun by
|
||||
// anything that starts faster, and a fan-out door that asks every subsystem at
|
||||
// once does — a hundred children inside ninety seconds, so the process held them
|
||||
// all until the next sweep, stopped answering its own liveness probe, and was
|
||||
// killed with the ceiling never applied. Warm is state now and makeRoom applies
|
||||
// it at the start path; this still runs it, because age and count are one sweep.
|
||||
//
|
||||
// Draining uses the plugin's own Drain, so a request in flight when the sweep
|
||||
// lands finishes on the old process exactly as it would across a Reload.
|
||||
func (a *App) Evict(warm int) int {
|
||||
func (a *App) Evict() int {
|
||||
now := time.Now()
|
||||
stopped := 0
|
||||
for _, p := range a.pluginSet() {
|
||||
@@ -59,7 +67,61 @@ func (a *App) Evict(warm int) int {
|
||||
stopped++
|
||||
}
|
||||
}
|
||||
return stopped + a.evictOver(warm, now)
|
||||
return stopped + a.evictOver(a.warm, now)
|
||||
}
|
||||
|
||||
// host is the ROOT app of the composition this plugin belongs to, which is the
|
||||
// only app that can answer how many processes are running and what the budget is.
|
||||
// Falls back to the defining app for a plugin whose composition was never built —
|
||||
// a Load'ed service used directly in a test, where the two are the same thing.
|
||||
func (p *plugin) host() *App {
|
||||
if o := p.owner.Load(); o != nil {
|
||||
return o
|
||||
}
|
||||
return p.app
|
||||
}
|
||||
|
||||
// makeRoom is the ceiling, and it runs where a process is about to exist rather
|
||||
// than on a ticker. Called from the start path with the starter's own lock held,
|
||||
// so it must never consider the starter itself: evicting p here would deadlock on
|
||||
// p.mu, and p is the one plugin about to be needed anyway.
|
||||
//
|
||||
// This is the half a sweep cannot do. Evict trims to warm once a minute, which is
|
||||
// a bound only while starts arrive slower than that. Measured on a host whose MCP
|
||||
// door asks every subsystem at once: ~100 children started inside 90 seconds, the
|
||||
// pod stopped answering its own liveness probe, and the kubelet killed it — the
|
||||
// "ceiling" observed only in the logs of a container that was already gone.
|
||||
func (a *App) makeRoom(starter *plugin) {
|
||||
if a.warm <= 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
for {
|
||||
var (
|
||||
live int
|
||||
evictable []*plugin
|
||||
)
|
||||
for _, p := range a.pluginSet() {
|
||||
if p.cur.Load() == nil {
|
||||
continue
|
||||
}
|
||||
live++
|
||||
if p != starter && p.spec.Lazy && p.spec.idleAfter() > 0 {
|
||||
evictable = append(evictable, p)
|
||||
}
|
||||
}
|
||||
// The starter is not running yet, so room for it means strictly under.
|
||||
if live < a.warm || len(evictable) == 0 {
|
||||
return
|
||||
}
|
||||
slices.SortFunc(evictable, func(x, y *plugin) int {
|
||||
return cmp.Compare(x.lastUse.Load(), y.lastUse.Load())
|
||||
})
|
||||
cold := evictable[0]
|
||||
if !cold.evict("room", now.Sub(time.Unix(0, cold.lastUse.Load()))) {
|
||||
return // already down; re-reading would spin
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pluginSet is every plugin this host and its composed hosts hold, read once
|
||||
@@ -187,7 +249,7 @@ func (p *plugin) evict(reason string, idle time.Duration) bool {
|
||||
// Reap runs Evict on a ticker until the returned function is called. A host
|
||||
// that wants the behaviour writes one line; a host that does not is unaffected,
|
||||
// because nothing here runs unless it is asked for.
|
||||
func (a *App) Reap(every time.Duration, warm int) (stop func()) {
|
||||
func (a *App) Reap(every time.Duration) (stop func()) {
|
||||
if every <= 0 {
|
||||
every = time.Minute
|
||||
}
|
||||
@@ -202,7 +264,7 @@ func (a *App) Reap(every time.Duration, warm int) (stop func()) {
|
||||
case <-done:
|
||||
return
|
||||
case <-t.C:
|
||||
a.Evict(warm)
|
||||
a.Evict()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ func TestIdle_ReclaimsRealMemory(t *testing.T) {
|
||||
warmRSS := totalRSS(t, warmPIDs)
|
||||
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if n := app.Evict(0); n != plugins {
|
||||
if n := app.Evict(); n != plugins {
|
||||
t.Fatalf("evicted %d of %d", n, plugins)
|
||||
}
|
||||
if !settles(3*time.Second, func() bool { return len(runningPIDs(t, app)) == 0 }) {
|
||||
|
||||
+5
-5
@@ -35,7 +35,7 @@ func TestIdle_EvictsAndComesBack(t *testing.T) {
|
||||
|
||||
// Fresh from a request, so it is NOT idle. An evictor that stops a plugin
|
||||
// that just served would be a request-latency bug that only shows in prod.
|
||||
if n := app.Evict(0); n != 0 {
|
||||
if n := app.Evict(); n != 0 {
|
||||
t.Fatalf("evicted %d plugins that had just served", n)
|
||||
}
|
||||
if !app.Plugins()[0].Running {
|
||||
@@ -43,7 +43,7 @@ func TestIdle_EvictsAndComesBack(t *testing.T) {
|
||||
}
|
||||
|
||||
time.Sleep(80 * time.Millisecond)
|
||||
if n := app.Evict(0); n != 1 {
|
||||
if n := app.Evict(); n != 1 {
|
||||
t.Fatalf("Evict stopped %d, want 1 after idling past IdleAfter", n)
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestIdle_ZeroNeverEvicts(t *testing.T) {
|
||||
pid := app.Plugins()[0].PID
|
||||
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if n := app.Evict(0); n != 0 {
|
||||
if n := app.Evict(); n != 0 {
|
||||
t.Fatalf("evicted %d with IdleAfter unset — zero must mean never", n)
|
||||
}
|
||||
if got := app.Plugins()[0]; !got.Running || got.PID != pid {
|
||||
@@ -120,7 +120,7 @@ func TestIdle_EagerIsNeverEvicted(t *testing.T) {
|
||||
t.Fatal("eager plugin did not serve")
|
||||
}
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
if n := app.Evict(0); n != 0 {
|
||||
if n := app.Evict(); n != 0 {
|
||||
t.Fatalf("evicted %d eager plugins — IdleAfter applies to lazy only", n)
|
||||
}
|
||||
if got := app.Plugins()[0]; !got.Running || got.PID != pid {
|
||||
@@ -146,7 +146,7 @@ func TestIdle_ReapRunsAndStops(t *testing.T) {
|
||||
if status, _ := call(t, app, "GET", "/v1/demo/version", ""); status != 200 {
|
||||
t.Fatal("first request failed")
|
||||
}
|
||||
stop := app.Reap(10*time.Millisecond, 0)
|
||||
stop := app.Reap(10 * time.Millisecond)
|
||||
if !settles(3*time.Second, func() bool { return !app.Plugins()[0].Running }) {
|
||||
t.Fatal("Reap never evicted an idle plugin")
|
||||
}
|
||||
|
||||
@@ -208,7 +208,12 @@ type plugin struct {
|
||||
prefixes []string // every subtree this plugin answers
|
||||
spec Plugin
|
||||
|
||||
app *App // for logging and supervision when started on demand
|
||||
app *App // for logging and supervision when started on demand
|
||||
// owner is the ROOT app of the composition this plugin ended up in, stamped
|
||||
// when a generation is built. app is the definition that Load'ed it, which for
|
||||
// a composed service is a sub-app holding one plugin and no ceiling — so it is
|
||||
// the wrong thing to ask "how many are running" or "what is the budget".
|
||||
owner atomic.Pointer[App]
|
||||
cur atomic.Pointer[instance]
|
||||
mu sync.Mutex
|
||||
reloads atomic.Int64
|
||||
@@ -328,6 +333,10 @@ func (p *plugin) startOnDemand() (Client, string) {
|
||||
if p.closed || p.disabled.Load() {
|
||||
return nil, ""
|
||||
}
|
||||
// The ceiling is applied HERE, before another process exists, so warm bounds
|
||||
// what this host holds rather than what a sweep restores a minute later.
|
||||
p.host().makeRoom(p)
|
||||
|
||||
in, err := start(p.spec)
|
||||
if err != nil {
|
||||
p.app.logger.Error("zip lazy plugin failed to start", "name", p.name, "err", err)
|
||||
|
||||
+106
-115
@@ -8,20 +8,20 @@ import (
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// warmHost loads n lazy plugins that never age out, so anything these tests see
|
||||
// evicted was evicted by the COUNT bound and not by IdleAfter. Every one is
|
||||
// warmed by a request, because a plugin nothing asked for is not running and
|
||||
// would not count toward the bound in the first place.
|
||||
// The ceiling is enforced where a plugin STARTS, so most of these tests never
|
||||
// call Evict: they ask for plugins and read how many processes exist. That is the
|
||||
// design — a bound restored by a sweep is not a bound, because the burst that
|
||||
// matters is over before the sweep runs.
|
||||
//
|
||||
// Dir is sockDir(t) and not t.TempDir(): a unix socket path is bounded at ~108
|
||||
// bytes, and t.TempDir() spells the TEST'S NAME into it, so a descriptive name
|
||||
// bytes and t.TempDir() spells the TEST'S NAME into it, so a descriptive name
|
||||
// pushes the child's listen past the limit and it dies with `bind: invalid
|
||||
// argument` — which surfaces as a 503 that reads exactly like a lazy plugin
|
||||
// declining to start.
|
||||
func warmHost(t *testing.T, n int, idle time.Duration) *zip.App {
|
||||
func warmHost(t *testing.T, n, warm int, idle time.Duration) *zip.App {
|
||||
t.Helper()
|
||||
bin := buildPlugin(t, "v1")
|
||||
app := zip.New(zip.Config{AppName: "host", DisableStartupMessage: true})
|
||||
app := zip.New(zip.Config{AppName: "host", DisableStartupMessage: true, Warm: warm})
|
||||
for i := range n {
|
||||
name := fmt.Sprintf("demo%d", i)
|
||||
app.Use(must(zip.Load(
|
||||
@@ -32,91 +32,115 @@ func warmHost(t *testing.T, n int, idle time.Duration) *zip.App {
|
||||
)))
|
||||
}
|
||||
t.Cleanup(func() { _ = app.Shutdown() })
|
||||
|
||||
// Warmed oldest-first with a gap, so last-use order is demo0 < demo1 < ...
|
||||
// and the coldest is deterministic rather than whatever the clock did.
|
||||
for i := range n {
|
||||
if status, _ := call(t, app, "GET", fmt.Sprintf("/v1/demo%d/version", i), ""); status != 200 {
|
||||
t.Fatalf("demo%d did not serve", i)
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
if got := len(runningPIDs(t, app)); got != n {
|
||||
t.Fatalf("warmed %d plugins, want %d", got, n)
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
// TestWarm_StopsTheColdestUntilUnderTheBound is the bound doing its job, and it
|
||||
// checks WHICH were stopped rather than only how many: a policy that evicted the
|
||||
// hottest would satisfy a count assertion and be exactly backwards.
|
||||
func TestWarm_StopsTheColdestUntilUnderTheBound(t *testing.T) {
|
||||
// ask serves one request per plugin, oldest first with a gap, so last-use order is
|
||||
// demo0 < demo1 < ... and "the coldest" is deterministic rather than whatever the
|
||||
// clock did. It returns the PEAK number of processes seen along the way, which is
|
||||
// the figure a trim-afterwards implementation gets wrong.
|
||||
func ask(t *testing.T, app *zip.App, n int) (peak int) {
|
||||
t.Helper()
|
||||
for i := range n {
|
||||
if status, body := call(t, app, "GET", fmt.Sprintf("/v1/demo%d/version", i), ""); status != 200 {
|
||||
t.Fatalf("demo%d did not serve: status=%d body=%q", i, status, body)
|
||||
}
|
||||
if live := len(runningPIDs(t, app)); live > peak {
|
||||
peak = live
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
return peak
|
||||
}
|
||||
|
||||
// TestWarm_CeilingHoldsDuringABurst is the property a sweep cannot provide, and
|
||||
// the one whose absence took a production pod down. NO sweep runs here — no Reap,
|
||||
// no Evict — so the only thing that can hold the line is the start path. The count
|
||||
// is read after EVERY request, because the end is exactly where a
|
||||
// trim-afterwards implementation also looks correct.
|
||||
func TestWarm_CeilingHoldsDuringABurst(t *testing.T) {
|
||||
const plugins, warm = 12, 4
|
||||
app := warmHost(t, plugins, warm, time.Hour) // an hour: age can reclaim nothing
|
||||
|
||||
peak := ask(t, app, plugins)
|
||||
if peak > warm {
|
||||
t.Fatalf("peak %d processes against a ceiling of %d — the bound is being "+
|
||||
"restored by a sweep instead of enforced at the start", peak, warm)
|
||||
}
|
||||
if peak != warm { // or this proves only that twelve fit
|
||||
t.Errorf("peak was %d, want exactly the ceiling %d", peak, warm)
|
||||
}
|
||||
t.Logf("%d sequential cold starts, peak live %d, ceiling %d", plugins, peak, warm)
|
||||
|
||||
// A ceiling is not an Unload: the route evicted first still answers.
|
||||
if status, _ := call(t, app, "GET", "/v1/demo0/version", ""); status != 200 {
|
||||
t.Fatal("the coldest plugin stopped answering after being evicted for room")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_KeepsTheMostRecentlyUsed checks WHICH survive, not just how many: a
|
||||
// policy that shed the hottest would satisfy a count assertion and be exactly
|
||||
// backwards.
|
||||
func TestWarm_KeepsTheMostRecentlyUsed(t *testing.T) {
|
||||
const plugins, warm = 6, 2
|
||||
app := warmHost(t, plugins, time.Hour) // an hour: nothing can age out here
|
||||
app := warmHost(t, plugins, warm, time.Hour)
|
||||
ask(t, app, plugins)
|
||||
|
||||
if n := app.Evict(warm); n != plugins-warm {
|
||||
t.Fatalf("Evict(%d) stopped %d of %d, want %d", warm, n, plugins, plugins-warm)
|
||||
}
|
||||
if !settles(3*time.Second, func() bool { return len(runningPIDs(t, app)) == warm }) {
|
||||
t.Fatalf("%d still running, want %d", len(runningPIDs(t, app)), warm)
|
||||
}
|
||||
|
||||
// The survivors must be the two most recently used, demo4 and demo5.
|
||||
live := map[string]bool{}
|
||||
for _, p := range app.Plugins() {
|
||||
if p.Running {
|
||||
live[p.Name] = true
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"demo4", "demo5"} {
|
||||
if !live[name] {
|
||||
t.Errorf("%s was evicted, but it was among the %d most recently used", name, warm)
|
||||
for _, n := range []string{"demo4", "demo5"} {
|
||||
if !live[n] {
|
||||
t.Errorf("%s is not running, but it is among the %d most recently used", n, warm)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"demo0", "demo1", "demo2", "demo3"} {
|
||||
if live[name] {
|
||||
t.Errorf("%s survived, but it was colder than the survivors", name)
|
||||
for _, n := range []string{"demo0", "demo1", "demo2", "demo3"} {
|
||||
if live[n] {
|
||||
t.Errorf("%s survived, but it is colder than the survivors", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_ZeroIsUnbounded pins the default. A host that never passes a bound
|
||||
// must keep the behaviour it had before the bound existed.
|
||||
// TestWarm_ZeroIsUnbounded pins the default: a host that states no ceiling keeps
|
||||
// the behaviour it had before one existed.
|
||||
func TestWarm_ZeroIsUnbounded(t *testing.T) {
|
||||
const plugins = 4
|
||||
app := warmHost(t, plugins, time.Hour)
|
||||
app := warmHost(t, plugins, 0, time.Hour)
|
||||
|
||||
if n := app.Evict(0); n != 0 {
|
||||
t.Fatalf("Evict(0) stopped %d — zero must mean unbounded", n)
|
||||
if peak := ask(t, app, plugins); peak != plugins {
|
||||
t.Fatalf("peak %d of %d with no ceiling — zero must mean unbounded", peak, plugins)
|
||||
}
|
||||
if got := len(runningPIDs(t, app)); got != plugins {
|
||||
t.Fatalf("%d running after Evict(0), want %d", got, plugins)
|
||||
if n := app.Evict(); n != 0 {
|
||||
t.Fatalf("Evict stopped %d with no ceiling and nothing idle", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_UnderTheBoundStopsNothing: the bound is a ceiling, not a target. A
|
||||
// host holding fewer processes than it may must not pay a cold start to reach it.
|
||||
func TestWarm_UnderTheBoundStopsNothing(t *testing.T) {
|
||||
app := warmHost(t, 2, time.Hour)
|
||||
|
||||
if n := app.Evict(5); n != 0 {
|
||||
t.Fatalf("Evict(5) stopped %d with only 2 running", n)
|
||||
// TestWarm_UnderTheCeilingStopsNothing: it is a ceiling, not a target. A host
|
||||
// holding fewer processes than it may must not pay a cold start to reach it.
|
||||
func TestWarm_UnderTheCeilingStopsNothing(t *testing.T) {
|
||||
app := warmHost(t, 2, 5, time.Hour)
|
||||
if peak := ask(t, app, 2); peak != 2 {
|
||||
t.Fatalf("peak %d, want 2", peak)
|
||||
}
|
||||
if got := len(runningPIDs(t, app)); got != 2 {
|
||||
t.Fatalf("%d running, want 2", got)
|
||||
if n := app.Evict(); n != 0 {
|
||||
t.Fatalf("Evict stopped %d with 2 running under a ceiling of 5", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_ExemptPluginsCountButAreNotStopped is the honest half of the bound.
|
||||
// IdleAfter zero means never-evict, and such a plugin holds memory like any
|
||||
// other — so it must count against the budget (or the budget is authorised
|
||||
// twice) and must still not be stopped (or "never" meant nothing).
|
||||
// TestWarm_ExemptPluginsCountButAreNotStopped is the honest half. IdleAfter zero
|
||||
// means never-evict, and such a plugin holds memory like any other — so it must
|
||||
// COUNT against the ceiling (or the budget is authorised twice) and must still not
|
||||
// be stopped (or "never" meant nothing). When the exempt alone fill the ceiling the
|
||||
// host goes OVER it rather than shedding the identity service every other call
|
||||
// goes through.
|
||||
func TestWarm_ExemptPluginsCountButAreNotStopped(t *testing.T) {
|
||||
bin := buildPlugin(t, "v1")
|
||||
app := zip.New(zip.Config{AppName: "host", DisableStartupMessage: true})
|
||||
// Two exempt (IdleAfter unset) and two evictable.
|
||||
app := zip.New(zip.Config{AppName: "host", DisableStartupMessage: true, Warm: 2})
|
||||
for _, p := range []zip.Plugin{
|
||||
{Name: "keep0", Bin: bin, Dir: sockDir(t), Lazy: true},
|
||||
{Name: "keep0", Bin: bin, Dir: sockDir(t), Lazy: true}, // IdleAfter unset: exempt
|
||||
{Name: "keep1", Bin: bin, Dir: sockDir(t), Lazy: true},
|
||||
{Name: "drop0", Bin: bin, Dir: sockDir(t), Lazy: true, IdleAfter: time.Hour},
|
||||
{Name: "drop1", Bin: bin, Dir: sockDir(t), Lazy: true, IdleAfter: time.Hour},
|
||||
@@ -124,6 +148,7 @@ func TestWarm_ExemptPluginsCountButAreNotStopped(t *testing.T) {
|
||||
app.Use(must(zip.Load(p, "/v1/"+p.Name)))
|
||||
}
|
||||
defer func() { _ = app.Shutdown() }()
|
||||
|
||||
for _, n := range []string{"keep0", "keep1", "drop0", "drop1"} {
|
||||
if status, body := call(t, app, "GET", "/v1/"+n+"/version", ""); status != 200 {
|
||||
t.Fatalf("%s did not serve: status=%d body=%q", n, status, body)
|
||||
@@ -131,79 +156,45 @@ func TestWarm_ExemptPluginsCountButAreNotStopped(t *testing.T) {
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Four running, bound of two: the only two it MAY stop are the evictable
|
||||
// ones, so it stops both and stays over the bound rather than touching the
|
||||
// exempt pair.
|
||||
if n := app.Evict(2); n != 2 {
|
||||
t.Fatalf("Evict(2) stopped %d, want 2 (both evictable)", n)
|
||||
}
|
||||
if !settles(3*time.Second, func() bool { return len(runningPIDs(t, app)) == 2 }) {
|
||||
t.Fatalf("running=%d, want the 2 exempt", len(runningPIDs(t, app)))
|
||||
}
|
||||
for _, p := range app.Plugins() {
|
||||
switch p.Name {
|
||||
case "keep0", "keep1":
|
||||
if !p.Running {
|
||||
t.Errorf("%s was evicted, but IdleAfter unset means never", p.Name)
|
||||
}
|
||||
case "drop0", "drop1":
|
||||
case "drop0":
|
||||
if p.Running {
|
||||
t.Errorf("%s survived a bound of 2 with 4 running", p.Name)
|
||||
t.Error("drop0 survived: drop1's start should have taken the coldest evictable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// And a bound BELOW the exempt count reclaims nothing further, rather than
|
||||
// looping or stopping something it may not.
|
||||
if n := app.Evict(1); n != 0 {
|
||||
t.Fatalf("Evict(1) stopped %d when only exempt plugins remain", n)
|
||||
// Two exempt plus the one evictable that just served: over the ceiling of 2,
|
||||
// and correctly so.
|
||||
if live := len(runningPIDs(t, app)); live != 3 {
|
||||
t.Fatalf("%d running, want 3 — two exempt over the ceiling plus the newest", live)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_EvictedRoutesStillAnswer: the bound must not take a route down. This
|
||||
// is what separates it from Unload, and it is the property a caller notices.
|
||||
func TestWarm_EvictedRoutesStillAnswer(t *testing.T) {
|
||||
app := warmHost(t, 4, time.Hour)
|
||||
before := map[string]int{}
|
||||
for _, p := range app.Plugins() {
|
||||
before[p.Name] = p.PID
|
||||
}
|
||||
if n := app.Evict(1); n != 3 {
|
||||
t.Fatalf("Evict(1) stopped %d of 4, want 3", n)
|
||||
}
|
||||
if !settles(3*time.Second, func() bool { return len(runningPIDs(t, app)) == 1 }) {
|
||||
t.Fatal("did not settle at the bound")
|
||||
}
|
||||
|
||||
// demo0 was the coldest, so it was certainly evicted. Asking again must
|
||||
// serve, from a DIFFERENT process.
|
||||
status, body := call(t, app, "GET", "/v1/demo0/version", "")
|
||||
if status != 200 {
|
||||
t.Fatalf("demo0 after eviction: status=%d body=%q", status, body)
|
||||
// A sweep does not shed the exempt pair either.
|
||||
app.Evict()
|
||||
if !settles(2*time.Second, func() bool { return len(runningPIDs(t, app)) == 2 }) {
|
||||
t.Fatalf("after a sweep: %d running, want the 2 exempt", len(runningPIDs(t, app)))
|
||||
}
|
||||
for _, p := range app.Plugins() {
|
||||
if p.Name == "demo0" {
|
||||
if !p.Running || p.PID == 0 {
|
||||
t.Fatalf("demo0 did not restart on demand: %+v", p)
|
||||
}
|
||||
if p.PID == before["demo0"] {
|
||||
t.Fatalf("same pid %d — nothing was actually stopped", p.PID)
|
||||
}
|
||||
if (p.Name == "keep0" || p.Name == "keep1") && !p.Running {
|
||||
t.Errorf("%s was evicted by the sweep, but it is exempt", p.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWarm_BothBoundsInOnePass: age and count are one sweep, and the count is
|
||||
// applied to what age LEFT. Without that ordering the returned figure and the
|
||||
// live set disagree.
|
||||
func TestWarm_BothBoundsInOnePass(t *testing.T) {
|
||||
const plugins, warm = 5, 3
|
||||
app := warmHost(t, plugins, 40*time.Millisecond)
|
||||
// TestWarm_AgeStillReclaimsUnderTheCeiling: the two bounds are one sweep, and age
|
||||
// reclaims plugins the ceiling had no reason to touch.
|
||||
func TestWarm_AgeStillReclaimsUnderTheCeiling(t *testing.T) {
|
||||
const plugins, warm = 3, 5 // ceiling never reached
|
||||
app := warmHost(t, plugins, warm, 40*time.Millisecond)
|
||||
ask(t, app, plugins)
|
||||
|
||||
time.Sleep(80 * time.Millisecond) // every one is now past IdleAfter
|
||||
if n := app.Evict(warm); n != plugins {
|
||||
t.Fatalf("Evict(%d) stopped %d, want all %d — age runs first and takes them all",
|
||||
warm, n, plugins)
|
||||
if n := app.Evict(); n != plugins {
|
||||
t.Fatalf("Evict stopped %d, want all %d on age alone", n, plugins)
|
||||
}
|
||||
if !settles(3*time.Second, func() bool { return len(runningPIDs(t, app)) == 0 }) {
|
||||
t.Fatalf("%d still running", len(runningPIDs(t, app)))
|
||||
|
||||
@@ -103,6 +103,16 @@ type Config struct {
|
||||
// DisableStartupMessage suppresses Fiber's startup banner.
|
||||
DisableStartupMessage bool
|
||||
|
||||
// Warm is how many plugin processes this app may hold at once. Zero means
|
||||
// unbounded, the historical behaviour. It is enforced where a plugin STARTS,
|
||||
// so it is a ceiling rather than something a sweep restores a minute later,
|
||||
// and again by [App.Evict] alongside each plugin's IdleAfter.
|
||||
//
|
||||
// The host states it because the number is a fact about the host's memory: the
|
||||
// processes are its own children in its own cgroup, and a library cannot know
|
||||
// how much of it they may have.
|
||||
Warm int
|
||||
|
||||
// ErrorHandler is the catch-all error handler. Defaults to zip.errorHandler
|
||||
// which renders {error, code, status} JSON.
|
||||
ErrorHandler fiber.ErrorHandler
|
||||
@@ -258,6 +268,12 @@ type App struct {
|
||||
open *plugin
|
||||
plugMu sync.Mutex
|
||||
|
||||
// warm is Config.Warm: how many plugin processes may run at once. It is state
|
||||
// rather than an argument to Evict because the ceiling must be known where a
|
||||
// process is STARTED — a bound only a periodic sweep consults is a bound a
|
||||
// burst outruns. See makeRoom.
|
||||
warm int
|
||||
|
||||
// mcpList is the whole tools/list array, rendered by installMCP and re-rendered
|
||||
// by any later load: own ops ++ every plugin catalogue, sorted by name. Serving
|
||||
// bytes is what makes the most-called MCP method free. Atomic because a Load
|
||||
@@ -343,6 +359,7 @@ func newApp(cfg Config) *App {
|
||||
logger: cfg.Logger,
|
||||
loader: cfg.Loader,
|
||||
born: time.Now(),
|
||||
warm: cfg.Warm,
|
||||
telemetry: newTelemetry(cfg),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user