mirror of
https://github.com/hanzoai/base.git
synced 2026-10-21 03:41:50 +00:00
a function starts a sandbox rather than becoming one
Running a coding agent or a research pass needs the four things a function refuses: a network, a writable disk, a process, and minutes rather than a request. Those belong on the far side of an isolation boundary that is a kernel, and Hanzo Runtime is where that boundary already is. So the third capability is start. It says what to run and hands back an id; it does not wait, because a function is stopped if it overruns and a run is not. The result arrives the way any other write does — the run authenticating to Base as itself. Starting grants a function none of what the sandbox has, which is why it is safe to bind beside two reads. The org is an argument beside the spec rather than a field in it, so naming another tenant is not overridden, it cannot be written down. A run costs more than a read, so a call is bounded whatever the deployment's limits say — a limiter that ships disabled protects nobody, and one invocation otherwise starts as many as it likes. An unowned run is refused where a deployment tells orgs apart, because a collection that anyone may view carries no org to charge the run to. What the sandbox is told is the run's own, and what it says when it fails is the deployment's: a function may catch, and its caller may be anyone. The seam that is deliberately empty is the credential a run authenticates with. Passing the caller's own is the thing not to do. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
Hanzo Dev
+169
-7
@@ -1,6 +1,7 @@
|
||||
package apis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -9,6 +10,8 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/base/core"
|
||||
@@ -55,6 +58,19 @@ const (
|
||||
// functionMaxRows is the most rows one host read answers with, whatever the
|
||||
// function asked for.
|
||||
functionMaxRows = 500
|
||||
|
||||
// functionMaxRuns is the most runs one call starts, whatever it asks for.
|
||||
//
|
||||
// A rule an operator writes is what prices a run; this is what holds when
|
||||
// nobody has written one, which is every Base out of the box — rate limits
|
||||
// ship disabled, and a run is far too expensive to be unbounded until an
|
||||
// operator gets around to saying so.
|
||||
//
|
||||
// A function decides what work one request implies, and a handful covers
|
||||
// that. Fanning out wider is a program in its own right, and it belongs
|
||||
// inside a run, which has minutes for it rather than the seconds a function
|
||||
// is given.
|
||||
functionMaxRuns = 8
|
||||
)
|
||||
|
||||
func bindFunctionsApi(app core.App, rg *router.RouterGroup[*core.RequestEvent]) {
|
||||
@@ -137,11 +153,19 @@ func functionInvoke(e *core.RequestEvent) error {
|
||||
ctx, cancel := context.WithTimeout(e.Request.Context(), functionTimeout)
|
||||
defer cancel()
|
||||
|
||||
out, err := run(ctx, record.GetString(core.FieldNameSource), payload, functionHost(e, requestInfo))
|
||||
call := &invocation{ctx: ctx, e: e, info: requestInfo, collection: collection}
|
||||
|
||||
out, err := run(ctx, record.GetString(core.FieldNameSource), payload, call.host())
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return e.Error(http.StatusGatewayTimeout,
|
||||
"The function did not finish within "+functionTimeout.String()+".", nil)
|
||||
case err != nil && call.refusal() != nil:
|
||||
// The host turned this caller down and the function did not handle it,
|
||||
// so what the host decided is the answer. Reporting a server failure
|
||||
// here would describe a limit doing exactly its job as the deployment
|
||||
// being broken.
|
||||
return call.refusal()
|
||||
case err != nil:
|
||||
// The source is the deployment's, so the failure is the deployment's:
|
||||
// it is logged whole and answered as the server error it is.
|
||||
@@ -157,20 +181,158 @@ func functionInvoke(e *core.RequestEvent) error {
|
||||
return e.JSON(http.StatusOK, json.RawMessage(out))
|
||||
}
|
||||
|
||||
// functionHost is what a function may ask of the Base it is running in: that
|
||||
// Base's own records, read as its caller would read them.
|
||||
// invocation is one call: what the function running may ask for, and the one
|
||||
// answer the host is allowed to decide on the caller's behalf.
|
||||
type invocation struct {
|
||||
ctx context.Context
|
||||
e *core.RequestEvent
|
||||
info *core.RequestInfo
|
||||
collection *core.Collection
|
||||
|
||||
mu sync.Mutex
|
||||
refused error
|
||||
runs int
|
||||
}
|
||||
|
||||
// count records that this call is about to start another run, and refuses once
|
||||
// it has started [functionMaxRuns]. Only an ask that survived its own checking
|
||||
// gets here, so a malformed one costs nothing and takes nothing.
|
||||
func (c *invocation) count() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.runs >= functionMaxRuns {
|
||||
return fmt.Errorf("start: one call starts at most %d runs", functionMaxRuns)
|
||||
}
|
||||
c.runs++
|
||||
return nil
|
||||
}
|
||||
|
||||
// host is what a function may ask of the Base it is running in: that Base's own
|
||||
// records, read as its caller would read them, and somewhere that is not this
|
||||
// process to run work.
|
||||
//
|
||||
// Both reads resolve everything from the request — the collection from e.App,
|
||||
// which the credential already moved onto this org's Base, and the rule from the
|
||||
// caller's own identity — so a function renders what its caller may see and is
|
||||
// never a way around it.
|
||||
func functionHost(e *core.RequestEvent, requestInfo *core.RequestInfo) extruntime.Host {
|
||||
// never a way around it. The third name is about cost rather than sight: it
|
||||
// spends the deployment's machines, so it is counted, and it answers with a
|
||||
// name instead of a result.
|
||||
func (c *invocation) host() extruntime.Host {
|
||||
return extruntime.Host{
|
||||
"list": func(arg []byte) ([]byte, error) { return functionListRead(e.App, requestInfo, arg) },
|
||||
"one": func(arg []byte) ([]byte, error) { return functionOneRead(e.App, requestInfo, arg) },
|
||||
"list": func(arg []byte) ([]byte, error) { return functionListRead(c.e.App, c.info, arg) },
|
||||
"one": func(arg []byte) ([]byte, error) { return functionOneRead(c.e.App, c.info, arg) },
|
||||
"start": c.start,
|
||||
}
|
||||
}
|
||||
|
||||
// refuse records an answer the host reached about the CALLER — a limit they are
|
||||
// over — rather than about the function. Only the first is kept, because that
|
||||
// is the one the call stopped on.
|
||||
func (c *invocation) refuse(err error) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.refused == nil {
|
||||
c.refused = err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// refusal reports what the host decided about the caller, if it decided
|
||||
// anything.
|
||||
func (c *invocation) refusal() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.refused
|
||||
}
|
||||
|
||||
// reservedLabel prefixes what a deployment writes about a run of its own
|
||||
// accord. Whose work a sandbox is gets stamped there by whoever starts it, so a
|
||||
// function's own labels stop short of it: the org a run belongs to has one
|
||||
// source, and it is not the source that asked for the run.
|
||||
const reservedLabel = "hanzo.ai/"
|
||||
|
||||
// start begins one sandbox for the org this request is acting in.
|
||||
//
|
||||
// It is the only thing a function may ask for that is not a read, so it is
|
||||
// counted on its own. A sandbox holds a machine for as long as its work runs,
|
||||
// and the invoke that asked for it is over in seconds, so the invoke cannot
|
||||
// stand in for it.
|
||||
//
|
||||
// What comes back is a name. The work is not waited for and its result does not
|
||||
// return through here: a function is answering a request, a run is not, and
|
||||
// pretending they are the same call would make every run as short as a request.
|
||||
func (c *invocation) start(arg []byte) ([]byte, error) {
|
||||
if err := checkCollectionRateLimit(c.e, c.collection, "start"); err != nil {
|
||||
return nil, c.refuse(err)
|
||||
}
|
||||
|
||||
sandboxes, _ := c.e.App.Store().Get(StoreKeySandboxes).(Sandboxes)
|
||||
if sandboxes == nil {
|
||||
return nil, errors.New("start: this deployment has nowhere to run work")
|
||||
}
|
||||
|
||||
// What a function hands over is built inside the runtime, so it is not the
|
||||
// invocation body and none of that body's bound reached it. It gets the
|
||||
// same one: an env or a label list larger than a whole request is not a
|
||||
// description of a run, it is the request being made twice.
|
||||
if len(arg) > functionMaxPayload {
|
||||
return nil, errors.New("start: the ask is too large")
|
||||
}
|
||||
|
||||
// The fields of [Sandbox] are the whole vocabulary, so anything else is
|
||||
// refused rather than quietly dropped: a caller that wrote a field is a
|
||||
// caller that expected it to mean something, and whose work a run is is
|
||||
// exactly the meaning that is not on offer.
|
||||
dec := json.NewDecoder(bytes.NewReader(arg))
|
||||
dec.DisallowUnknownFields()
|
||||
|
||||
var s Sandbox
|
||||
if err := dec.Decode(&s); err != nil {
|
||||
return nil, fmt.Errorf("start: %w", err)
|
||||
}
|
||||
if s.Snapshot == "" {
|
||||
return nil, errors.New("start: a run needs a snapshot to run as")
|
||||
}
|
||||
for name := range s.Labels {
|
||||
if strings.HasPrefix(name, reservedLabel) {
|
||||
return nil, fmt.Errorf("start: %q is the deployment's to write", name)
|
||||
}
|
||||
}
|
||||
|
||||
if err := c.count(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The org is read off the request, where the credential resolved it before
|
||||
// any function ran, and passed beside what was asked for rather than folded
|
||||
// into it.
|
||||
//
|
||||
// A deployment that tells orgs apart is one where a run belongs to somebody,
|
||||
// so an unnamed org is refused there. A Base serving one tenant names no org
|
||||
// on any request and never has — there is nobody for a run to be confused
|
||||
// with — so the question is asked of the deployment rather than the request.
|
||||
org, _ := c.e.Get(RequestEventKeyOrg).(string)
|
||||
if org == "" && c.e.Deployment().Store().Get(StoreKeyBases) != nil {
|
||||
return nil, errors.New("start: a run needs an org on a Base that serves many")
|
||||
}
|
||||
|
||||
id, err := sandboxes.Start(c.ctx, org, s)
|
||||
if err != nil {
|
||||
// Where work runs is the deployment's, so what went wrong there is the
|
||||
// deployment's: it is logged whole and the function is told only that
|
||||
// the run did not start. This is the rule functionInvoke already applies
|
||||
// to a failed function, applied one call earlier — a function may catch
|
||||
// what start raises and answer its own caller with it.
|
||||
c.e.App.Logger().Error("base: start failed",
|
||||
"function", c.e.Request.PathValue("name"), "org", org, "error", err)
|
||||
return nil, errors.New("start: the run did not start")
|
||||
}
|
||||
|
||||
return json.Marshal(struct {
|
||||
Id string `json:"id"`
|
||||
}{Id: id})
|
||||
}
|
||||
|
||||
// readInfo is the caller, presented as the GET this read is.
|
||||
//
|
||||
// The identity and the headers carry over unchanged, because they are the
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
package apis_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/base/apis"
|
||||
"github.com/hanzoai/base/core"
|
||||
_ "github.com/hanzoai/base/plugins/gojavm" // the "js" runtime a function runs on
|
||||
"github.com/hanzoai/base/tests"
|
||||
"github.com/hanzoai/base/tools/types"
|
||||
)
|
||||
|
||||
// A function starts work somewhere else, and these are the statements about
|
||||
// what that does and does not give it. The interesting half is the second one:
|
||||
// a sandbox has a network and processes, so the thing worth proving is that
|
||||
// asking for one leaves the function exactly where it was.
|
||||
|
||||
// recorder is a [apis.Sandboxes] that starts nothing and remembers what it was
|
||||
// asked to start. It is the only place the org and the spec are visible at the
|
||||
// same moment, which is what makes it the place to assert that they came from
|
||||
// different sources.
|
||||
type recorder struct {
|
||||
mu sync.Mutex
|
||||
orgs []string
|
||||
specs []apis.Sandbox
|
||||
fails error
|
||||
}
|
||||
|
||||
func (r *recorder) Start(_ context.Context, org string, s apis.Sandbox) (string, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.fails != nil {
|
||||
return "", r.fails
|
||||
}
|
||||
r.orgs = append(r.orgs, org)
|
||||
r.specs = append(r.specs, s)
|
||||
return fmt.Sprintf("sbx%d", len(r.specs)), nil
|
||||
}
|
||||
|
||||
func (r *recorder) started() int {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return len(r.specs)
|
||||
}
|
||||
|
||||
func (r *recorder) at(i int) (string, apis.Sandbox) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return r.orgs[i], r.specs[i]
|
||||
}
|
||||
|
||||
// seedRuns builds a Base whose functions are the given sources and whose work
|
||||
// runs into a recorder.
|
||||
func seedRuns(t testing.TB, sources map[string]string) (*tests.TestApp, *recorder) {
|
||||
t.Helper()
|
||||
|
||||
app, err := tests.NewTestApp()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
functions, err := app.FindCollectionByNameOrId(core.CollectionNameFunctions)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
functions.ViewRule = types.Pointer("")
|
||||
if err := app.Save(functions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for name, src := range sources {
|
||||
r := core.NewRecord(functions)
|
||||
r.Id = name
|
||||
r.Set(core.FieldNameSource, src)
|
||||
if err := app.Save(r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
rec := &recorder{}
|
||||
app.Store().Set(apis.StoreKeySandboxes, apis.Sandboxes(rec))
|
||||
|
||||
return app, rec
|
||||
}
|
||||
|
||||
// invokeIn calls a function on a request that is acting in org.
|
||||
//
|
||||
// Naming the org on the request is what a resolved credential does before any
|
||||
// handler runs, so standing in for it here is standing in for the door and not
|
||||
// for the rule — the rule is what is under test.
|
||||
func invokeIn(t *testing.T, app *tests.TestApp, name, token, body, org string, status int, want ...string) {
|
||||
t.Helper()
|
||||
|
||||
if body == "" {
|
||||
body = "{}"
|
||||
}
|
||||
|
||||
scenario := tests.ApiScenario{
|
||||
Name: name + " in " + org,
|
||||
Method: http.MethodPost,
|
||||
URL: "/v1/functions/" + name,
|
||||
Body: strings.NewReader(body),
|
||||
Headers: map[string]string{"Authorization": token, "Content-Type": "application/json"},
|
||||
BeforeTestFunc: func(t testing.TB, app *tests.TestApp, e *core.ServeEvent) {
|
||||
e.Router.BindFunc(func(re *core.RequestEvent) error {
|
||||
re.Set(apis.RequestEventKeyOrg, org)
|
||||
return re.Next()
|
||||
})
|
||||
},
|
||||
TestAppFactory: func(testing.TB) *tests.TestApp { return app },
|
||||
DisableTestAppCleanup: true,
|
||||
ExpectedStatus: status,
|
||||
ExpectedContent: want,
|
||||
|
||||
// Whatever the host said to the function, it said to the function. What
|
||||
// a run was refused for is between the deployment and its own code, and
|
||||
// none of it is the answer to somebody's HTTP request — so every call
|
||||
// in this file asserts the wording never turns up in a body.
|
||||
NotExpectedContent: []string{
|
||||
"nowhere to run work",
|
||||
"the deployment's to write",
|
||||
"unknown field",
|
||||
"needs a snapshot",
|
||||
"needs an org",
|
||||
"sk-deployment-secret",
|
||||
"10.4.2.9",
|
||||
},
|
||||
}
|
||||
scenario.Test(t)
|
||||
}
|
||||
|
||||
// What a function gets for starting work is a name, and it gets it straight
|
||||
// away. The run itself is minutes long and the request is not, so the two are
|
||||
// not the same call: nothing about the work's result comes back here, and the
|
||||
// name is what the caller looks under once it does.
|
||||
func TestFunctionStartsWorkAndIsAnsweredWithItsName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"research": `function handler(p, base){ return base.start({snapshot:"hanzo/research", env:{TOPIC:p.topic}}) }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
invokeIn(t, app, "research", token, `{"topic":"kelp"}`, "acme", 200, `{"id":"sbx1"}`)
|
||||
|
||||
if rec.started() != 1 {
|
||||
t.Fatalf("expected 1 run, got %d", rec.started())
|
||||
}
|
||||
|
||||
org, spec := rec.at(0)
|
||||
if org != "acme" {
|
||||
t.Fatalf("expected the request's org, got %q", org)
|
||||
}
|
||||
if spec.Snapshot != "hanzo/research" {
|
||||
t.Fatalf("expected the asked-for snapshot, got %q", spec.Snapshot)
|
||||
}
|
||||
if spec.Env["TOPIC"] != "kelp" {
|
||||
t.Fatalf("expected the payload to reach the run, got %q", spec.Env["TOPIC"])
|
||||
}
|
||||
}
|
||||
|
||||
// Starting one changes nothing about what the function itself can touch.
|
||||
//
|
||||
// A sandbox has a network, a filesystem and processes; a function has three
|
||||
// names and no way to reach any of that, before or after asking for one. The
|
||||
// two lists this returns are the whole surface: what is reachable in the
|
||||
// language, and what the host bound.
|
||||
func TestFunctionReachesNoNetworkAndNoProcess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
probe := `function handler(p, base){
|
||||
var reachable = [];
|
||||
["fetch","XMLHttpRequest","WebSocket","process","Buffer","$os","$http","$app","$security"].forEach(function(n){
|
||||
try { if (eval("typeof " + n) !== "undefined") { reachable.push(n) } } catch (e) {}
|
||||
});
|
||||
["http","https","net","dgram","child_process","fs","os"].forEach(function(m){
|
||||
try { require(m); reachable.push("require:" + m) } catch (e) {}
|
||||
});
|
||||
base.start({snapshot:"hanzo/research"});
|
||||
["fetch","process","$os"].forEach(function(n){
|
||||
try { if (eval("typeof " + n) !== "undefined") { reachable.push("after:" + n) } } catch (e) {}
|
||||
});
|
||||
return {reachable: reachable, host: Object.keys(base).sort()};
|
||||
}`
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{"probe": probe})
|
||||
defer app.Cleanup()
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
invokeIn(t, app, "probe", token, "", "acme", 200,
|
||||
`{"reachable":[],"host":["list","one","start"]}`)
|
||||
|
||||
// The run was started, so the emptiness above is the state of a function
|
||||
// that has one rather than of a function that never asked.
|
||||
if rec.started() != 1 {
|
||||
t.Fatalf("expected 1 run, got %d", rec.started())
|
||||
}
|
||||
}
|
||||
|
||||
// Starting is counted on its own, because it is not priced like a read.
|
||||
//
|
||||
// Two reads cost two queries against a file that is already open. Two runs are
|
||||
// two machines, held for as long as the work takes, so the second one here is
|
||||
// refused by the rule that says so — and refused as the limit it is, not as a
|
||||
// deployment that broke.
|
||||
func TestStartingCountsAgainstItsOwnLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"twice": `function handler(p, base){ return {a: base.start({snapshot:"x"}).id, b: base.start({snapshot:"x"}).id} }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
app.Settings().RateLimits.Enabled = true
|
||||
app.Settings().RateLimits.Rules = []core.RateLimitRule{
|
||||
{Label: "_functions:start", MaxRequests: 1, Duration: 10},
|
||||
}
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
invokeIn(t, app, "twice", token, "", "acme", 429, `"status":429`)
|
||||
|
||||
if rec.started() != 1 {
|
||||
t.Fatalf("expected the rule to stop the second run, got %d started", rec.started())
|
||||
}
|
||||
}
|
||||
|
||||
// One call starts the work one request implies, and stops there.
|
||||
//
|
||||
// The rule above is what an operator writes; this is what holds when nobody has
|
||||
// written one, which is every Base out of the box — rate limits ship disabled,
|
||||
// so without this a single invoke starts as many machines as it can name in
|
||||
// five seconds. Measured before the cap existed: five thousand.
|
||||
func TestOneCallStartsABoundedNumberOfRuns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"burst": `function handler(p, base){
|
||||
var n = 0;
|
||||
try { for (var i = 0; i < 5000; i++) { base.start({snapshot:"x"}); n++ } }
|
||||
catch (e) { return {n: n, stopped: true} }
|
||||
return {n: n, stopped: false};
|
||||
}`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Nothing is configured here: these are the settings a Base boots with.
|
||||
invokeIn(t, app, "burst", token, "", "acme", 200, `{"n":8,"stopped":true}`)
|
||||
|
||||
if rec.started() != 8 {
|
||||
t.Fatalf("expected the cap to hold at 8, got %d", rec.started())
|
||||
}
|
||||
}
|
||||
|
||||
// Whose work a run is comes from the request, so there is nothing for a
|
||||
// function to name.
|
||||
//
|
||||
// The org is not a field of what gets asked for — it is passed beside it, off a
|
||||
// credential resolved before any function ran. So a function that writes one is
|
||||
// not overriding anything; it is naming a field that does not exist, and the
|
||||
// same goes for the labels a deployment stamps a run's owner into.
|
||||
func TestFunctionStartsWorkOnlyInItsOwnOrg(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"named": `function handler(p, base){ return base.start({snapshot:"x", org:"victim"}) }`,
|
||||
"labeled": `function handler(p, base){ return base.start({snapshot:"x", labels:{"hanzo.ai/org":"victim"}}) }`,
|
||||
"plain": `function handler(p, base){ return base.start({snapshot:"x", labels:{mine:"yes"}}) }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A function asking for something that is not on offer is the deployment's
|
||||
// own code being wrong, which is what it is answered as.
|
||||
invokeIn(t, app, "named", token, "", "acme", 500, `"status":500`)
|
||||
invokeIn(t, app, "labeled", token, "", "acme", 500, `"status":500`)
|
||||
|
||||
if rec.started() != 0 {
|
||||
t.Fatalf("expected nothing to have been started, got %d", rec.started())
|
||||
}
|
||||
|
||||
// The same function, asking for nothing it may not, runs in the org the
|
||||
// request is acting in — never in the one the source mentioned.
|
||||
invokeIn(t, app, "plain", token, "", "acme", 200, `{"id":"sbx1"}`)
|
||||
|
||||
org, spec := rec.at(0)
|
||||
if org != "acme" {
|
||||
t.Fatalf("expected the request's org, got %q", org)
|
||||
}
|
||||
if spec.Labels["mine"] != "yes" {
|
||||
t.Fatalf("expected the caller's own labels to carry through, got %v", spec.Labels)
|
||||
}
|
||||
}
|
||||
|
||||
// On a Base that tells orgs apart, a run belongs to one of them.
|
||||
//
|
||||
// A function is invokable by whoever the collection's rule says, and a rule of
|
||||
// "" says everybody — so a request can reach here carrying no credential and
|
||||
// therefore no org. On a deployment serving many that is not work belonging to
|
||||
// nobody, it is work whose owner was never established, and it is refused. A
|
||||
// Base serving one tenant names no org on any request and is unaffected: the
|
||||
// question is asked of the deployment, not of the request.
|
||||
func TestARunNeedsAnOrgWhereThereAreOrgs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"work": `function handler(p, base){ return base.start({snapshot:"x"}) }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A Base serving one tenant: nobody to confuse the run with, so it runs.
|
||||
invokeIn(t, app, "work", token, "", "", 200, `{"id":"sbx1"}`)
|
||||
|
||||
if org, _ := rec.at(0); org != "" {
|
||||
t.Fatalf("expected the one tenant's empty org, got %q", org)
|
||||
}
|
||||
|
||||
// The same request on a deployment that serves many is refused, because
|
||||
// there the answer to "whose run is this" was never established.
|
||||
app.Store().Set(apis.StoreKeyBases, apis.Bases(func(string) (core.App, error) {
|
||||
return app, nil
|
||||
}))
|
||||
|
||||
invokeIn(t, app, "work", token, "", "", 500, `"status":500`)
|
||||
|
||||
if rec.started() != 1 {
|
||||
t.Fatalf("expected the unowned run to be refused, got %d started", rec.started())
|
||||
}
|
||||
}
|
||||
|
||||
// What went wrong where the work runs is not the caller's to read.
|
||||
//
|
||||
// A function may catch what start raises, and a function is the tenant's while
|
||||
// its caller may be anybody the rule admits — so the words start uses are the
|
||||
// words that reach an end user. The deployment's own failure is logged whole
|
||||
// and the function is told only that the run did not start, which is the rule
|
||||
// functionInvoke already applies to a function that fails outright.
|
||||
func TestStartTellsTheFunctionNothingAboutTheDeployment(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, rec := seedRuns(t, map[string]string{
|
||||
"leaky": `function handler(p, base){ try { base.start({snapshot:"x"}) } catch (e) { return {said: e.message} } return {said:"no error"} }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
rec.fails = errors.New(`dial tcp 10.4.2.9:8443: bad bearer sk-deployment-secret`)
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
invokeIn(t, app, "leaky", token, "", "acme", 200, `{"said":"start: the run did not start"}`)
|
||||
}
|
||||
|
||||
// A deployment with nowhere to run work still answers the name.
|
||||
//
|
||||
// The alternative is binding the name only where a runtime is configured, which
|
||||
// makes the same source read as a missing function on one deployment and a
|
||||
// refusal on another. One vocabulary everywhere; the answer is what differs.
|
||||
func TestStartRefusesWhenTheDeploymentRunsNoWork(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
app, _ := seedRuns(t, map[string]string{
|
||||
"work": `function handler(p, base){ return {names: Object.keys(base).sort(), started: base.start({snapshot:"x"})} }`,
|
||||
})
|
||||
defer app.Cleanup()
|
||||
|
||||
app.Store().Remove(apis.StoreKeySandboxes)
|
||||
|
||||
token, err := tests.GetUserAuthToken(app, "users", tests.TestUserID1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
invokeIn(t, app, "work", token, "", "acme", 500, `"status":500`)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package apis
|
||||
|
||||
import "context"
|
||||
|
||||
// Where a function's work runs.
|
||||
//
|
||||
// A function starts a sandbox; it never becomes one. Starting hands back the
|
||||
// name the work is known by, and that is the whole of what crosses back: the
|
||||
// network, the filesystem and the processes a sandbox has belong to the code
|
||||
// running there, and a function that started one is still a function with two
|
||||
// reads and no wire. So this is bound because work costs machines, not because
|
||||
// it widens what a function can see — those are different questions and only
|
||||
// the first one is answered here.
|
||||
//
|
||||
// The result arrives the way any other writer's does. The work authenticates to
|
||||
// Base and writes a record; nobody holds a request open for the minutes it
|
||||
// takes, and the caller finds what it wrote under the name it was given.
|
||||
|
||||
// Sandbox is what to run — an image, and the environment it runs under.
|
||||
//
|
||||
// It says nothing about WHOSE work it is. That is [Sandboxes.Start]'s org, taken
|
||||
// from the credential the request already resolved, which is what makes it the
|
||||
// one thing about a run that the asking cannot state.
|
||||
type Sandbox struct {
|
||||
// Snapshot names the image to run. The work is that image's own entry
|
||||
// point, so a run is described by what it starts as and what it starts
|
||||
// with, and there is no command here for a caller to compose.
|
||||
Snapshot string `json:"snapshot"`
|
||||
|
||||
// Env is what the image reads its instructions from.
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
|
||||
// Labels is the caller's own bookkeeping, carried through untouched.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
// Sandboxes is somewhere isolated to run work, as a Base reaches it.
|
||||
//
|
||||
// A deployment installs one under [StoreKeySandboxes]. A deployment that
|
||||
// installs none still answers the name — it refuses in as many words rather
|
||||
// than leaving it undefined, so what a function may ask for reads the same
|
||||
// everywhere and a deployment without a runtime is a clear answer instead of a
|
||||
// different language.
|
||||
type Sandboxes interface {
|
||||
// Start begins one sandbox for org and answers with the name it is known
|
||||
// by. It returns once the work has begun and never waits for it to finish.
|
||||
//
|
||||
// It must return when ctx is done. The call is made from the function's own
|
||||
// slot on a pool the whole process shares, so a Start that outlives its
|
||||
// context holds that slot past the deadline the function was given, and the
|
||||
// next function waits behind it.
|
||||
//
|
||||
// org is whose work this is. An implementation carries it to the runtime
|
||||
// itself and never reads it back out of s, because s is what a function
|
||||
// asked for and org is what the request already proved.
|
||||
//
|
||||
// It is empty on a Base that serves one tenant, which is every deployment
|
||||
// that registers no [Bases] — there is no org because there is nobody to
|
||||
// tell apart. An implementation serving many decides what an empty one
|
||||
// means to it, and refusing is a reasonable thing to decide.
|
||||
Start(ctx context.Context, org string, s Sandbox) (string, error)
|
||||
}
|
||||
Reference in New Issue
Block a user