Stage a plugin binary under ForkLock, so starts at once all exec

A child forked by one start holds a copy of every descriptor the host has
open until it execs, including the write handle another start is copying
its binary through. Linux refuses to exec a file anyone holds open for
writing, so plugins started together failed each other with "text file
busy": 2 of 16 in TestStart_TogetherAllRun without this, none with it.

Holding the read side of syscall.ForkLock while the copy is written keeps
every fork out of that window.

Co-authored-by: Hanzo Dev <[email protected]>

load_test.go

@@ -10,7 +10,9 @@ import (
 	"path/filepath"
 	"runtime"
 	"slices"
+	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
@@ -687,3 +689,40 @@ func TestUnload_LazyStaysDown(t *testing.T) {
 		t.Fatal("still marked disabled after a successful Reload")
 	}
 }
+
+// TestStart_TogetherAllRun starts many plugins at once, as a host does when it
+// brings its pinned set up together. Each start copies its binary before it
+// execs it, and a child forked meanwhile by another start inherits that write
+// handle until it execs in turn; an exec of a file someone holds open for writing
+// fails with "text file busy". Every start must succeed.
+func TestStart_TogetherAllRun(t *testing.T) {
+	bin := buildPlugin(t, "v1")
+	const n = 16
+	app := zip.New(zip.Config{AppName: "host", DisableStartupMessage: true})
+	defer func() { _ = app.Shutdown() }()
+	dir := sockDir(t)
+	for i := range n {
+		name := "p" + strconv.Itoa(i)
+		app.Use(must(zip.Load(zip.Plugin{Name: name, Bin: bin, Dir: dir, Lazy: true}, "/v1/"+name)))
+	}
+	errs := make([]error, n)
+	var wg sync.WaitGroup
+	for i := range n {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, errs[i] = app.Start("p" + strconv.Itoa(i))
+		}()
+	}
+	wg.Wait()
+	for i, err := range errs {
+		if err != nil {
+			t.Errorf("p%d did not start: %v", i, err)
+		}
+	}
+	for _, s := range app.Plugins() {
+		if !s.Running {
+			t.Errorf("%s is not running: %s", s.Name, s.Error)
+		}
+	}
+}

verify.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"syscall"
 )
 
 // A plugin host is the one place a service becomes an arbitrary-code-execution
@@ -94,12 +95,21 @@ func openVerified(src, priv, name, want string) (*binary, error) {
 }
 
 // stage copies src to dst, which lives in a directory only the host can write.
+//
+// The write handle is open under syscall.ForkLock. A child forked by another
+// goroutine holds a copy of every descriptor this process has open until it
+// execs, O_CLOEXEC or not, and Linux refuses to exec a file anyone holds open for
+// writing. So two plugins starting at once could each fail the other's exec with
+// "text file busy". Holding the read side of ForkLock keeps any fork out while
+// the copy is written; the exec that follows runs with no writer left.
 func stage(src, dst, name string) error {
 	in, err := os.Open(src)
 	if err != nil {
 		return err
 	}
 	defer in.Close()
+	syscall.ForkLock.RLock()
+	defer syscall.ForkLock.RUnlock()
 	out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o700)
 	if err != nil {
 		return fmt.Errorf("plugin %s: stage binary: %w", name, err)