visor/runsc/cmd/sentry/sentrycmd/boot.go

// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package sentrycmd holds the subcommands run by the sentry process.
package sentrycmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strconv"
	"strings"
	"time"

	"github.com/google/subcommands"
	"github.com/moby/sys/capability"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"golang.org/x/sys/unix"

	"gvisor.dev/gvisor/pkg/coretag"
	"gvisor.dev/gvisor/pkg/cpuid"
	"gvisor.dev/gvisor/pkg/fd"
	"gvisor.dev/gvisor/pkg/hostarch"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/metric"
	"gvisor.dev/gvisor/pkg/prometheus"
	"gvisor.dev/gvisor/pkg/rdma"
	"gvisor.dev/gvisor/pkg/ring0"
	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf"
	"gvisor.dev/gvisor/pkg/sentry/hostmm"
	"gvisor.dev/gvisor/pkg/sentry/platform"
	"gvisor.dev/gvisor/pkg/sentry/syscalls/linux"
	"gvisor.dev/gvisor/pkg/tcpip/nftables"
	"gvisor.dev/gvisor/runsc/boot"
	"gvisor.dev/gvisor/runsc/cmd/sandboxsetup"
	"gvisor.dev/gvisor/runsc/cmd/util"
	"gvisor.dev/gvisor/runsc/config"
	"gvisor.dev/gvisor/runsc/flag"
	"gvisor.dev/gvisor/runsc/profile"
	"gvisor.dev/gvisor/runsc/specutils"
	"gvisor.dev/gvisor/runsc/starttime"
)

// Note that directfsSandboxCaps is the same as caps defined in gofer.go
// except CAP_SYS_CHROOT because we don't need to chroot in directfs mode.
var (
	directfsSandboxCaps = []string{
		"CAP_CHOWN",
		"CAP_DAC_OVERRIDE",
		"CAP_DAC_READ_SEARCH",
		"CAP_FOWNER",
		"CAP_FSETID",
	}

	// DirectfsSandboxLinuxCaps is the minimal set of capabilities needed by the
	// sandbox to operate on files in directfs mode.
	DirectfsSandboxLinuxCaps = &specs.LinuxCapabilities{
		Bounding:  directfsSandboxCaps,
		Effective: directfsSandboxCaps,
		Permitted: directfsSandboxCaps,
	}

	hostnetSandboxLinuxCaps = map[capability.Cap]string{
		capability.CAP_NET_ADMIN:        "CAP_NET_ADMIN",
		capability.CAP_NET_BIND_SERVICE: "CAP_NET_BIND_SERVICE",
		capability.CAP_NET_RAW:          "CAP_NET_RAW",
	}
)

// Boot implements subcommands.Command for the "boot" command which starts a
// new sandbox. It should not be called directly.
type Boot struct {
	util.InternalSubCommand

	// bundleDir is the directory containing the OCI spec.
	bundleDir string

	// specFD is the file descriptor that the spec will be read from.
	specFD int

	// controllerFD is the file descriptor of a stream socket for the
	// control server that is donated to this process.
	controllerFD int

	// deviceFD is the file descriptor for the platform device file.
	deviceFD int

	// pinRingFD is the file descriptor of the donated pin ring.
	// See `//pkg/pinring`.
	pinRingFD int

	// ioFDs is the list of FDs used to connect to FS gofers.
	ioFDs sandboxsetup.IntFlags

	// devIoFD is the FD to connect to dev gofer.
	devIoFD int

	// goferFilestoreFDs are FDs to the regular files that will back the tmpfs or
	// overlayfs mount for certain gofer mounts.
	goferFilestoreFDs sandboxsetup.IntFlags

	// goferMountConfs contains information about how the gofer mounts have been
	// configured. The first entry is for rootfs and the following entries are
	// for bind mounts in Spec.Mounts (in the same order).
	goferMountConfs specutils.GoferMountConfFlags

	// stdioFDs are the fds for stdin, stdout, and stderr. They must be
	// provided in that order.
	stdioFDs sandboxsetup.IntFlags

	// passFDs are mappings of user-supplied host to guest file descriptors.
	passFDs sandboxsetup.FDMappings

	// execFD is the host file descriptor used for program execution.
	execFD int

	// applyCaps determines if capabilities defined in the spec should be applied
	// to the process.
	applyCaps bool

	// setUpChroot is set to true if the sandbox is started in an empty root.
	setUpRoot bool

	// cpuNum number of CPUs to create inside the sandbox.
	cpuNum int

	// cpuQuota and cpuPeriod carry the raw host CFS CPU settings that should be
	// exposed through sandbox cgroupfs.
	cpuQuota  int64
	cpuPeriod int64

	// totalMem sets the initial amount of total memory to report back to the
	// container.
	totalMem uint64

	// totalHostMem is the total memory reported by host /proc/meminfo.
	totalHostMem uint64

	// userLogFD is the file descriptor to write user logs to.
	userLogFD int

	// startSyncFD is the file descriptor to synchronize runsc and sandbox.
	startSyncFD int

	// mountsFD is the file descriptor to read list of mounts after they have
	// been resolved (direct paths, no symlinks). They are resolved outside the
	// sandbox (e.g. gofer) and sent through this FD. When mountsFD is not
	// provided, there is no cleaning required for mounts and the mounts in
	// the spec can be used as is.
	mountsFD int

	podInitConfigFD int

	sinkFDs sandboxsetup.IntFlags

	saveFDs             sandboxsetup.IntFlags
	saveCheckpointGofer bool

	fsSaveFDs             sandboxsetup.IntFlags
	fsSaveCheckpointGofer bool

	fsRestoreFDs             sandboxsetup.IntFlags
	fsRestoreCheckpointGofer bool

	// attached is set to true to kill the sandbox process when the parent process
	// terminates. This flag is set when the command execve's itself because
	// parent death signal doesn't propagate through execve when uid/gid changes.
	attached bool

	// productName is the value to show in
	// /sys/devices/virtual/dmi/id/product_name.
	productName string

	hostTHP boot.HostTHP

	// FDs for profile data.
	profileFDs profile.FDArgs

	// finalMetricsFD is a file descriptor to write metric data to upon sandbox
	// termination.
	finalMetricsFD int

	// profilingMetricsFD is a file descriptor to write Sentry metrics data to.
	profilingMetricsFD int

	// profilingMetricsLossy sets whether profilingMetricsFD is a lossy channel.
	// If so, the format used to write to it will contain a checksum.
	profilingMetricsLossy bool

	// procMountSyncFD is a file descriptor that has to be closed when the
	// procfs mount isn't needed anymore.
	procMountSyncFD int

	// procMountSyncFile holds the *os.File for procMountSyncFD when this
	// process continues without re-exec'ing itself (see
	// `applyCapsAllThreads`). It keeps the FD alive.
	procMountSyncFile *os.File

	// syncUsernsFD is the file descriptor that has to be closed when the
	// boot process should invoke setuid/setgid for root user. This is mainly
	// used to synchronize rootless user namespace initialization.
	syncUsernsFD int

	// nvidiaDriverVersion is the Nvidia driver version on the host.
	nvidiaDriverVersion string

	// These correspond to fields in nvconf.HostSettings, which is not used
	// here because nvconf.HostSettings.FabricIMEXManagementDevMinor in
	// particular is uint32 and there is no flag.Uint32Var().
	procDriverNvidiaParams             string
	nvidiaFabricIMEXManagementDevMinor int64

	// uid and gid are the user and group IDs to switch to after setting up the
	// user namespace.
	uid int
	gid int

	// rootfsUpperTarFD is the file descriptor to a tar file that has rootfs change at startup.
	rootfsUpperTarFD int
}

// Name implements subcommands.Command.Name.
func (*Boot) Name() string {
	return "boot"
}

// Synopsis implements subcommands.Command.Synopsis.
func (*Boot) Synopsis() string {
	return "launch a sandbox process"
}

// Usage implements subcommands.Command.Usage.
func (*Boot) Usage() string {
	return "boot [flags] <container id>\n"
}

// SetFlags implements subcommands.Command.SetFlags.
func (b *Boot) SetFlags(f *flag.FlagSet) {
	f.StringVar(&b.bundleDir, "bundle", "", "required path to the root of the bundle directory")
	f.BoolVar(&b.applyCaps, "apply-caps", false, "if true, apply capabilities defined in the spec to the process")
	f.BoolVar(&b.setUpRoot, "setup-root", false, "if true, set up an empty root for the process")
	f.IntVar(&b.cpuNum, "cpu-num", 0, "number of CPUs to create inside the sandbox")
	// cpu-quota defaults to -1 (unlimited), matching the Linux CFS default
	// for cpu.cfs_quota_us. A positive value is passed through as-is to sandbox
	// cgroupfs.
	f.Int64Var(&b.cpuQuota, "cpu-quota", -1, "raw CFS cpu quota in usecs to expose via sandbox cgroupfs (-1 means unlimited)")
	// cpu-period defaults to 0 (unset). When unset, mountCgroupMounts
	// uses the Linux default of 100000us (100ms). A positive value is
	// passed through as-is.
	f.Int64Var(&b.cpuPeriod, "cpu-period", 0, "raw CFS cpu period in usecs to expose via sandbox cgroupfs (0 means use default 100ms)")
	f.IntVar(&b.procMountSyncFD, "proc-mount-sync-fd", -1, "file descriptor that has to be written to when /proc isn't needed anymore and can be unmounted")
	f.IntVar(&b.syncUsernsFD, "sync-userns-fd", -1, "file descriptor used to synchronize rootless user namespace initialization.")
	f.Uint64Var(&b.totalMem, "total-memory", 0, "sets the initial amount of total memory to report back to the container")
	f.Uint64Var(&b.totalHostMem, "total-host-memory", 0, "total memory reported by host /proc/meminfo")
	f.BoolVar(&b.attached, "attached", false, "if attached is true, kills the sandbox process when the parent process terminates")
	f.StringVar(&b.productName, "product-name", "", "value to show in /sys/devices/virtual/dmi/id/product_name")
	f.StringVar(&b.hostTHP.ShmemEnabled, "host-thp-shmem-enabled", "", "value of /sys/kernel/mm/transparent_hugepage/shmem_enabled on the host")
	f.StringVar(&b.hostTHP.Defrag, "host-thp-defrag", "", "value of /sys/kernel/mm/transparent_hugepage/defrag on the host")
	f.IntVar(&b.uid, "uid", 0, "user ID")
	f.IntVar(&b.gid, "gid", 0, "user ID")

	// Open FDs that are donated to the sandbox.
	f.IntVar(&b.specFD, "spec-fd", -1, "required fd with the container spec")
	f.IntVar(&b.controllerFD, "controller-fd", -1, "required FD of a stream socket for the control server that must be donated to this process")
	f.IntVar(&b.deviceFD, "device-fd", -1, "FD for the platform device file")
	f.IntVar(&b.pinRingFD, "pin-ring-fd", -1, "FD of a disabled io_uring in which the sentry pins its expensive-to-release files")
	f.Var(&b.ioFDs, "io-fds", "list of image FDs and/or socket FDs to connect gofer clients. They must follow this order: root first, then mounts as defined in the spec")
	f.IntVar(&b.devIoFD, "dev-io-fd", -1, "FD to connect dev gofer client")
	f.Var(&b.stdioFDs, "stdio-fds", "list of FDs containing sandbox stdin, stdout, and stderr in that order")
	f.Var(&b.passFDs, "pass-fd", "mapping of host to guest FDs. They must be in M:N format. M is the host and N the guest descriptor.")
	f.IntVar(&b.execFD, "exec-fd", -1, "host file descriptor used for program execution.")
	f.Var(&b.goferFilestoreFDs, "gofer-filestore-fds", "FDs to the regular files that will back the overlayfs or tmpfs mount if a gofer mount is to be overlaid.")
	f.Var(&b.goferMountConfs, "gofer-mount-confs", "information about how the gofer mounts have been configured.")
	f.IntVar(&b.userLogFD, "user-log-fd", 0, "file descriptor to write user logs to. 0 means no logging.")
	f.IntVar(&b.startSyncFD, "start-sync-fd", -1, "required FD to used to synchronize sandbox startup")
	f.IntVar(&b.mountsFD, "mounts-fd", -1, "mountsFD is an optional file descriptor to read list of mounts after they have been resolved (direct paths, no symlinks).")
	f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.")
	f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.")
	f.Var(&b.saveFDs, "save-fds", "ordered list of file descriptors to be used save checkpoints. Order: kernel state, page metadata, page file")
	f.BoolVar(&b.saveCheckpointGofer, "save-checkpoint-gofer", false, "if true, -save-fds is a socket connected to checkpoint gofer")
	f.Var(&b.fsSaveFDs, "fs-save-fds", "ordered list of file descriptors for filesystem checkpoint save")
	f.BoolVar(&b.fsSaveCheckpointGofer, "fs-save-checkpoint-gofer", false, "if true, -fs-save-fds is a socket connected to checkpoint gofer")
	f.Var(&b.fsRestoreFDs, "fs-restore-fds", "ordered list of file descriptors for filesystem checkpoint restore")
	f.BoolVar(&b.fsRestoreCheckpointGofer, "fs-restore-checkpoint-gofer", false, "if true, -fs-restore-fds is a socket connected to checkpoint gofer")
	f.IntVar(&b.rootfsUpperTarFD, "rootfs-upper-tar-fd", -1, "file descriptor to the tar file containing the rootfs upper layer changes.")

	// Profiling flags.
	b.profileFDs.SetFromFlags(f)
	f.IntVar(&b.finalMetricsFD, "final-metrics-log-fd", -1, "file descriptor to write metrics to upon sandbox termination.")
	f.IntVar(&b.profilingMetricsFD, "profiling-metrics-fd", -1, "file descriptor to write sentry profiling metrics.")
	f.BoolVar(&b.profilingMetricsLossy, "profiling-metrics-fd-lossy", false, "if true, treat the sentry profiling metrics FD as lossy and write a checksum to it.")

	// Nvidia driver properties.
	f.StringVar(&b.nvidiaDriverVersion, "nvidia-driver-version", "", "Nvidia driver version on the host")
	f.StringVar(&b.procDriverNvidiaParams, "nvidia-host-params", "", "value of /proc/driver/nvidia/params on the host")
	f.Int64Var(&b.nvidiaFabricIMEXManagementDevMinor, "nvidia-fabric-imex-mgmt-minor", -1, "DeviceFileMinor in /proc/driver/nvidia/capabilities/fabric-imex-mgmt on the host")
}

// willReexec returns true if this boot process will re-execute itself to
// apply credential/capability changes to all of its threads, rather than
// applying capabilities to all threads in-process.
func (b *Boot) willReexec() bool {
	if config.CgoEnabled {
		// When cgo is on, we can't control non-Go threads, so need reexec.
		return true
	}
	if !b.applyCaps {
		return true // No caps to change but will change UID/GID to `nobody`.
	}
	if b.syncUsernsFD >= 0 {
		return true // Rootless mode.
	}
	return false
}

// Execute implements subcommands.Command.Execute.  It starts a sandbox in a
// waiting state.
func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus {
	if b.specFD == -1 || b.controllerFD == -1 || b.startSyncFD == -1 || f.NArg() != 1 {
		f.Usage()
		return subcommands.ExitUsageError
	}

	conf := args[0].(*config.Config)

	timer := starttime.Timer("runsc boot")
	timer.ReachedAt("Go code running", starttime.GoStartTime())
	timer.Reached("boot subcommand dispatched")

	if hostPageSize := unix.Getpagesize(); hostPageSize != hostarch.PageSize {
		util.Fatalf("host page size (%d) does not match compiled page size (%d)", hostPageSize, hostarch.PageSize)
	}

	// Set traceback level
	debug.SetTraceback(conf.Traceback)

	// Initialize CPUID information.
	cpuid.Initialize()
	timer.Reached("CPUID initialized")

	// Initialize ring0 library.
	ring0.InitDefault()
	timer.Reached("ring0 initialized")

	argOverride := make(map[string]string)

	// Do these before chroot takes effect, otherwise we can't read /proc and /sys.
	if len(b.productName) == 0 {
		if product, err := os.ReadFile("/sys/devices/virtual/dmi/id/product_name"); err != nil {
			log.Warningf("Not setting product_name: %v", err)
		} else {
			b.productName = strings.TrimSpace(string(product))
			log.Infof("Setting product_name: %q", b.productName)
			argOverride["product-name"] = b.productName
		}
	}
	if conf.AppHugePages {
		if len(b.hostTHP.ShmemEnabled) == 0 {
			val, err := hostmm.ReadTransparentHugepageEnum("shmem_enabled")
			if err != nil {
				log.Warningf("Failed to infer --host-thp-shmem-enabled: %v", err)
			} else {
				b.hostTHP.ShmemEnabled = val
				log.Infof("Setting host-thp-shmem-enabled: %q", b.hostTHP.ShmemEnabled)
				argOverride["host-thp-shmem-enabled"] = b.hostTHP.ShmemEnabled
			}
		}
		if len(b.hostTHP.Defrag) == 0 {
			val, err := hostmm.ReadTransparentHugepageEnum("defrag")
			if err != nil {
				log.Warningf("Failed to infer --host-thp-defrag: %v", err)
			} else {
				b.hostTHP.Defrag = val
				log.Infof("Setting host-thp-defrag: %q", b.hostTHP.Defrag)
				argOverride["host-thp-defrag"] = b.hostTHP.Defrag
			}
		}
	}
	timer.Reached("host info read")

	if b.attached {
		// Ensure this process is killed after parent process terminates when
		// attached mode is enabled. In the unfortunate event that the parent
		// terminates before this point, this process leaks.
		if err := unix.Prctl(unix.PR_SET_PDEATHSIG, uintptr(unix.SIGKILL), 0, 0, 0); err != nil {
			util.Fatalf("error setting parent death signal: %v", err)
		}
	}

	if b.syncUsernsFD >= 0 {
		sandboxsetup.SyncUsernsForRootless(b.syncUsernsFD, uint32(b.uid), uint32(b.gid))
		argOverride["sync-userns-fd"] = "-1"
	}

	// Get the spec from the specFD. We *must* keep this os.File alive past
	// the call setCapsAndCallSelf, otherwise the FD will be closed and the
	// child process cannot read it
	specFile := os.NewFile(uintptr(b.specFD), "spec file")
	spec, err := specutils.ReadSpecFromFile(b.bundleDir, specFile, conf)
	if err != nil {
		util.Fatalf("reading spec: %v", err)
	}
	timer.Reached("spec read")

	if specutils.NVProxyEnabled(spec, conf) && b.procDriverNvidiaParams == "" {
		driverCaps, driverCapsErr := specutils.NVProxyDriverCapsAllowed(conf)
		nvhs, err := nvconf.GetHostSettings(nvconf.HostSettingsOptions{
			WantFabricIMEXManagement: driverCapsErr == nil && driverCaps&nvconf.CapFabricIMEXManagement != 0,
		})
		if err != nil {
			log.Warningf("Failed to get nvconf.HostSettings: %v", err)
		} else {
			b.procDriverNvidiaParams = nvhs.ProcDriverNvidiaParams
			argOverride["nvidia-host-params"] = nvhs.ProcDriverNvidiaParams
			if nvhs.HaveFabricIMEXManagement {
				b.nvidiaFabricIMEXManagementDevMinor = int64(nvhs.FabricIMEXManagementDevMinor)
				argOverride["nvidia-fabric-imex-mgmt-minor"] = strconv.FormatUint(uint64(nvhs.FabricIMEXManagementDevMinor), 10)
			}
		}
	}

	if b.setUpRoot {
		if err := setUpChroot(spec, conf, timer); err != nil {
			util.Fatalf("error setting up chroot: %v", err)
		}
		timer.Reached("chroot set up")
		argOverride["setup-root"] = "false"

		if !conf.Rootless {
			// /proc is umounted from a forked process, because the
			// current one is going to drop capabilities and won't be
			// able to umount it.
			cmd, w := sandboxsetup.ExecProcUmounter()
			if b.willReexec() {
				defer cmd.Wait()
				defer w.Close()
			} else {
				// Keep `w` alive until UmountProcFile uses it.
				b.procMountSyncFile = w
			}
			if b.procMountSyncFD != -1 {
				panic("procMountSyncFD is set")
			}
			b.procMountSyncFD = int(w.Fd())
			argOverride["proc-mount-sync-fd"] = strconv.Itoa(b.procMountSyncFD)

			// Clear FD_CLOEXEC in case this process re-executes itself.
			// procMountSyncFD should remain open.
			if _, _, errno := unix.RawSyscall(unix.SYS_FCNTL, w.Fd(), unix.F_SETFD, 0); errno != 0 {
				util.Fatalf("error clearing CLOEXEC: %v", errno)
			}

			if !b.applyCaps {
				// Remove the args that have already been done before calling self.
				args := sandboxsetup.PrepareArgs(b.Name(), f, argOverride)

				// Note that we've already read the spec from the spec FD, and
				// we will read it again after the exec call. This works
				// because the ReadSpecFromFile function seeks to the beginning
				// of the file before reading.
				util.Fatalf("callSelfAsNobody(%v): %v", args, sandboxsetup.CallSelfAsNobody(args))

				// This prevents the specFile finalizer from running and closed
				// the specFD, which we have passed to ourselves when
				// re-execing.
				runtime.KeepAlive(specFile)
				panic("unreachable")
			}
		}
	}

	specutils.LogSpecDebug(spec, conf.OCISeccomp)

	if b.applyCaps {
		caps := &specs.LinuxCapabilities{}

		gPlatform, err := platform.Lookup(conf.Platform)
		if err != nil {
			util.Fatalf("loading platform: %v", err)
		}
		if gPlatform.Requirements().RequiresCapSysPtrace {
			// Ptrace platform requires extra capabilities.
			const c = "CAP_SYS_PTRACE"
			caps.Bounding = append(caps.Bounding, c)
			caps.Effective = append(caps.Effective, c)
			caps.Permitted = append(caps.Permitted, c)
		}

		if conf.DirectFS {
			caps = specutils.MergeCapabilities(caps, DirectfsSandboxLinuxCaps)
		}
		if conf.Network == config.NetworkHost {
			curCaps, err := capability.NewPid2(0)
			if err != nil {
				util.Fatalf("capability.NewPid2(0) failed: %v", err)
			}
			if err := curCaps.Load(); err != nil {
				util.Fatalf("unable to load capabilities: %v", err)
			}
			addCaps := []string{}
			for c, strCap := range hostnetSandboxLinuxCaps {
				if c == capability.CAP_NET_RAW && !conf.EnableRaw {
					continue
				}
				if curCaps.Get(capability.PERMITTED, c) {
					addCaps = append(addCaps, strCap)
				}
			}
			if len(addCaps) != 0 {
				caps = specutils.MergeCapabilities(caps, &specs.LinuxCapabilities{
					Bounding:  addCaps,
					Effective: addCaps,
					Permitted: addCaps,
				})
			}
		}
		if b.willReexec() {
			if config.CgoEnabled {
				log.Warningf("Need to re-exec in order to drop capabilities, due to cgo build. This slows down gVisor startup. Use a pure-Go gVisor build for faster startup.")
			} else {
				log.Infof("Re-execing. FYI, this step isn't necessary when running with root. gVisor will start up faster (and is just as secure) when started as root, as it has to do fewer hoops to get to its as-sandboxed-as-possible state.")
			}
			argOverride["apply-caps"] = "false"

			// Remove the args that have already been done before calling self.
			args := sandboxsetup.PrepareArgs(b.Name(), f, argOverride)

			// Note that we've already read the spec from the spec FD, and
			// we will read it again after the exec call. This works
			// because the ReadSpecFromFile function seeks to the beginning
			// of the file before reading.
			util.Fatalf("setCapsAndCallSelf(%v, %v): %v", args, caps, sandboxsetup.SetCapsAndCallSelf(args, caps))

			// This prevents the specFile finalizer from running and closed
			// the specFD, which we have passed to ourselves when
			// re-execing.
			runtime.KeepAlive(specFile)
			panic("unreachable")
		}

		timer.Reached("sandbox capabilities computed")
		if err := sandboxsetup.ApplyCapsAllThreads(caps, timer); err != nil {
			util.Fatalf("applying caps to all threads: %v", err)
		}
		timer.Reached("capabilities applied")
		if b.attached {
			// Re-arm the parent death signal in case the credential change
			// cleared it. (A pure capability drop should not clear it, but
			// re-arming is free and preserves the re-exec path's behavior,
			// which re-arms after the exec.)
			if err := unix.Prctl(unix.PR_SET_PDEATHSIG, uintptr(unix.SIGKILL), 0, 0, 0); err != nil {
				util.Fatalf("error setting parent death signal: %v", err)
			}
		}
	}

	if b.syncUsernsFD >= 0 {
		// syncUsernsFD is set, but runsc hasn't been re-executed with a new UID and GID.
		// We expect that setCapsAndCallSelf has to be called in this case.
		panic("unreachable")
	}

	// Close specFile to avoid exposing it to the sandbox.
	if err := specFile.Close(); err != nil {
		util.Fatalf("closing specFile: %v", err)
	}

	// At this point we won't re-execute, so it's safe to limit via rlimits. Any
	// limit >= 0 works. If the limit is lower than the current number of open
	// files, then Setrlimit will succeed, and the next open will fail.
	if conf.FDLimit > -1 {
		rlimit := unix.Rlimit{
			Cur: uint64(conf.FDLimit),
			Max: uint64(conf.FDLimit),
		}
		switch err := unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit); err {
		case nil:
		case unix.EPERM:
			log.Warningf("FD limit %d is higher than the current hard limit or system-wide maximum", conf.FDLimit)
		default:
			util.Fatalf("Failed to set RLIMIT_NOFILE: %v", err)
		}
		timer.Reached("FD limit applied")
	}

	// When mountsFD is not provided, there is no cleaning required.
	if b.mountsFD >= 0 {
		// Read resolved mount list and replace the original one from the spec.
		mountsFile := os.NewFile(uintptr(b.mountsFD), "mounts file")
		cleanMounts, err := specutils.ReadMounts(mountsFile)
		if err != nil {
			mountsFile.Close()
			util.Fatalf("Error reading mounts file: %v", err)
		}
		mountsFile.Close()
		spec.Mounts = cleanMounts
		timer.Reached("resolved mounts read")
	}

	if conf.DirectFS {
		// sandbox should run with a umask of 0, because we want to preserve file
		// modes exactly as sent by the sentry, which would have already applied
		// the application umask.
		unix.Umask(0)
	}

	if conf.EnableCoreTags {
		if err := coretag.Enable(); err != nil {
			util.Fatalf("Failed to core tag sentry: %v", err)
		}

		// Verify that all sentry threads are properly core tagged, and log
		// current core tag.
		coreTags, err := coretag.GetAllCoreTags(0)
		if err != nil {
			util.Fatalf("Failed read current core tags: %v", err)
		}
		if len(coreTags) != 1 {
			util.Fatalf("Not all child threads were core tagged the same. Tags=%v", coreTags)
		}
		log.Infof("Core tag enabled (core tag=%d)", coreTags[0])
		timer.Reached("core tags enabled")
	}

	// NFtables is only supported for netstack.
	if (conf.Network == config.NetworkNone || conf.Network == config.NetworkSandbox) && conf.Nftables {
		nftables.EnableNFTables()
	}

	var nvidiaDriverVersion nvconf.DriverVersion
	if b.nvidiaDriverVersion != "" {
		nvidiaDriverVersion, err = nvconf.DriverVersionFrom(b.nvidiaDriverVersion)
		if err != nil {
			util.Fatalf("Failed to parse nvidia driver version: %v", err)
		}
	}

	linux.SetAFSSyscallPanic(conf.TestOnlyAFSSyscallPanic)

	var rdmaSnap *rdma.Snapshot
	if specutils.RDMAEnabled(spec, conf) {
		// Load the RDMA sysfs snapshot serialized by the chroot stage.
		var err error
		if rdmaSnap, err = rdma.Load(rdma.Path); err != nil {
			util.Fatalf("loading RDMA sysfs snapshot: %v", err)
		}
		timer.Reached("RDMA snapshot loaded")
	}

	// Create the loader.
	bootArgs := boot.Args{
		ID:                  f.Arg(0),
		Spec:                spec,
		Conf:                conf,
		ControllerFD:        b.controllerFD,
		Device:              fd.New(b.deviceFD),
		PinRingFD:           b.pinRingFD,
		GoferFDs:            b.ioFDs.GetArray(),
		DevGoferFD:          b.devIoFD,
		StdioFDs:            b.stdioFDs.GetArray(),
		PassFDs:             b.passFDs.GetArray(),
		ExecFD:              b.execFD,
		GoferFilestoreFDs:   b.goferFilestoreFDs.GetArray(),
		GoferMountConfs:     b.goferMountConfs.GetArray(),
		NumCPU:              b.cpuNum,
		CPUQuota:            b.cpuQuota,
		CPUPeriod:           b.cpuPeriod,
		TotalMem:            b.totalMem,
		TotalHostMem:        b.totalHostMem,
		UserLogFD:           b.userLogFD,
		RDMASysfs:           rdmaSnap,
		ProductName:         b.productName,
		PodInitConfigFD:     b.podInitConfigFD,
		SinkFDs:             b.sinkFDs.GetArray(),
		ProfileOpts:         profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval),
		NvidiaDriverVersion: nvidiaDriverVersion,
		NvidiaHostSettings: &nvconf.HostSettings{
			ProcDriverNvidiaParams:       b.procDriverNvidiaParams,
			HaveFabricIMEXManagement:     b.nvidiaFabricIMEXManagementDevMinor >= 0,
			FabricIMEXManagementDevMinor: uint32(b.nvidiaFabricIMEXManagementDevMinor),
		},
		HostTHP:                  b.hostTHP,
		SaveFDs:                  b.saveFDs.GetFDs(),
		SaveCheckpointGofer:      b.saveCheckpointGofer,
		FSSaveFDs:                b.fsSaveFDs.GetFDs(),
		FSSaveCheckpointGofer:    b.fsSaveCheckpointGofer,
		FSRestoreFDs:             b.fsRestoreFDs.GetFDs(),
		FSRestoreCheckpointGofer: b.fsRestoreCheckpointGofer,
		RootfsUpperTarFD:         b.rootfsUpperTarFD,
		StartupTimer:             timer,
	}
	l, err := boot.New(bootArgs)
	if err != nil {
		util.Fatalf("creating loader: %v", err)
	}
	timer.Reached("loader created")

	// Fatalf exits the process and doesn't run defers.
	// 'l' must be destroyed explicitly after this point!

	if b.procMountSyncFD != -1 {
		l.PreSeccompCallback = func() {
			// Call validateOpenFDs() before umounting /proc.
			validateOpenFDs(bootArgs.PassFDs)
			// This callback runs on the same goroutine as the rest of sandbox
			// startup (for now at least...), so safe to record here.
			timer.Reached("open FDs validated")
			// Umount /proc right before installing seccomp filters.
			if b.procMountSyncFile != nil {
				sandboxsetup.UmountProcFile(b.procMountSyncFile)
			} else {
				sandboxsetup.UmountProc(b.procMountSyncFD)
			}
		}
	}

	// Prepare metrics.
	// This needs to happen after the kernel is initialized (such that all metrics are registered)
	// but before the start-sync file is notified, as the parent process needs to query for
	// registered metrics prior to sending the start signal.
	metric.Initialize()
	timer.Reached("metrics initialized")
	var finalMetricsFile *os.File
	if b.finalMetricsFD != -1 {
		finalMetricsFile = os.NewFile(uintptr(b.finalMetricsFD), "final metrics file")
	}
	if b.profilingMetricsFD != -1 {
		if err := metric.StartProfilingMetrics(metric.ProfilingMetricsOptions[*os.File]{
			Sink:    os.NewFile(uintptr(b.profilingMetricsFD), "metrics file"),
			Lossy:   b.profilingMetricsLossy,
			Metrics: conf.ProfilingMetrics,
			Rate:    time.Duration(conf.ProfilingMetricsRate) * time.Microsecond,
		}); err != nil {
			l.Destroy()
			util.Fatalf("unable to start profiling metrics: %v", err)
		}
		defer metric.StopProfilingMetrics()
	}

	// Notify the parent process the sandbox has booted (and that the controller
	// is up).
	timer.Reached("notifying parent")
	startSyncFile := os.NewFile(uintptr(b.startSyncFD), "start-sync file")
	buf := make([]byte, 1)
	if w, err := startSyncFile.Write(buf); err != nil || w != 1 {
		l.Destroy()
		util.Fatalf("unable to write into the start-sync descriptor: %v", err)
	}
	// Closes startSyncFile because 'l.Run()' only returns when the sandbox exits.
	startSyncFile.Close()

	// Wait for the start signal from runsc.
	l.WaitForStartSignal()
	timer.Reached("start signal received")

	// Run the application and wait for it to finish.
	if err := l.Run(); err != nil {
		l.Destroy()
		util.Fatalf("running sandbox: %v", err)
	}

	timer.Log()

	ws := l.WaitExit()
	log.Infof("application exiting with %+v", ws)
	waitStatus := args[1].(*unix.WaitStatus)
	*waitStatus = unix.WaitStatus(ws)

	if finalMetricsFile != nil {
		if err := exportFinalMetrics(finalMetricsFile); err != nil {
			l.Destroy()
			util.Fatalf("unable to export final metrics: %v", err)
		}
	}

	l.Destroy()
	return subcommands.ExitSuccess
}

// validateOpenFDs checks that the sandbox process does not have any open
// directory FDs.
func validateOpenFDs(passFDs []boot.FDMapping) {
	passHostFDs := make(map[int]struct{})
	for _, passFD := range passFDs {
		passHostFDs[passFD.Host] = struct{}{}
	}
	const selfFDDir = "/proc/self/fd"
	if err := filepath.WalkDir(selfFDDir, func(path string, d os.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.Type() != os.ModeSymlink {
			// All entries are symlinks. Ignore the callback for fd directory itself.
			return nil
		}
		if fdInfo, err := os.Stat(path); err != nil {
			if os.IsNotExist(err) {
				// Ignore FDs that are now closed. For example, the FD to selfFDDir that
				// was opened by filepath.WalkDir() to read dirents.
				return nil
			}
			return fmt.Errorf("os.Stat(%s) failed: %v", path, err)
		} else if !fdInfo.IsDir() {
			return nil
		}
		// Uh-oh. This is a directory FD.
		fdNo, err := strconv.Atoi(d.Name())
		if err != nil {
			return fmt.Errorf("strconv.Atoi(%s) failed: %v", d.Name(), err)
		}
		dirLink, err := os.Readlink(path)
		if err != nil {
			return fmt.Errorf("os.Readlink(%s) failed: %v", path, err)
		}
		if _, ok := passHostFDs[fdNo]; ok {
			// Passed FDs are allowed to be directories. The user must be knowing
			// what they are doing. Log a warning regardless.
			log.Warningf("Sandbox has access to FD %d, which is a directory for %s", fdNo, dirLink)
			return nil
		}
		return fmt.Errorf("FD %d is a directory for %s", fdNo, dirLink)
	}); err != nil {
		util.Fatalf("WalkDir(%s) failed: %v", selfFDDir, err)
	}
}

// exportFinalMetrics exports all metrics to the given file.
// The file is closed as part of this function.
func exportFinalMetrics(f *os.File) error {
	snapshot, err := metric.GetSnapshot(metric.SnapshotOptions{})
	if err != nil {
		return fmt.Errorf("getting metrics snapshot: %w", err)
	}
	_, err = prometheus.Write(f, prometheus.ExportOptions{}, map[*prometheus.Snapshot]prometheus.SnapshotExportOptions{
		snapshot: {
			ExporterPrefix: "runsc_",
		},
	})
	if err != nil {
		return fmt.Errorf("writing metrics snapshot: %w", err)
	}
	if err = f.Close(); err != nil {
		return fmt.Errorf("closing metrics snapshot file: %w", err)
	}
	return nil
}