visor/runsc/boot/vfs.go

// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boot

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"slices"
	"sort"
	"strconv"
	"strings"

	specs "github.com/opencontainers/runtime-spec/specs-go"
	"gvisor.dev/gvisor/pkg/abi/ib"
	"gvisor.dev/gvisor/pkg/abi/linux"
	"gvisor.dev/gvisor/pkg/abi/nvgpu"
	"gvisor.dev/gvisor/pkg/cleanup"
	"gvisor.dev/gvisor/pkg/context"
	"gvisor.dev/gvisor/pkg/devutil"
	"gvisor.dev/gvisor/pkg/errors/linuxerr"
	"gvisor.dev/gvisor/pkg/fd"
	"gvisor.dev/gvisor/pkg/fspath"
	"gvisor.dev/gvisor/pkg/fsutil"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/rdma"
	"gvisor.dev/gvisor/pkg/sentry/checkpoint"
	"gvisor.dev/gvisor/pkg/sentry/devices/memdev"
	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy"
	"gvisor.dev/gvisor/pkg/sentry/devices/nvproxy/nvconf"
	"gvisor.dev/gvisor/pkg/sentry/devices/rdmaproxy"
	"gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy"
	"gvisor.dev/gvisor/pkg/sentry/devices/tpuproxy/vfio"
	"gvisor.dev/gvisor/pkg/sentry/devices/ttydev"
	"gvisor.dev/gvisor/pkg/sentry/devices/tundev"
	cgroup2fs "gvisor.dev/gvisor/pkg/sentry/fsimpl/cgroup2fs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/cgroupfs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/dev"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/devpts"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/devtmpfs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/erofs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/fuse"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/gofer"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/mqfs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/overlay"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/proc"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/sys"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
	"gvisor.dev/gvisor/pkg/sentry/fsimpl/user"
	"gvisor.dev/gvisor/pkg/sentry/inet"
	"gvisor.dev/gvisor/pkg/sentry/kernel"
	"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
	"gvisor.dev/gvisor/pkg/sentry/usage"
	"gvisor.dev/gvisor/pkg/sentry/vfs"
	"gvisor.dev/gvisor/pkg/timing"
	"gvisor.dev/gvisor/runsc/config"
	"gvisor.dev/gvisor/runsc/specutils"
)

// Supported filesystems that map to different internal filesystems.
const (
	Bind   = "bind"
	Nonefs = "none"
)

// SelfFilestorePath returns the path at which the self filestore file is
// stored for a given mount.
func SelfFilestorePath(mountSrc, sandboxID string) string {
	// We will place the filestore file in a gVisor specific hidden file inside
	// the mount being overlaid itself. The same volume can be overlaid by
	// multiple sandboxes. So make the filestore file unique to a sandbox by
	// suffixing the sandbox ID.
	return path.Join(mountSrc, selfFilestoreName(sandboxID))
}

// SelfFilestoreName returns the name of the self filestore file for a given sandbox.
func SelfFilestoreName(sandboxID string) string {
	return selfFilestoreName(sandboxID)
}

func selfFilestoreName(sandboxID string) string {
	return fsutil.SelfFilestorePrefix + sandboxID
}

// tmpfs has some extra supported options that we must pass through.
var tmpfsAllowedData = []string{"mode", "size", "uid", "gid"}

func cgroupfsCPUDefaults(rawQuota, rawPeriod int64) map[string]int64 {
	const defaultPeriod int64 = 100000
	const defaultQuota int64 = -1
	period := rawPeriod
	if period <= 0 {
		period = defaultPeriod
	}
	quota := rawQuota
	if quota <= 0 {
		quota = defaultQuota
	}
	return map[string]int64{
		"cpu.cfs_quota_us":  quota,
		"cpu.cfs_period_us": period,
	}
}

func cgroupfsMemoryDefaults(memoryLimit uint64) map[string]int64 {
	if memoryLimit == 0 {
		return nil
	}
	return map[string]int64{
		"memory.limit_in_bytes": int64(memoryLimit),
	}
}

func registerFilesystems(k *kernel.Kernel, info *containerInfo, rdmaSnapshot *rdma.Snapshot) error {
	ctx := k.SupervisorContext()
	vfsObj := k.VFS()

	vfsObj.MustRegisterFilesystemType(cgroupfs.Name, &cgroupfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	if info.conf.InSandboxCgroup == config.InSandboxCgroupV2 {
		vfsObj.MustRegisterFilesystemType(cgroup2fs.Name, &cgroup2fs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
			AllowUserMount: true,
			AllowUserList:  true,
		})
	}
	vfsObj.MustRegisterFilesystemType(devpts.Name, &devpts.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList:  true,
		AllowUserMount: true,
	})
	vfsObj.MustRegisterFilesystemType(dev.Name, &dev.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{})
	vfsObj.MustRegisterFilesystemType(devtmpfs.Name, &devtmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(erofs.Name, &erofs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
	})
	vfsObj.MustRegisterFilesystemType(fuse.Name, &fuse.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(gofer.Name, &gofer.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserList: true,
	})
	vfsObj.MustRegisterFilesystemType(overlay.Name, &overlay.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(proc.Name, &proc.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(sys.Name, &sys.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(tmpfs.Name, &tmpfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})
	vfsObj.MustRegisterFilesystemType(mqfs.Name, &mqfs.FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{
		AllowUserMount: true,
		AllowUserList:  true,
	})

	// Register devices.
	if err := memdev.Register(vfsObj); err != nil {
		return fmt.Errorf("registering memdev: %w", err)
	}
	if err := ttydev.Register(vfsObj); err != nil {
		return fmt.Errorf("registering ttydev: %w", err)
	}
	tunSupported := tundev.IsNetTunSupported(inet.StackFromContext(ctx))
	if tunSupported {
		if err := tundev.Register(vfsObj); err != nil {
			return fmt.Errorf("registering tundev: %v", err)
		}
	}
	if err := fuse.Register(vfsObj); err != nil {
		return fmt.Errorf("registering fusedev: %w", err)
	}

	if err := nvproxyRegisterDevices(info, vfsObj, k.NvidiaDriverVersion); err != nil {
		return err
	}

	if err := rdmaproxyRegisterDevices(info, vfsObj, rdmaSnapshot); err != nil {
		return err
	}

	return nil
}

// rdmaproxyRegisterDevices registers a proxied VFS char device for each
// /dev/infiniband/uverbs* device in the spec, so the sandbox can drive RDMA
// verbs against the host. Devices are registered at the fixed uverbs
// char-device major (see rdmaproxy.Register), which matches the major the
// guest device node and its /sys .../dev entry carry. The vendor driver
// plug-in is selected per device from the PCI driver name in the host sysfs
// snapshot (e.g. mlx5_core -> cxproxy).
func rdmaproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem, snapshot *rdma.Snapshot) error {
	if snapshot == nil || !specutils.RDMAEnabled(info.spec, info.conf) {
		return nil
	}
	// Map host uverbs minor -> PCI driver name, resolved from the leaf PCI
	// node's uevent in the snapshot.
	leafUevent := make(map[string]string, len(snapshot.PCINodes))
	for i := range snapshot.PCINodes {
		leafUevent[snapshot.PCINodes[i].Path] = snapshot.PCINodes[i].Attrs["uevent"]
	}
	driverByMinor := make(map[uint32]string)
	for i := range snapshot.Devices {
		dev := &snapshot.Devices[i]
		// Snapshot attribute values are verbatim file contents; trim before
		// parsing "major:minor".
		devStr := strings.TrimSpace(dev.Dev)
		sep := strings.IndexByte(devStr, ':')
		if sep < 0 {
			continue
		}
		minor64, err := strconv.ParseUint(devStr[sep+1:], 10, 32)
		if err != nil {
			continue
		}
		for _, line := range strings.Split(leafUevent[dev.LeafPCI], "\n") {
			if driver, ok := strings.CutPrefix(line, "DRIVER="); ok {
				driverByMinor[uint32(minor64)] = driver
				break
			}
		}
	}
	for _, devSpec := range info.spec.Linux.Devices {
		if !strings.HasPrefix(devSpec.Path, "/dev/infiniband/uverbs") {
			continue
		}
		// We expect the host uverbs devices to carry the fixed InfiniBand
		// uverbs major (IB_UVERBS_MAJOR). A device with any other major is one
		// the host assigned a dynamically-allocated number (happen with a host
		// with >32 RDMA devices), which we currently do not support.
		if uint32(devSpec.Major) != ib.IB_UVERBS_MAJOR {
			return fmt.Errorf("rdma: uverbs device %s has char-device major %d, want %d: devices with a dynamically-allocated major are not supported", devSpec.Path, devSpec.Major, ib.IB_UVERBS_MAJOR)
		}
		minor := uint32(devSpec.Minor)
		devName := filepath.Base(devSpec.Path)
		driverName := driverByMinor[minor]
		if err := rdmaproxy.Register(vfsObj, devName, minor, driverName, snapshot.VerbsABIVersion); err != nil {
			log.Warningf("rdma: register %s: %v", devSpec.Path, err)
			continue
		}
		log.Infof("rdma: registered %s minor=%d driver=%q", devSpec.Path, minor, driverName)
	}
	return nil
}

func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *containerMounter, procArgs *kernel.CreateProcessArgs) error {
	// Create context with root credentials to mount the filesystem (the current
	// user may not be privileged enough).
	rootCreds := auth.NewRootCredentials(procArgs.Credentials.UserNamespace)
	rootProcArgs := *procArgs
	rootProcArgs.WorkingDirectory = "/"
	rootProcArgs.Credentials = rootCreds
	rootProcArgs.Umask = 0022
	rootProcArgs.MaxSymlinkTraversals = linux.MaxSymlinkTraversals
	rootCtx := rootProcArgs.NewContext(mntr.l.k)

	mns, err := mntr.mountAll(rootCtx, rootCreds, info.spec, info.conf, &rootProcArgs)
	if err != nil {
		return fmt.Errorf("failed to setupFS: %w", err)
	}
	procArgs.MountNamespace = mns

	// If cgroups are mounted, then only check for the cgroup mounts per
	// container. Otherwise the root cgroups will be enabled.
	if mntr.cgroupsMounted && info.conf.InSandboxCgroup != config.InSandboxCgroupV2 {
		cgroupRegistry := mntr.l.k.CgroupRegistry()
		for _, ctrl := range kernel.CgroupCtrls {
			cg, err := cgroupRegistry.FindCgroup(ctx, ctrl, "/"+mntr.containerID)
			if err != nil {
				return fmt.Errorf("cgroup mount for controller %v not found", ctrl)
			}
			if procArgs.InitialCgroups == nil {
				procArgs.InitialCgroups = make(map[kernel.Cgroup]struct{}, len(kernel.CgroupCtrls))
			}
			procArgs.InitialCgroups[cg] = struct{}{}
		}
	}

	mnsRoot := mns.Root(rootCtx)
	defer mnsRoot.DecRef(rootCtx)

	if err := createDeviceFiles(rootCtx, rootCreds, info, mntr.l.k.VFS(), mnsRoot); err != nil {
		return fmt.Errorf("failed to create device files: %w", err)
	}
	procArgs.StartupTimeline.Reached("device files created")

	if err := mntr.l.k.VFS().MkdirAllAt(
		ctx, procArgs.WorkingDirectory, mnsRoot, rootCreds,
		&vfs.MkdirOptions{Mode: 0755}, true, /* mustBeDir */
	); err != nil {
		return fmt.Errorf("failed to create process working directory %q: %w",
			procArgs.WorkingDirectory, err)
	}

	// We are executing a file directly. Do not resolve the executable path.
	if procArgs.File != nil {
		return nil
	}
	// Resolve the executable path from working dir and environment.
	resolved, err := user.ResolveExecutablePath(ctx, procArgs)
	if err != nil {
		return err
	}
	procArgs.Filename = resolved
	procArgs.StartupTimeline.Reached("executable path resolved")
	return nil
}

// compileMounts returns the supported mounts from the mount spec, adding any
// mandatory mounts that are required by the OCI specification.
//
// This function must NOT add/remove any gofer mounts or change their order.
func compileMounts(spec *specs.Spec, conf *config.Config, containerID string) []specs.Mount {
	// Keep track of whether proc and sys were mounted.
	var procMounted, sysMounted, devMounted, devptsMounted, cgroupsMounted bool
	var mounts []specs.Mount

	// Mount all submounts from the spec.
	for _, m := range spec.Mounts {
		if conf.InSandboxCgroup == config.InSandboxCgroupV2 {
			// Under this flag, we only want a single unified mount at
			// /sys/fs/cgroup. Skip any legacy v1 controller sub-mounts
			// (e.g., /sys/fs/cgroup/cpu) requested by the OCI spec.
			if (m.Type == cgroupfs.Name || m.Type == cgroup2fs.Name) && strings.HasPrefix(filepath.Clean(m.Destination), "/sys/fs/cgroup/") {
				continue
			}
		} else {
			// Mount all the cgroup controllers when "/sys/fs/cgroup" mount
			// is present. If any other cgroup controller mounts are there,
			// it will be a no-op, drop them.
			if m.Type == cgroupfs.Name && cgroupsMounted {
				continue
			}
		}

		switch filepath.Clean(m.Destination) {
		case "/proc":
			procMounted = true
		case "/sys":
			sysMounted = true
		case "/dev":
			m.Type = dev.Name
			devMounted = true
		case "/dev/pts":
			m.Type = devpts.Name
			devptsMounted = true
		case "/sys/fs/cgroup":
			cgroupsMounted = true
		}

		mounts = append(mounts, m)
	}

	// Mount proc and sys even if the user did not ask for it, as the spec
	// says we SHOULD.
	var mandatoryMounts []specs.Mount

	if !procMounted {
		mandatoryMounts = append(mandatoryMounts, specs.Mount{
			Type:        proc.Name,
			Destination: "/proc",
		})
	}
	if !sysMounted {
		mandatoryMounts = append(mandatoryMounts, specs.Mount{
			Type:        sys.Name,
			Destination: "/sys",
		})
	}
	if !devMounted {
		mandatoryMounts = append(mandatoryMounts, specs.Mount{
			Type:        dev.Name,
			Destination: "/dev",
		})
	}
	if !devptsMounted {
		mandatoryMounts = append(mandatoryMounts, specs.Mount{
			Type:        devpts.Name,
			Destination: "/dev/pts",
		})
	}

	// The mandatory mounts should be ordered right after the root, in case
	// there are submounts of these mandatory mounts already in the spec.
	mounts = append(mounts[:0], append(mandatoryMounts, mounts[0:]...)...)

	return mounts
}

// goferMountData creates a slice of gofer mount data.
func goferMountData(fd int, fa config.FileAccessType, conf *config.Config, suppressDirectFS bool) []string {
	opts := []string{
		"trans=fd",
		"rfdno=" + strconv.Itoa(fd),
		"wfdno=" + strconv.Itoa(fd),
	}
	if fa == config.FileAccessShared {
		opts = append(opts, "cache=remote_revalidating")
	}
	if conf.DirectFS && !suppressDirectFS {
		opts = append(opts, "directfs")
	}
	if !conf.HostFifo.AllowOpen() {
		opts = append(opts, "disable_fifo_open")
	}
	if conf.CharacterDevicePolicy.AllowsPassthrough() {
		opts = append(opts, "char_device_policy="+conf.CharacterDevicePolicy.String())
	}
	return opts
}

// consumeMountOptions consumes mount options from opts based on allowedKeys
// and returns the remaining and consumed options.
func consumeMountOptions(opts []string, allowedKeys ...string) ([]string, []string, error) {
	var rem, out []string
	for _, o := range opts {
		ok, err := parseMountOption(o, allowedKeys...)
		if err != nil {
			return nil, nil, err
		}
		if ok {
			out = append(out, o)
		} else {
			rem = append(rem, o)
		}
	}
	return rem, out, nil
}

func parseMountOption(opt string, allowedKeys ...string) (bool, error) {
	kv := strings.SplitN(opt, "=", 3)
	if len(kv) > 2 {
		return false, fmt.Errorf("invalid option %q", opt)
	}
	return slices.Contains(allowedKeys, kv[0]), nil
}

type fdDispenser struct {
	fds []*fd.FD
}

func (f *fdDispenser) remove() int {
	return f.removeAsFD().Release()
}

func (f *fdDispenser) removeAsFD() *fd.FD {
	if f.empty() {
		panic("fdDispenser out of fds")
	}
	rv := f.fds[0]
	f.fds = f.fds[1:]
	return rv
}

func (f *fdDispenser) empty() bool {
	return len(f.fds) == 0
}

type containerMounter struct {
	l *Loader

	root *specs.Root

	// mounts is the set of submounts for the container. It's a copy from the spec
	// that may be freely modified without affecting the original spec.
	mounts []specs.Mount

	// goferFDs is the list of FDs to be dispensed for gofer mounts.
	goferFDs fdDispenser

	// goferFilestoreFDs are FDs to the regular files that will back the tmpfs or
	// overlayfs mount for certain gofer mounts.
	goferFilestoreFDs fdDispenser

	// devGoferFD is the FD to attach the sandbox to the dev gofer.
	devGoferFD *fd.FD

	// goferMountConfs contains information about how the gofer mounts have been
	// configured. The first entry is for rootfs and the following entries are
	// for bind mounts in Spec.Mounts (in the same order).
	goferMountConfs []specutils.GoferMountConf

	// sharedMounts is a map of shared mounts that can be reused across
	// containers.
	//
	// FIXME: This is equivalent to l.sharedMounts, but lacks the checklocks
	// annotation.
	sharedMounts map[string]*vfs.Mount

	// containerID is the ID for the container.
	containerID   string
	containerName string

	// cgroupsMounted indicates if cgroups are mounted in the container.
	// This is used to set the InitialCgroups before starting the container
	// process.
	cgroupsMounted bool

	// rootfsUpperTarFD is the file descriptor to the tar file containing the rootfs
	// upper layer changes.
	rootfsUpperTarFD *fd.FD
}

// +checklocks:l.mu
func (l *Loader) newContainerMounter(info *containerInfo) *containerMounter {
	return &containerMounter{
		l:                 l,
		root:              info.spec.Root,
		mounts:            compileMounts(info.spec, info.conf, info.procArgs.ContainerID),
		goferFDs:          fdDispenser{fds: info.goferFDs},
		goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs},
		devGoferFD:        info.devGoferFD,
		goferMountConfs:   info.goferMountConfs,
		sharedMounts:      l.sharedMounts,
		containerID:       info.cid,
		containerName:     info.containerName,
		rootfsUpperTarFD:  info.rootfsUpperTarFD,
	}
}

func (c *containerMounter) checkDispenser() error {
	if !c.goferFDs.empty() {
		return fmt.Errorf("not all gofer FDs were consumed, remaining: %v", c.goferFDs)
	}
	if !c.goferFilestoreFDs.empty() {
		return fmt.Errorf("not all gofer Filestore FDs were consumed, remaining: %v", c.goferFilestoreFDs)
	}
	if c.devGoferFD != nil && c.devGoferFD.FD() >= 0 {
		return fmt.Errorf("dev gofer FD was not consumed: %d", c.devGoferFD.FD())
	}
	return nil
}

func getMountAccessType(conf *config.Config, hint *MountHint) config.FileAccessType {
	if hint != nil {
		return hint.fileAccessType()
	}
	return conf.FileAccessMounts
}

func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Credentials, spec *specs.Spec, conf *config.Config, rootProcArgs *kernel.CreateProcessArgs) (*vfs.MountNamespace, error) {
	log.Infof("Configuring container's file system")

	mns, err := c.createMountNamespace(rootCtx, spec, conf, rootCreds)
	if err != nil {
		return nil, fmt.Errorf("creating mount namespace: %w", err)
	}
	rootProcArgs.StartupTimeline.Reached("rootfs mounted")
	rootProcArgs.MountNamespace = mns

	root := mns.Root(rootCtx)
	defer root.DecRef(rootCtx)
	if root.Mount().ReadOnly() {
		// Switch to ReadWrite while we setup submounts.
		if err := c.l.k.VFS().SetMountReadOnly(root.Mount(), false); err != nil {
			return nil, fmt.Errorf(`failed to set mount at "/" readwrite: %w`, err)
		}
		// Restore back to ReadOnly at the end.
		defer func() {
			if err := c.l.k.VFS().SetMountReadOnly(root.Mount(), true); err != nil {
				panic(fmt.Sprintf(`failed to restore mount at "/" back to readonly: %v`, err))
			}
		}()
	}

	// Mount submounts.
	if err := c.mountSubmounts(rootCtx, spec, conf, mns, rootCreds, rootProcArgs.StartupTimeline); err != nil {
		return nil, fmt.Errorf("mounting submounts: %w", err)
	}
	rootProcArgs.StartupTimeline.Reached("submounts mounted")

	return mns, nil
}

// createMountNamespace creates the container's root mount and namespace.
func (c *containerMounter) createMountNamespace(ctx context.Context, spec *specs.Spec, conf *config.Config, creds *auth.Credentials) (*vfs.MountNamespace, error) {
	ioFD := c.goferFDs.remove()
	rootfsConf := c.goferMountConfs[0]

	rootfsHint, err := NewRootfsHint(spec)
	if err != nil {
		return nil, fmt.Errorf("parsing rootfs hint: %w", err)
	}
	suppressDirectFS := rootfsHint != nil && rootfsHint.SuppressDirectFS

	var (
		fsName string
		opts   *vfs.MountOptions
	)
	switch {
	case rootfsConf.ShouldUseLisafs():
		fsName = gofer.Name

		data := goferMountData(ioFD, conf.FileAccess, conf, suppressDirectFS)

		// We can't check for overlayfs here because sandbox is chroot'ed and gofer
		// can only send mount options for specs.Mounts (specs.Root is missing
		// Options field). So assume root is always on top of overlayfs.
		data = append(data, "overlayfs_stale_read")

		// Configure the gofer dentry cache size.
		gofer.SetDentryCacheSize(conf.DCache)

		opts = &vfs.MountOptions{
			ReadOnly: c.root.Readonly,
			GetFilesystemOptions: vfs.GetFilesystemOptions{
				InternalMount: true,
				Data:          strings.Join(data, ","),
				InternalData: gofer.InternalFilesystemOptions{
					UniqueID: checkpoint.ResourceID{
						ContainerName: c.containerName,
						Path:          "/",
					},
				},
			},
		}

	case rootfsConf.ShouldUseErofs():
		fsName = erofs.Name
		opts = &vfs.MountOptions{
			ReadOnly: c.root.Readonly,
			GetFilesystemOptions: vfs.GetFilesystemOptions{
				InternalMount: true,
				Data:          fmt.Sprintf("ifd=%d", ioFD),
				InternalData: erofs.InternalFilesystemOptions{
					UniqueID: checkpoint.ResourceID{
						ContainerName: c.containerName,
						Path:          "/",
					},
				},
			},
		}

	default:
		return nil, fmt.Errorf("unsupported rootfs config: %+v", rootfsConf)
	}

	log.Infof("Mounting root with %s, ioFD: %d", fsName, ioFD)

	if rootfsConf.ShouldUseOverlayfs() {
		log.Infof("Adding overlay on top of root")
		var (
			err         error
			cleanup     func()
			filestoreFD *fd.FD
		)
		if rootfsConf.IsFilestorePresent() {
			filestoreFD = c.goferFilestoreFDs.removeAsFD()
		}
		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, filestoreFD, rootfsConf, "/", c.rootfsUpperTarFD)
		if err != nil {
			return nil, fmt.Errorf("mounting root with overlay: %w", err)
		}
		defer cleanup()
		fsName = overlay.Name
	} else {
		if c.rootfsUpperTarFD != nil {
			return nil, fmt.Errorf("rootfs-upper-tar-fd is set when overlay is disabled for rootfs: rootfsConf=%s", rootfsConf)
		}
	}

	// The namespace root mount can't be changed, so let's mount a dummy
	// read-only tmpfs here. It simplifies creation of containers without
	// leaking the root file system.
	mns, err := c.l.k.VFS().NewMountNamespace(ctx, creds, "rootfs", "tmpfs",
		&vfs.MountOptions{ReadOnly: true, Locked: true}, c.l.k)
	if err != nil {
		return nil, fmt.Errorf("setting up mount namespace: %w", err)
	}
	defer mns.DecRef(ctx)

	mnt, err := c.l.k.VFS().MountDisconnected(ctx, creds, "root", fsName, opts)
	if err != nil {
		return nil, fmt.Errorf("creating root file system: %w", err)
	}
	defer mnt.DecRef(ctx)
	root := mns.Root(ctx)
	defer root.DecRef(ctx)
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
	}
	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
		return nil, fmt.Errorf("mounting root file system: %w", err)
	}

	mns.IncRef()
	return mns, nil
}

// configureOverlay mounts the lower layer using "lowerOpts", mounts the upper
// layer using tmpfs, and return overlay mount options. "cleanup" must be called
// after the options have been used to mount the overlay, to release refs on
// lower and upper mounts.
func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, filestoreFD *fd.FD, mountConf specutils.GoferMountConf, dst string, rootfsUpperTarFD *fd.FD) (*vfs.MountOptions, func(), error) {
	// First copy options from lower layer to upper layer and overlay. Clear
	// filesystem specific options.
	upperOpts := *lowerOpts
	upperOpts.GetFilesystemOptions = vfs.GetFilesystemOptions{InternalMount: true}
	if mountConf.Size != "" {
		if upperOpts.GetFilesystemOptions.Data != "" {
			upperOpts.GetFilesystemOptions.Data += ","
		}
		upperOpts.GetFilesystemOptions.Data += "size=" + mountConf.Size
	}
	overlayOpts := *lowerOpts
	overlayOpts.GetFilesystemOptions = vfs.GetFilesystemOptions{InternalMount: true}

	// All writes go to the upper layer, be paranoid and make lower readonly.
	lowerOpts.ReadOnly = true
	lower, err := c.l.k.VFS().MountDisconnected(ctx, creds, "" /* source */, lowerFSName, lowerOpts)
	if err != nil {
		return nil, nil, err
	}
	cu := cleanup.Make(func() { lower.DecRef(ctx) })
	defer cu.Clean()

	// Determine the lower layer's root's type.
	lowerRootVD := vfs.MakeVirtualDentry(lower, lower.Root())
	stat, err := c.l.k.VFS().StatAt(ctx, creds, &vfs.PathOperation{
		Root:  lowerRootVD,
		Start: lowerRootVD,
	}, &vfs.StatOptions{
		Mask: linux.STATX_UID | linux.STATX_GID | linux.STATX_MODE | linux.STATX_TYPE,
	})
	if err != nil {
		return nil, nil, fmt.Errorf("failed to stat lower layer's root: %v", err)
	}
	if stat.Mask&linux.STATX_TYPE == 0 {
		return nil, nil, fmt.Errorf("failed to get file type of lower layer's root")
	}
	rootType := stat.Mode & linux.S_IFMT
	if rootType != linux.S_IFDIR && rootType != linux.S_IFREG {
		return nil, nil, fmt.Errorf("lower layer's root has unsupported file type %v", rootType)
	}

	// Upper is a tmpfs mount to keep all modifications inside the sandbox.
	tmpfsOpts := tmpfs.FilesystemOpts{
		RootFileType: uint16(rootType),
		// If a mount is being overlaid, it should not be limited by the default
		// tmpfs size limit.
		DisableDefaultSizeLimit: true,
	}
	if filestoreFD != nil {
		// Create memory file for disk-backed overlays.
		resourceID := checkpoint.ResourceID{ContainerName: c.containerName, Path: dst}
		mf, err := createPrivateMemoryFile(filestoreFD.ReleaseToFile("overlay-filestore"), resourceID, c.containerID, c.l.fsRestore)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to create memory file for overlay: %v", err)
		}
		tmpfsOpts.MemoryFile = mf
		sourceTar, err := c.l.fsRestore.tmpfsSourceTar(resourceID, c.containerID)
		if err != nil {
			mf.Destroy()
			return nil, nil, fmt.Errorf("failed to get tar archive from filesystem checkpoint: %w", err)
		}
		if sourceTar != nil {
			log.Infof("Loading filesystem checkpoint tree for %q", resourceID)
			tmpfsOpts.SourceTar = sourceTar
			tmpfsOpts.SourceTarFSCheckpoint = true
		}
	}
	// If the rootfs upper tar file is provided, it will be applied to the
	// tmpfs which is on the upper layer of the root's overlay fs.
	if rootfsUpperTarFD != nil {
		tmpfsOpts.SourceTar = rootfsUpperTarFD.ReleaseToFile("rootfs-upper-tar-fd")
	}
	upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts
	upper, err := c.l.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to create upper layer for overlay, opts: %+v: %v", upperOpts, err)
	}
	cu.Add(func() { upper.DecRef(ctx) })

	// If the overlay mount consists of a regular file, copy up its contents
	// from the lower layer, since in the overlay the otherwise-empty upper
	// layer file will take precedence.
	upperRootVD := vfs.MakeVirtualDentry(upper, upper.Root())
	if rootType == linux.S_IFREG {
		lowerFD, err := c.l.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
			Root:  lowerRootVD,
			Start: lowerRootVD,
		}, &vfs.OpenOptions{
			Flags: linux.O_RDONLY,
		})
		if err != nil {
			return nil, nil, fmt.Errorf("failed to open lower layer root for copying: %v", err)
		}
		defer lowerFD.DecRef(ctx)
		upperFD, err := c.l.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
			Root:  upperRootVD,
			Start: upperRootVD,
		}, &vfs.OpenOptions{
			Flags: linux.O_WRONLY,
		})
		if err != nil {
			return nil, nil, fmt.Errorf("failed to open upper layer root for copying: %v", err)
		}
		defer upperFD.DecRef(ctx)
		if _, err := vfs.CopyRegularFileData(ctx, upperFD, lowerFD); err != nil {
			return nil, nil, fmt.Errorf("failed to copy up overlay file: %v", err)
		}
	}

	// We need to hide the filestore from the containerized application.
	if mountConf.IsSelfBacked() {
		if err := overlay.CreateWhiteout(ctx, c.l.k.VFS(), creds, &vfs.PathOperation{
			Root:  upperRootVD,
			Start: upperRootVD,
			Path:  fspath.Parse(selfFilestoreName(c.l.sandboxID)),
		}); err != nil {
			return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err)
		}
	}

	// Propagate the lower layer's root's owner, group, and mode to the upper
	// layer's root for consistency with VFS1.
	err = c.l.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{
		Root:  upperRootVD,
		Start: upperRootVD,
	}, &vfs.SetStatOptions{
		Stat: linux.Statx{
			Mask: (linux.STATX_UID | linux.STATX_GID | linux.STATX_MODE) & stat.Mask,
			UID:  stat.UID,
			GID:  stat.GID,
			Mode: stat.Mode,
		},
	})
	if err != nil {
		return nil, nil, err
	}

	// Configure overlay with both layers.
	overlayOpts.GetFilesystemOptions.InternalData = overlay.FilesystemOptions{
		UpperRoot:  upperRootVD,
		LowerRoots: []vfs.VirtualDentry{lowerRootVD},
	}
	return &overlayOpts, cu.Release(), nil
}

func (c *containerMounter) mountSubmounts(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, timeline *timing.Timeline) error {
	mounts, err := c.prepareMounts()
	if err != nil {
		return err
	}
	timeline.Reached("submounts prepared")

	for i := range mounts {
		submount := &mounts[i]
		log.Debugf("Mounting %q to %q, type: %s, options: %s", submount.mount.Source, submount.mount.Destination, submount.mount.Type, submount.mount.Options)
		var (
			mnt *vfs.Mount
			err error
		)

		if submount.hint != nil && submount.hint.ShouldShareMount() {
			sharedMount, err := c.getSharedMount(ctx, spec, conf, submount, creds)
			if err != nil {
				return fmt.Errorf("getting shared mount %q: %w", submount.hint.Name, err)
			}
			mnt, err = c.mountSharedSubmount(ctx, conf, mns, creds, submount, sharedMount)
			if err != nil {
				return fmt.Errorf("mount shared mount %q to %q: %v", submount.hint.Name, submount.mount.Destination, err)
			}
		} else if submount.mount.Type == cgroupfs.Name || submount.mount.Type == cgroup2fs.Name {
			if conf.InSandboxCgroup == config.InSandboxCgroupV2 {
				// There is no "type: cgroup2" defined in the OCI spec.
				// So, when the runsc flag InSandboxCgroup is set to v2, we honor the
				// OCI request for "type: cgroupfs" by a v2 mount.
				if submount.mount.Type == cgroupfs.Name {
					submount.mount.Type = cgroup2fs.Name
				}
				// The mount is rooted per the container's cgroup namespace
				// (see cgroup2fs.mountRoot): private-ns containers see only their
				// "/<cid>" subtree, created earlier by createContainerCgroup2.
				mnt, err = c.mountSubmount(ctx, spec, conf, mns, creds, submount)
				if err != nil {
					return fmt.Errorf("mount cgroup2 %q: %w", submount.mount.Destination, err)
				}
			} else {
				// Mount all the cgroups controllers.
				if err := c.mountCgroupSubmounts(ctx, spec, conf, mns, creds, submount); err != nil {
					return fmt.Errorf("mount cgroup %q: %w", submount.mount.Destination, err)
				}
			}
		} else {
			mnt, err = c.mountSubmount(ctx, spec, conf, mns, creds, submount)
			if err != nil {
				return fmt.Errorf("mount submount %q: %w", submount.mount.Destination, err)
			}
		}

		if mnt != nil && mnt.ReadOnly() {
			// Switch to ReadWrite while we setup submounts.
			if err := c.l.k.VFS().SetMountReadOnly(mnt, false); err != nil {
				return fmt.Errorf("failed to set mount at %q readwrite: %w", submount.mount.Destination, err)
			}
			// Restore back to ReadOnly at the end.
			defer func() {
				if err := c.l.k.VFS().SetMountReadOnly(mnt, true); err != nil {
					panic(fmt.Sprintf("failed to restore mount at %q back to readonly: %v", submount.mount.Destination, err))
				}
			}()
		}
		if timeline != nil {
			timeline.Reached(fmt.Sprintf("mounted %q", submount.mount.Destination))
		}
	}

	if err := c.mountTmp(ctx, spec, conf, creds, mns); err != nil {
		return fmt.Errorf(`mount submount "/tmp": %w`, err)
	}
	return nil
}

type mountInfo struct {
	mount          *specs.Mount
	goferFD        *fd.FD
	hint           *MountHint
	goferMountConf specutils.GoferMountConf
	filestoreFD    *fd.FD
}

func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
	// If device gofer exists, connect to it.
	if c.devGoferFD != nil {
		if err := c.l.k.AddDevGofer(c.containerName, c.devGoferFD.Release()); err != nil {
			return nil, err
		}
	}
	// Associate bind mounts with their FDs before sorting since there is an
	// undocumented assumption that FDs are dispensed in the order in which
	// they are required by mounts.
	var mounts []mountInfo
	goferMntIdx := 1 // First index is for rootfs.
	for i := range c.mounts {
		info := mountInfo{
			mount: &c.mounts[i],
			hint:  c.l.mountHints.FindMount(c.mounts[i].Source),
		}
		specutils.MaybeConvertToBindMount(info.mount)
		if specutils.HasMountConfig(*info.mount) {
			info.goferMountConf = c.goferMountConfs[goferMntIdx]
			if info.goferMountConf.ShouldUseLisafs() || info.goferMountConf.ShouldUseErofs() {
				info.goferFD = c.goferFDs.removeAsFD()
			}
			if info.goferMountConf.IsFilestorePresent() {
				info.filestoreFD = c.goferFilestoreFDs.removeAsFD()
			}
			if info.goferMountConf.ShouldUseTmpfs() {
				specutils.ChangeMountType(info.mount, tmpfs.Name)
			}
			goferMntIdx++
		}
		mounts = append(mounts, info)
	}
	if err := c.checkDispenser(); err != nil {
		return nil, err
	}

	// Sort the mounts so that we don't place children before parents.
	sort.Slice(mounts, func(i, j int) bool {
		return len(mounts[i].mount.Destination) < len(mounts[j].mount.Destination)
	})

	return mounts, nil
}

// getPathMode returns file mode for the specified path.
func (c *containerMounter) getPathMode(ctx context.Context, creds *auth.Credentials, path *vfs.PathOperation) (linux.FileMode, error) {
	stat, err := c.l.k.VFS().StatAt(ctx, creds, path, &vfs.StatOptions{
		Mask: linux.STATX_TYPE,
	})
	if err != nil {
		return 0, err
	}
	if stat.Mask&linux.STATX_TYPE == 0 {
		return 0, fmt.Errorf("failed to get file type")
	}
	return linux.FileMode(stat.Mode), nil
}

func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) (*vfs.Mount, error) {
	fsName, opts, err := getMountNameAndOptions(spec, conf, submount, c.l.productName, c.containerName, c.containerID, c.l.fsRestore, c.l.rdmaSysfs)
	if err != nil {
		return nil, fmt.Errorf("mountOptions failed: %w", err)
	}
	if len(fsName) == 0 {
		// Filesystem is not supported (e.g. cgroup), just skip it.
		return nil, nil
	}

	if submount.goferMountConf.ShouldUseOverlayfs() {
		log.Infof("Adding overlay on top of mount %q", submount.mount.Destination)
		var cleanup func()
		opts, cleanup, err = c.configureOverlay(ctx, conf, creds, opts, fsName, submount.filestoreFD, submount.goferMountConf, submount.mount.Destination, nil)
		if err != nil {
			return nil, fmt.Errorf("mounting volume with overlay at %q: %w", submount.mount.Destination, err)
		}
		defer cleanup()
		fsName = overlay.Name
	}

	mount := submount.mount
	root := mns.Root(ctx)
	defer root.DecRef(ctx)
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(mount.Destination),
	}
	mnt, err := c.l.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
	if err != nil {
		return nil, fmt.Errorf("mounting %q (type:%s, dest: %q): %w, opts: %v",
			mount.Source, mount.Type, mount.Destination, err, opts)
	}
	defer mnt.DecRef(ctx)

	vd := vfs.MakeVirtualDentry(mnt, mnt.Root())
	rootPath := &vfs.PathOperation{
		Root:  vd,
		Start: vd,
	}

	rootMode, err := c.getPathMode(ctx, creds, rootPath)
	if err != nil {
		return nil, fmt.Errorf("stat mount %q (dest: %q): %w", mount.Source, mount.Destination, err)
	}
	if err := c.makeMountPoint(ctx, creds, mns, mount.Destination, rootMode); err != nil {
		return nil, fmt.Errorf("creating mount point %q: %w", mount.Destination, err)
	}

	// Avoid mounting on top of symlinks. The mount syscall on Linux always follows symlinks.
	target.FollowFinalSymlink = true
	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
		return nil, fmt.Errorf("attaching %q to %q (type: %s): %w, opts: %v",
			mount.Source, mount.Destination, mount.Type, err, opts)
	}

	log.Infof("Mounted %q to %q type: %s, internal-options: %q",
		mount.Source, mount.Destination, mount.Type, opts.GetFilesystemOptions.Data)
	return mnt, nil
}

// getMountNameAndOptions retrieves the fsName, opts, and useOverlay values
// used for mounts.
func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, productName, containerName, containerID string, fsr *fsRestore, rdmaSysfs *rdma.Snapshot) (string, *vfs.MountOptions, error) {
	fsName := m.mount.Type
	var (
		mopts        = m.mount.Options
		data         []string
		internalData any
	)

	// Find filesystem name and FS specific data field.
	switch m.mount.Type {
	case devpts.Name, dev.Name, cgroup2fs.Name:
		// Nothing to do.

	case Nonefs:
		fsName = sys.Name

	case proc.Name:
		internalData = newProcInternalData(conf, spec)

	case sys.Name:
		sysData := &sys.InternalData{
			EnableTPUProxyPaths: specutils.TPUProxyEnabled(spec, conf),
			RDMASysfs:           rdmaSysfs,
		}
		if len(productName) > 0 {
			sysData.ProductName = productName
		}
		internalData = sysData

	case tmpfs.Name:
		var err error
		mopts, data, err = consumeMountOptions(mopts, tmpfsAllowedData...)
		if err != nil {
			return "", nil, err
		}
		if m.filestoreFD != nil {
			resourceID := checkpoint.ResourceID{ContainerName: containerName, Path: m.mount.Destination}
			mf, err := createPrivateMemoryFile(m.filestoreFD.ReleaseToFile("tmpfs-filestore"), resourceID, containerID, fsr)
			if err != nil {
				return "", nil, fmt.Errorf("failed to create memory file for tmpfs: %w", err)
			}
			tmpfsOpts := tmpfs.FilesystemOpts{
				MemoryFile: mf,
				// If a mount is being overlaid with tmpfs, it should not be limited by
				// the default tmpfs size limit.
				DisableDefaultSizeLimit: true,
			}
			sourceTar, err := fsr.tmpfsSourceTar(resourceID, containerID)
			if err != nil {
				return "", nil, fmt.Errorf("failed to get tar archive from filesystem checkpoint: %w", err)
			}
			if sourceTar != nil {
				log.Infof("Loading filesystem checkpoint tree for %q", resourceID)
				tmpfsOpts.SourceTar = sourceTar
				tmpfsOpts.SourceTarFSCheckpoint = true
			}
			internalData = tmpfsOpts
		}

	case Bind:
		fsName = gofer.Name
		if m.goferFD == nil {
			// Check that an FD was provided to fails fast.
			return "", nil, fmt.Errorf("gofer mount requires a connection FD")
		}
		var err error
		mopts, data, err = consumeMountOptions(mopts, gofer.SupportedMountOptions...)
		if err != nil {
			return "", nil, err
		}
		data = append(data, goferMountData(m.goferFD.Release(), getMountAccessType(conf, m.hint), conf, m.hint != nil && m.hint.SuppressDirectFS)...)
		internalData = gofer.InternalFilesystemOptions{
			UniqueID: checkpoint.ResourceID{
				ContainerName: containerName,
				Path:          m.mount.Destination,
			},
		}

	case cgroupfs.Name:
		var err error
		mopts, data, err = consumeMountOptions(mopts, cgroupfs.SupportedMountOptions...)
		if err != nil {
			return "", nil, err
		}

	case erofs.Name:
		if m.goferFD == nil {
			return "", nil, fmt.Errorf("EROFS mount requires an image file FD")
		}
		data = []string{fmt.Sprintf("ifd=%d", m.goferFD.Release())}
		internalData = erofs.InternalFilesystemOptions{
			UniqueID: checkpoint.ResourceID{
				ContainerName: containerName,
				Path:          m.mount.Destination,
			},
		}

	default:
		log.Warningf("ignoring unknown filesystem type %q", m.mount.Type)
		return "", nil, nil
	}

	opts := ParseMountOptions(mopts)
	opts.GetFilesystemOptions = vfs.GetFilesystemOptions{
		Data:          strings.Join(data, ","),
		InternalData:  internalData,
		InternalMount: true,
	}

	return fsName, opts, nil
}

// ParseMountOptions converts specs.Mount.Options to vfs.MountOptions.
func ParseMountOptions(opts []string) *vfs.MountOptions {
	mountOpts := &vfs.MountOptions{
		GetFilesystemOptions: vfs.GetFilesystemOptions{
			InternalMount: true,
		},
	}
	// Note: update mountHint.CheckCompatible when more options are added.
	for _, o := range opts {
		switch o {
		case "ro":
			mountOpts.ReadOnly = true
		case "noatime":
			mountOpts.Flags.NoATime = true
		case "noexec":
			mountOpts.Flags.NoExec = true
		case "nosuid":
			mountOpts.Flags.NoSUID = true
		case "rw", "atime", "exec":
			// These use the default value and don't need to be set.
		case "bind", "rbind":
			// These are the same as a mount with type="bind".
		default:
			log.Warningf("ignoring unknown mount option %q", o)
		}
	}
	return mountOpts
}

func parseKeyValue(s string) (string, string, bool) {
	tokens := strings.SplitN(s, "=", 2)
	if len(tokens) < 2 {
		return "", "", false
	}
	return strings.TrimSpace(tokens[0]), strings.TrimSpace(tokens[1]), true
}

func createPrivateMemoryFile(file *os.File, resourceID checkpoint.ResourceID, cid string, fsr *fsRestore) (*pgalloc.MemoryFile, error) {
	pagesMetadataReader, pagesFileOffset, onLoadEnd, err := fsr.memoryFileLoadArgs(resourceID, cid)
	if err != nil {
		return nil, err
	}
	mfOpts := pgalloc.MemoryFileOpts{
		// Private memory files are usually backed by files on disk. Ideally we
		// would confirm with fstatfs(2) but that is prohibited by seccomp.
		DiskBackedFile: true,
		// Disk backed files need to be decommited on destroy to release disk space.
		DecommitOnDestroy: true,
		// sentry's seccomp filters don't allow the mmap(2) syscalls that
		// pgalloc.IMAWorkAroundForMemFile() uses. Users of private memory files
		// are expected to have performed the work around outside the sandbox.
		DisableIMAWorkAround: true,
		// Private memory files need to be restored correctly using this ID.
		ResourceID: resourceID,
	}
	mf, err := pgalloc.NewMemoryFile(file, mfOpts)
	if err != nil {
		onLoadEnd(err)
		return mf, err
	}
	if pagesMetadataReader != nil {
		log.Infof("Loading filesystem checkpoint data for %q", resourceID)
		if err := mf.LoadFrom(context.Background(), pagesMetadataReader, &pgalloc.LoadOpts{
			PagesFile:       fsr.apfl,
			PagesFileOffset: pagesFileOffset,
			DoneCallback:    onLoadEnd,
		}); err != nil {
			mf.Destroy()
			return nil, err
		}
	}
	return mf, nil
}

// mountTmp mounts an internal tmpfs at '/tmp' if it's safe to do so.
// Technically we don't have to mount tmpfs at /tmp, as we could just rely on
// the host /tmp, but this is a nice optimization, and fixes some apps that call
// mknod in /tmp. It's unsafe to mount tmpfs if:
//  1. /tmp is mounted explicitly: we should not override user's wish
//  2. /tmp is not empty: mounting tmpfs would hide existing files in /tmp
//
// Note that when there are submounts inside of '/tmp', directories for the
// mount points must be present, making '/tmp' not empty anymore.
func (c *containerMounter) mountTmp(ctx context.Context, spec *specs.Spec, conf *config.Config, creds *auth.Credentials, mns *vfs.MountNamespace) error {
	for _, m := range c.mounts {
		// m.Destination has been cleaned, so it's to use equality here.
		if m.Destination == "/tmp" {
			log.Debugf(`Explicit "/tmp" mount found, skipping internal tmpfs, mount: %+v`, m)
			return nil
		}
	}

	root := mns.Root(ctx)
	defer root.DecRef(ctx)
	pop := vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse("/tmp"),
	}
	fd, err := c.l.k.VFS().OpenAt(ctx, creds, &pop, &vfs.OpenOptions{Flags: linux.O_RDONLY | linux.O_DIRECTORY})
	switch {
	case err == nil:
		defer fd.DecRef(ctx)

		err := fd.IterDirents(ctx, vfs.IterDirentsCallbackFunc(func(dirent vfs.Dirent) error {
			if dirent.Name != "." && dirent.Name != ".." {
				return linuxerr.ENOTEMPTY
			}
			return nil
		}))
		switch {
		case err == nil:
			log.Infof(`Mounting internal tmpfs on top of empty "/tmp"`)
		case linuxerr.Equals(linuxerr.ENOTEMPTY, err):
			// If more than "." and ".." is found, skip internal tmpfs to prevent
			// hiding existing files.
			log.Infof(`Skipping internal tmpfs mount for "/tmp" because it's not empty`)
			return nil
		default:
			return fmt.Errorf("fd.IterDirents failed: %v", err)
		}
		fallthrough

	case linuxerr.Equals(linuxerr.ENOENT, err):
		// No '/tmp' found (or fallthrough from above). It's safe to mount internal
		// tmpfs.
		tmpMount := specs.Mount{
			Type:        tmpfs.Name,
			Destination: "/tmp",
			// Sticky bit is added to prevent accidental deletion of files from
			// another user. This is normally done for /tmp.
			Options: []string{"mode=01777"},
		}
		if _, err := c.mountSubmount(ctx, spec, conf, mns, creds, &mountInfo{mount: &tmpMount}); err != nil {
			return fmt.Errorf("mountSubmount failed: %v", err)
		}
		return nil

	case linuxerr.Equals(linuxerr.ENOTDIR, err):
		// Not a dir?! Let it be.
		return nil

	default:
		return fmt.Errorf(`opening "/tmp" inside container: %w`, err)
	}
}

func (c *containerMounter) getSharedMount(ctx context.Context, spec *specs.Spec, conf *config.Config, mount *mountInfo, creds *auth.Credentials) (*vfs.Mount, error) {
	sharedMount, ok := c.sharedMounts[mount.hint.Mount.Source]
	if ok {
		log.Infof("Using existing shared mount %q from %q type %q", mount.hint.Name, mount.hint.Mount.Source, mount.hint.Mount.Type)
		if mount.goferFD != nil {
			panic(fmt.Errorf("extra goferFD provided for shared mount %q", mount.hint.Name))
		}
		if mount.filestoreFD != nil {
			mount.filestoreFD.Close()
		}
		return sharedMount, nil
	}
	log.Infof("Mounting master of shared mount %q from %q type %q", mount.hint.Name, mount.hint.Mount.Source, mount.hint.Mount.Type)
	sharedMount, err := c.mountSharedMaster(ctx, spec, conf, mount, creds)
	if err != nil {
		return nil, fmt.Errorf("mounting shared master %q: %v", mount.hint.Name, err)
	}
	c.sharedMounts[mount.hint.Mount.Source] = sharedMount
	return sharedMount, nil
}

// setupCgroup2 initializes sandbox-wide cgroup2 state: an internal mount of
// the cgroup2fs singleton used to manage per-container cgroups, the
// system-wide nsdelegate flag, and controller enablement for per-container
// child cgroups.
//
// +checklocks:l.mu
func (l *Loader) setupCgroup2() error {
	cg2Mount, err := cgroup2fs.InitHierarchy(l.k.SupervisorContext(), l.k)
	if err != nil {
		return err
	}
	l.cgroup2Mount = cg2Mount
	return nil
}

func specHasCgroupMount(spec *specs.Spec) bool {
	for _, m := range spec.Mounts {
		if m.Type == cgroupfs.Name || m.Type == cgroup2fs.Name {
			return true
		}
	}
	return false
}

// setupContainerCgroup2 sets up the container's cgroup2 state: it creates
// the per-container cgroup "/<cid>", sets it as the container's initial
// cgroup, and resolves the container's cgroup namespace from its spec,
// mirroring PID namespace policy: no spec entry means the root namespace, a
// matching path joins that container's namespace, and anything else gets a
// new namespace rooted at "/<cid>".
//
// The returned cleanup releases the references that keep the container's
// cgroup and cgroup namespace alive until CreateProcess takes its own;
// call it (typically via defer) once the container's process has been
// created, or once creation has failed.
//
// +checklocks:l.mu
func (l *Loader) setupContainerCgroup2(info *containerInfo) (func(), error) {
	ctx := l.k.SupervisorContext()
	creds := auth.NewRootCredentials(l.k.RootUserNamespace())
	var (
		cg kernel.Cgroup2
		vd vfs.VirtualDentry
	)

	if specHasCgroupMount(info.spec) {
		root := vfs.MakeVirtualDentry(l.cgroup2Mount, l.cgroup2Mount.Root())
		pop := vfs.PathOperation{
			Root:  root,
			Start: root,
			Path:  fspath.Parse(info.cid),
		}
		// EEXIST is tolerated to allow container ID reuse after a destroy that
		// could not remove the cgroup.
		if err := l.k.VFS().MkdirAt(ctx, creds, &pop, &vfs.MkdirOptions{Mode: 0755}); err != nil &&
			!linuxerr.Equals(linuxerr.EEXIST, err) {
			return nil, fmt.Errorf("creating container cgroup %q: %w", info.cid, err)
		}
		var err error
		vd, err = l.k.VFS().GetDentryAt(ctx, creds, &pop, &vfs.GetDentryOptions{})
		if err != nil {
			return nil, fmt.Errorf("resolving container cgroup %q: %w", info.cid, err)
		}
		var ok bool
		cg, ok = cgroup2fs.CgroupFromDentry(vd.Dentry())
		if !ok {
			vd.DecRef(ctx)
			return nil, fmt.Errorf("container cgroup %q is not a cgroup", info.cid)
		}
		info.procArgs.InitialCgroupV2 = cg
	} else {
		// No cgroup mount in the spec, no per-container cgroup.
		cg = l.k.Cgroup2FS().RootCgroup()
	}

	ns, ok := specutils.GetNS(specs.CgroupNamespace, info.spec)
	if !ok {
		// No namespace entry in the spec: stay in the root namespace.
		return func() {
			if vd.Ok() {
				vd.DecRef(ctx)
			}
		}, nil
	}

	var cgroupns *kernel.CgroupNamespace
	if ns.Path != "" {
		cgroupns = l.findCgroupNamespace(ns.Path)
	}
	if cgroupns != nil {
		log.Debugf("Joining cgroup namespace named %q", ns.Path)
	} else {
		if ns.Path != "" {
			log.Warningf("Cgroup namespace %q not found, running in new cgroup namespace", ns.Path)
		}
		cgroupns = l.k.NewCgroupNamespace(ctx, cg, info.procArgs.Credentials.UserNamespace)
	}

	info.procArgs.CgroupNamespace = cgroupns
	if ep := l.processes[execID{cid: info.cid}]; ep != nil {
		ep.cgroupnsPath = ns.Path
	}

	return func() {
		if vd.Ok() {
			vd.DecRef(ctx)
		}
		cgroupns.DecRef(ctx)
	}, nil
}

// findCgroupNamespace returns the cgroup namespace of the container whose
// spec declared the given cgroup namespace path, or nil if no such container
// exists. A non-nil return holds a reference for the caller.
//
// +checklocks:l.mu
func (l *Loader) findCgroupNamespace(path string) *kernel.CgroupNamespace {
	for _, p := range l.processes {
		if p.cgroupnsPath != path || p.tg == nil {
			continue
		}
		if nsref := p.tg.Leader().GetCgroupNamespace(); nsref != nil {
			return nsref
		}
	}
	return nil
}

// removeContainerCgroup2 makes a best-effort attempt at removing the
// destroyed container's cgroup, including any sub-cgroups the container
// created. It may legitimately fail, e.g. if another container migrated a
// task into the subtree.
//
// +checklocks:l.mu
func (l *Loader) removeContainerCgroup2(cid string) {
	if l.cgroup2Mount == nil {
		return
	}
	ctx := l.k.SupervisorContext()
	creds := auth.NewRootCredentials(l.k.RootUserNamespace())
	root := vfs.MakeVirtualDentry(l.cgroup2Mount, l.cgroup2Mount.Root())
	if err := l.removeCgroup2Subtree(ctx, creds, root, cid); err != nil && !linuxerr.Equals(linuxerr.ENOENT, err) {
		log.Warningf("Failed to remove cgroup of container %q: %v", cid, err)
	}
}

// removeCgroup2Subtree removes the cgroup directory at path along with its
// descendant cgroups, depth first, mirroring runc's recursive cgroup removal
// on container deletion. It is best-effort: a descendant that cannot be
// removed (e.g. one populated by a task that another container migrated into
// the subtree) leaves its ancestors in place.
func (l *Loader) removeCgroup2Subtree(ctx context.Context, creds *auth.Credentials, root vfs.VirtualDentry, path string) error {
	pop := vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(path),
	}
	if fd, err := l.k.VFS().OpenAt(ctx, creds, &pop, &vfs.OpenOptions{Flags: linux.O_RDONLY | linux.O_DIRECTORY}); err == nil {
		var children []string
		_ = fd.IterDirents(ctx, vfs.IterDirentsCallbackFunc(func(d vfs.Dirent) error {
			if d.Type == linux.DT_DIR && d.Name != "." && d.Name != ".." {
				children = append(children, d.Name)
			}
			return nil
		}))
		fd.DecRef(ctx)
		for _, child := range children {
			_ = l.removeCgroup2Subtree(ctx, creds, root, path+"/"+child)
		}
	}
	return l.k.VFS().RmdirAt(ctx, creds, &pop)
}

// mountCgroupMounts mounts the cgroups which are shared across all containers.
// Postcondition: Initialized k.cgroupMounts on success.
func (l *Loader) mountCgroupMounts(conf *config.Config, creds *auth.Credentials) error {
	ctx := l.k.SupervisorContext()
	for _, sopts := range kernel.CgroupCtrls {
		var internalData any
		switch string(sopts) {
		case "cpu":
			if defaults := cgroupfsCPUDefaults(l.cpuQuota, l.cpuPeriod); defaults != nil {
				internalData = &cgroupfs.InternalData{DefaultControlValues: defaults}
				log.Infof("Setting cgroupfs cpu defaults: quota=%d, period=%d (raw quota=%d, raw period=%d)", defaults["cpu.cfs_quota_us"], defaults["cpu.cfs_period_us"], l.cpuQuota, l.cpuPeriod)
			}
		case "memory":
			// Set memory limit from --total-memory if specified.
			// This allows applications to see the correct memory limit in cgroup.
			if defaults := cgroupfsMemoryDefaults(usage.MaximumTotalMemoryBytes); defaults != nil {
				internalData = &cgroupfs.InternalData{DefaultControlValues: defaults}
				log.Infof("Setting cgroupfs memory defaults: limit=%d bytes (%.2f GB)", usage.MaximumTotalMemoryBytes, float64(usage.MaximumTotalMemoryBytes)/(1<<30))
			}
		}

		mopts := &vfs.MountOptions{
			GetFilesystemOptions: vfs.GetFilesystemOptions{
				Data:          string(sopts),
				InternalMount: true,
				InternalData:  internalData,
			},
		}
		fs, root, err := l.k.VFS().NewFilesystem(ctx, creds, "cgroup", cgroupfs.Name, mopts)
		if err != nil {
			return err
		}

		mount := l.k.VFS().NewDisconnectedMount(fs, root, mopts)
		// Private so that mounts created by containers do not appear
		// in other container's cgroup paths.
		l.k.VFS().SetMountPropagation(mount, linux.MS_PRIVATE, false)
		l.k.AddCgroupMount(string(sopts), &kernel.CgroupMount{
			Fs:    fs,
			Root:  root,
			Mount: mount,
		})
	}
	log.Infof("created cgroup mounts for controllers %v", kernel.CgroupCtrls)
	return nil
}

// mountCgroupSubmounts mounts all the cgroup controller submounts for the
// container. The cgroup submounts are created under the root controller mount
// with containerID as the directory name and then bind mounts this directory
// inside the container's mount namespace.
func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) error {
	root := mns.Root(ctx)
	defer root.DecRef(ctx)

	// Mount "/sys/fs/cgroup" in the container's mount namespace.
	submount.mount.Type = tmpfs.Name
	mnt, err := c.mountSubmount(ctx, spec, conf, mns, creds, submount)
	if err != nil {
		return err
	}
	if mnt != nil && mnt.ReadOnly() {
		// Switch to ReadWrite while we setup submounts.
		if err := c.l.k.VFS().SetMountReadOnly(mnt, false); err != nil {
			return fmt.Errorf("failed to set mount at %q readwrite: %w", submount.mount.Destination, err)
		}
		// Restore back to ReadOnly at the end.
		defer func() {
			if err := c.l.k.VFS().SetMountReadOnly(mnt, true); err != nil {
				panic(fmt.Sprintf("failed to restore mount at %q back to readonly: %v", submount.mount.Destination, err))
			}
		}()
	}

	// Mount all the cgroup controllers in the container's mount namespace.
	mountCtx := vfs.WithRoot(vfs.WithMountNamespace(ctx, mns), root)
	for _, ctrl := range kernel.CgroupCtrls {
		ctrlName := string(ctrl)
		cgroupMnt := c.l.k.GetCgroupMount(ctrlName)
		if cgroupMnt == nil {
			return fmt.Errorf("cgroup mount for controller %s not found", ctrlName)
		}

		cgroupMntVD := vfs.MakeVirtualDentry(cgroupMnt.Mount, cgroupMnt.Root)
		sourcePop := vfs.PathOperation{
			Root:  cgroupMntVD,
			Start: cgroupMntVD,
			// Use the containerID as the cgroup path.
			Path: fspath.Parse(c.containerID),
		}
		if err := c.l.k.VFS().MkdirAt(mountCtx, creds, &sourcePop, &vfs.MkdirOptions{
			Mode: 0755,
		}); err != nil {
			log.Infof("error in creating directory %v", err)
			return err
		}

		// Bind mount the new cgroup directory into the container's mount namespace.
		destination := "/sys/fs/cgroup/" + ctrlName
		if err := c.l.k.VFS().MakeSyntheticMountpoint(mountCtx, destination, root, creds); err != nil {
			// Log a warning, but attempt the mount anyway.
			log.Warningf("Failed to create mount point %q: %v", destination, err)
		}

		target := &vfs.PathOperation{
			Root:  root,
			Start: root,
			Path:  fspath.Parse(destination),
		}
		if err := c.l.k.VFS().BindAt(mountCtx, creds, &sourcePop, target, false); err != nil {
			log.Infof("error in bind mounting %v", err)
			return err
		}
	}
	c.cgroupsMounted = true
	return nil
}

// mountSharedMaster mounts the master of a volume that is shared among
// containers in a pod.
func (c *containerMounter) mountSharedMaster(ctx context.Context, spec *specs.Spec, conf *config.Config, mntInfo *mountInfo, creds *auth.Credentials) (*vfs.Mount, error) {
	// Mount the master using the options from the hint (mount annotations).
	origOpts := mntInfo.mount.Options
	mntInfo.mount.Options = mntInfo.hint.Mount.Options
	fsName, opts, err := getMountNameAndOptions(spec, conf, mntInfo, c.l.productName, c.containerName, c.containerID, c.l.fsRestore, c.l.rdmaSysfs)
	mntInfo.mount.Options = origOpts
	if err != nil {
		return nil, err
	}
	if len(fsName) == 0 {
		return nil, fmt.Errorf("mount type not supported %q", mntInfo.hint.Mount.Type)
	}
	return c.l.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
}

// mountSharedSubmount binds mount to a previously mounted volume that is shared
// among containers in the same pod.
func (c *containerMounter) mountSharedSubmount(ctx context.Context, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, mntInfo *mountInfo, sharedMount *vfs.Mount) (*vfs.Mount, error) {
	if err := mntInfo.hint.checkCompatible(mntInfo.mount); err != nil {
		return nil, err
	}

	// Generate mount point specific opts using mntInfo.mount.
	opts := ParseMountOptions(mntInfo.mount.Options)
	newMnt := c.l.k.VFS().NewDisconnectedMount(sharedMount.Filesystem(), sharedMount.Root(), opts)
	defer newMnt.DecRef(ctx)

	root := mns.Root(ctx)
	defer root.DecRef(ctx)
	target := &vfs.PathOperation{
		Root:  root,
		Start: root,
		Path:  fspath.Parse(mntInfo.mount.Destination),
	}

	vd := vfs.MakeVirtualDentry(newMnt, newMnt.Root())
	rootPath := &vfs.PathOperation{
		Root:  vd,
		Start: vd,
	}
	rootMode, err := c.getPathMode(ctx, creds, rootPath)
	if err != nil {
		return nil, err
	}

	if err := c.makeMountPoint(ctx, creds, mns, mntInfo.mount.Destination, rootMode); err != nil {
		return nil, fmt.Errorf("creating mount point %q: %w", mntInfo.mount.Destination, err)
	}

	// Avoid mounting on top of symlinks. The mount syscall on Linux always follows symlinks.
	target.FollowFinalSymlink = true
	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, newMnt, target); err != nil {
		return nil, err
	}
	log.Infof("Mounted %q type shared bind to %q", mntInfo.mount.Destination, mntInfo.hint.Name)
	return newMnt, nil
}

// makeMountPoint creates the mount point destination, ensuring its type (file
// or directory) matches `rootMode`. If the destination already exists, it just
// verifies that its type is consistent with `rootMode`; otherwise, the ENOTDIR
// error is returned.
func (c *containerMounter) makeMountPoint(
	ctx context.Context,
	creds *auth.Credentials,
	mns *vfs.MountNamespace,
	dest string,
	rootMode linux.FileMode,
) error {
	root := mns.Root(ctx)
	defer root.DecRef(ctx)

	target := &vfs.PathOperation{
		Root:               root,
		Start:              root,
		Path:               fspath.Parse(dest),
		FollowFinalSymlink: true,
	}

	fs := c.l.k.VFS()

	mode, err := c.getPathMode(ctx, creds, target)
	// First check if mount point exists.
	switch {
	case err == nil:
		if mode.IsDir() != rootMode.IsDir() {
			if rootMode.IsDir() {
				return fmt.Errorf("mountpoint %q isn't a directory, got mode %s", dest, mode)
			} else {
				return fmt.Errorf("mountpoint %q isn't not a file, got mode %s", dest, mode)
			}
		}
		// Target already exists.
		return nil
	case linuxerr.Equals(linuxerr.ENOENT, err):
		// Expected, we will create the mount point.
	default:
		return fmt.Errorf("stat failed for %q during mountpoint creation: %w", dest, err)
	}

	// FollowFinalSymlink should be false to create new file or directory.
	target.FollowFinalSymlink = false
	mkdirOpts := &vfs.MkdirOptions{Mode: 0755, ForSyntheticMountpoint: true}

	// Make sure the parent directory of target exists.
	if err := fs.MkdirAllAt(ctx, path.Dir(dest), root, creds, mkdirOpts, true /* mustBeDir */); err != nil {
		return fmt.Errorf("failed to create parent directory of mountpoint %q: %w", dest, err)
	}

	if rootMode.IsDir() {
		if err := fs.MkdirAt(ctx, creds, target, mkdirOpts); err != nil {
			return fmt.Errorf("failed to create directory mountpoint %q: %w", dest, err)
		}
	} else {
		mknodOpts := &vfs.MknodOptions{
			Mode: linux.FileMode(linux.S_IFREG | 0644),
		}
		if err := fs.MknodAt(ctx, creds, target, mknodOpts); err != nil {
			return fmt.Errorf("failed to create file mountpoint %q: %w", dest, err)
		}
	}
	return nil
}

// configureRestore returns an updated context.Context including filesystem
// state used by restore defined by conf.
func (c *containerMounter) configureRestore(restoreMnts *restoreMounts) error {
	// Compare createMountNamespace(); rootfs always consumes a gofer FD and a
	// filestore FD is consumed if the rootfs GoferMountConf indicates so.
	rootKey := checkpoint.ResourceID{ContainerName: c.containerName, Path: "/"}
	restoreMnts.fdmap[rootKey] = c.goferFDs.remove()

	if rootfsConf := c.goferMountConfs[0]; rootfsConf.IsFilestorePresent() {
		mf, err := createPrivateMemoryFile(c.goferFilestoreFDs.removeAsFD().ReleaseToFile("overlay-filestore"), rootKey, c.containerID, c.l.fsRestore)
		if err != nil {
			return fmt.Errorf("failed to create private memory file for mount rootfs: %w", err)
		}
		restoreMnts.mfmap[rootKey] = mf
	}
	// prepareMounts() consumes the remaining FDs for submounts.
	mounts, err := c.prepareMounts()
	if err != nil {
		return err
	}
	for i := range mounts {
		submount := &mounts[i]

		if submount.hint != nil && submount.hint.ShouldShareMount() {
			if restoreMnts.sharedMfs[submount.hint.Name] {
				// This shared mount has already been restored by another container.
				if submount.goferFD != nil {
					submount.goferFD.Close()
					submount.goferFD = nil
				}
				if submount.filestoreFD != nil {
					submount.filestoreFD.Close()
					submount.filestoreFD = nil
				}
				continue
			}
			restoreMnts.sharedMfs[submount.hint.Name] = true
		}

		if submount.goferFD != nil {
			key := checkpoint.ResourceID{ContainerName: c.containerName, Path: submount.mount.Destination}
			restoreMnts.fdmap[key] = submount.goferFD.Release()
		}
		if submount.filestoreFD != nil {
			key := checkpoint.ResourceID{ContainerName: c.containerName, Path: submount.mount.Destination}
			mf, err := createPrivateMemoryFile(submount.filestoreFD.ReleaseToFile("overlay-filestore"), key, c.containerID, c.l.fsRestore)
			if err != nil {
				return fmt.Errorf("failed to create private memory file for mount %q: %w", submount.mount.Destination, err)
			}
			restoreMnts.mfmap[key] = mf
		}
	}
	return nil
}

func createDeviceFiles(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry) error {
	if info.spec.Linux != nil {
		// Register vfio control device early if TPUv5 is enabled, so that
		// subsequent TPU devices can find the tpuproxy object.
		tpuv5Enabled := false
		for _, dev := range info.spec.Linux.Devices {
			if tpuproxy.TPUv5DeviceRegex.MatchString(dev.Path) {
				tpuv5Enabled = true
				break
			}
		}
		if tpuv5Enabled {
			if err := vfio.Register(vfsObj, true /* useDevGofer */); err != nil {
				return fmt.Errorf("registering vfio driver: %w", err)
			}
		}

		// Create any device files specified in the spec.
		for _, dev := range info.spec.Linux.Devices {
			if err := createDeviceFile(ctx, creds, info, vfsObj, root, dev); err != nil {
				return err
			}
		}
	}
	if specutils.GPUFunctionalityRequestedViaHook(info.spec, info.conf) {
		// When using nvidia-container-runtime-hook, devices are not injected into
		// spec.Linux.Devices. So manually create appropriate device files.
		nvidiaDevs := []specs.LinuxDevice{
			{Path: "/dev/nvidiactl", Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: nvgpu.NV_MINOR_DEVICE_NUMBER_CONTROL_DEVICE},
			{Path: "/dev/nvidia-uvm", Type: "c", Major: int64(info.nvproxyDevInfo.UVMDevMajor), Minor: nvgpu.NVIDIA_UVM_PRIMARY_MINOR_NUMBER},
		}
		// There is no nvidia-container-cli flag to enable fabric-imex-mgmt, so
		// we never create a /dev/nvidia-caps/nvidia-cap# file for it here.
		devClient := devutil.GoferClientFromContext(ctx)
		if devClient == nil {
			return fmt.Errorf("dev gofer client not found in context")
		}
		names, err := devClient.DirentNames(ctx)
		if err != nil {
			return fmt.Errorf("failed to get names of dirents from dev gofer: %w", err)
		}
		nvidiaDeviceRegex := regexp.MustCompile(`^nvidia(\d+)$`)
		for _, name := range names {
			ms := nvidiaDeviceRegex.FindStringSubmatch(name)
			if ms == nil {
				continue
			}
			minor, err := strconv.ParseUint(ms[1], 10, 32)
			if err != nil {
				return fmt.Errorf("invalid nvidia device name %q: %w", name, err)
			}
			if minor > nvgpu.NV_MINOR_DEVICE_NUMBER_REGULAR_MAX {
				return fmt.Errorf("invalid nvidia regular minor device number %d", minor)
			}
			nvidiaDevs = append(nvidiaDevs, specs.LinuxDevice{Path: fmt.Sprintf("/dev/nvidia%d", minor), Type: "c", Major: nvgpu.NV_MAJOR_DEVICE_NUMBER, Minor: int64(minor)})
		}
		for _, nvidiaDev := range nvidiaDevs {
			if err := createDeviceFile(ctx, creds, info, vfsObj, root, nvidiaDev); err != nil {
				return err
			}
		}
	}
	return nil
}

func createDeviceFile(ctx context.Context, creds *auth.Credentials, info *containerInfo, vfsObj *vfs.VirtualFilesystem, root vfs.VirtualDentry, devSpec specs.LinuxDevice) error {
	mode := linux.FileMode(0666)
	if devSpec.FileMode != nil {
		mode = linux.FileMode(devSpec.FileMode.Perm())
	}
	var major, minor uint32
	// See https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#devices.
	switch devSpec.Type {
	case "b":
		mode |= linux.S_IFBLK
		major = uint32(devSpec.Major)
		minor = uint32(devSpec.Minor)
	case "c", "u":
		mode |= linux.S_IFCHR
		major = uint32(devSpec.Major)
		minor = uint32(devSpec.Minor)
	case "p":
		mode |= linux.S_IFIFO
	default:
		return fmt.Errorf("specified device at %q has invalid type %q", devSpec.Path, devSpec.Type)
	}
	// Convert host-assigned device major numbers to sentry-assigned ones.
	if strings.HasPrefix(devSpec.Path, "/dev/vfio") || strings.HasPrefix(devSpec.Path, "/dev/accel") {
		if devSpec.Path == "/dev/vfio/vfio" {
			if err := vfio.Register(vfsObj, true /* useDevGofer */); err != nil {
				return fmt.Errorf("registering vfio driver: %w", err)
			}
		} else if tpuproxy.TPUv4DeviceRegex.MatchString(devSpec.Path) {

			if err := tpuproxy.RegisterTPUv4Device(ctx, creds, root, vfsObj, devSpec.Path, minor); err != nil {
				return fmt.Errorf("registering TPUv4 device: %w", err)
			}
		} else if tpuproxy.TPUv5DeviceRegex.MatchString(devSpec.Path) {
			if err := tpuproxy.RegisterTPUv5Device(vfsObj, devSpec.Path, minor); err != nil {
				return fmt.Errorf("registering TPUv5 device: %w", err)
			}
			var err error
			major, err = vfio.GetTPUDeviceMajor(vfsObj)
			if err != nil {
				return fmt.Errorf("getting TPU device major number: %w", err)
			}
			log.Infof("Switching %v device major number from %d to %d", devSpec.Path, devSpec.Major, major)
		}
	} else if devSpec.Path == "/dev/nvidia-uvm" {
		if info.nvproxyDevInfo.UVMDevMajor != 0 && major != info.nvproxyDevInfo.UVMDevMajor {
			major = info.nvproxyDevInfo.UVMDevMajor
			log.Infof("Switching /dev/nvidia-uvm device major number from %d to %d", devSpec.Major, major)
		}
	} else if strings.HasPrefix(devSpec.Path, "/dev/nvidia-caps/") {
		if info.nvproxyDevInfo.CapsDevMajor != 0 && major != info.nvproxyDevInfo.CapsDevMajor {
			major = info.nvproxyDevInfo.CapsDevMajor
			log.Infof("Switching %s device major number from %d to %d", devSpec.Path, devSpec.Major, major)
		}
	} else if strings.HasPrefix(devSpec.Path, "/dev/nvidia-caps-imex-channels/") {
		if info.nvproxyDevInfo.CapsIMEXChannelsDevMajor != 0 && major != info.nvproxyDevInfo.CapsIMEXChannelsDevMajor {
			major = info.nvproxyDevInfo.CapsIMEXChannelsDevMajor
			log.Infof("Switching %s device major number from %d to %d", devSpec.Path, devSpec.Major, major)
		}
	}
	return dev.CreateDeviceFile(ctx, vfsObj, creds, root, devSpec.Path, major, minor, mode, devSpec.UID, devSpec.GID)
}

func nvproxyRegisterDevices(info *containerInfo, vfsObj *vfs.VirtualFilesystem, nvidiaDriverVersion nvconf.DriverVersion) error {
	if !specutils.NVProxyEnabled(info.spec, info.conf) {
		return nil
	}
	driverCaps, err := specutils.NVProxyDriverCapsAllowed(info.conf)
	if err != nil {
		return fmt.Errorf("NVIDIA driver capabilities: %w", err)
	}
	devInfo, err := nvproxy.Register(vfsObj, &nvproxy.Options{
		DriverVersion:          nvidiaDriverVersion,
		DriverCaps:             driverCaps,
		AllowUnsupportedDriver: info.conf.NVProxyAllowUnsupportedDriver,
		HostSettings:           info.nvidiaHostSettings,
		UseDevGofer:            true,
	})
	if err != nil {
		return fmt.Errorf("registering nvproxy driver: %w", err)
	}
	info.nvproxyDevInfo = devInfo
	return nil
}