fsimpl/erofs: read regular file data from the image FD on FUSE

Regular file reads copy data out of the image's shared mapping. When the
image is backed by slow storage such as a FUSE mount, walking the mapping
faults it in one page at a time, turning a large read into many small,
largely serialized requests to the FUSE server. This makes erofs much
slower than fsimpl/gofer for large reads on such backends.

On those filesystems, read the data directly from the image file
descriptor with preadv2(2) instead, so the host serves a large read with a
single syscall and its own readahead, matching how fsimpl/gofer reads from
host FDs. The mapping copy is kept everywhere else, since it avoids a
syscall and is faster when the page cache is warm. The backing filesystem
is detected at mount time via fstatfs(2); currently only FUSE opts into FD
reads.

Inode.Data, which returned the data as a mapping-backed BlockSeq, is
replaced by Inode.DataRanges, which returns the image byte ranges that
hold the inode's data.

pkg/erofs/BUILD

@@ -21,7 +21,6 @@ go_library(
         "//pkg/hostarch",
         "//pkg/log",
         "//pkg/marshal",
-        "//pkg/safemem",
         "@org_golang_x_sys//unix:go_default_library",
     ],
 )

pkg/erofs/erofs.go

@@ -29,6 +29,7 @@ import (
 	"os"
 
 	"golang.org/x/sys/unix"
+
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/errors/linuxerr"
@@ -36,7 +37,6 @@ import (
 	"gvisor.dev/gvisor/pkg/hostarch"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/marshal"
-	"gvisor.dev/gvisor/pkg/safemem"
 )
 
 const (
@@ -637,37 +637,46 @@ func (i *Inode) DataOffset() (uint64, error) {
 	return i.dataOff, nil
 }
 
-// Data returns the read-only file data of this inode.
-func (i *Inode) Data() (safemem.BlockSeq, error) {
+// FileRange represents a range of bytes within the image file.
+type FileRange struct {
+	// Off is the offset of the range within the image file.
+	Off uint64
+	// Size is the length of the range in bytes.
+	Size uint64
+	// Bytes is the byte slice of the range within the image file.
+	Bytes []byte
+}
+
+// DataRanges returns the ranges within the image file that hold this inode's
+// data, in file order. An inode's data occupies at most two ranges: a run of
+// plain data blocks and, for files with tail-packed inline data, the inline
+// tail. Entries that are not used (or hold no data) have a zero Size.
+func (i *Inode) DataRanges() ([2]FileRange, error) {
+	var ranges [2]FileRange
 	switch dataLayout := i.DataLayout(); dataLayout {
 	case InodeDataLayoutFlatPlain:
-		bytes, err := i.image.BytesAt(i.dataOff, i.size)
-		if err != nil {
-			return safemem.BlockSeq{}, err
-		}
-		return safemem.BlockSeqOf(safemem.BlockFromSafeSlice(bytes)), nil
+		ranges[0] = FileRange{Off: i.dataOff, Size: i.size}
 
 	case InodeDataLayoutFlatInline:
-		sl := make([]safemem.Block, 0, 2)
 		idataSize := i.size & (uint64(i.image.BlockSize()) - 1)
-		if i.size > idataSize {
-			if bytes, err := i.image.BytesAt(i.dataOff, i.size-idataSize); err != nil {
-				return safemem.BlockSeq{}, err
-			} else {
-				sl = append(sl, safemem.BlockFromSafeSlice(bytes))
-			}
-		}
-		if bytes, err := i.image.BytesAt(i.idataOff, idataSize); err != nil {
-			return safemem.BlockSeq{}, err
-		} else {
-			sl = append(sl, safemem.BlockFromSafeSlice(bytes))
-		}
-		return safemem.BlockSeqFromSlice(sl), nil
+		ranges[0] = FileRange{Off: i.dataOff, Size: i.size - idataSize}
+		ranges[1] = FileRange{Off: i.idataOff, Size: idataSize}
 
 	default:
 		log.Warningf("Unsupported data layout 0x%x at inode (nid=%v)", dataLayout, i.Nid())
-		return safemem.BlockSeq{}, linuxerr.ENOTSUP
+		return [2]FileRange{}, linuxerr.ENOTSUP
 	}
+	for j := range ranges {
+		if ranges[j].Size == 0 {
+			continue
+		}
+		bytes, err := i.image.BytesAt(ranges[j].Off, ranges[j].Size)
+		if err != nil {
+			return [2]FileRange{}, err
+		}
+		ranges[j].Bytes = bytes
+	}
+	return ranges, nil
 }
 
 // blockData represents the information of the data in a block.

pkg/sentry/fsimpl/erofs/BUILD

@@ -68,11 +68,13 @@ go_library(
         "//pkg/sentry/checkpoint",
         "//pkg/sentry/fsimpl/lock",
         "//pkg/sentry/fsutil",
+        "//pkg/sentry/hostfd",
         "//pkg/sentry/kernel/auth",
         "//pkg/sentry/memmap",
         "//pkg/sentry/socket/unix/transport",
         "//pkg/sentry/vfs",
         "//pkg/sync",
         "//pkg/usermem",
+        "@org_golang_x_sys//unix:go_default_library",
     ],
 )

pkg/sentry/fsimpl/erofs/erofs.go

@@ -21,6 +21,8 @@ import (
 	"strconv"
 	"sync/atomic"
 
+	"golang.org/x/sys/unix"
+
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/context"
@@ -69,6 +71,11 @@ type filesystem struct {
 	// mf implements memmap.File for this image.
 	mf imageMemmapFile
 
+	// useReadForIO indicates that file I/O should be driven by read syscalls
+	// from the image file descriptor instead of the image mapping.
+	// useReadForIO is immutable.
+	useReadForIO bool
+
 	// inodeBuckets contains the inodes in use. Multiple buckets are used to
 	// reduce the lock contention. Bucket is chosen based on the hash calculation
 	// on nid in filesystem.inodeBucket.
@@ -122,17 +129,23 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		return nil, nil, linuxerr.EINVAL
 	}
 
+	useReadForIO, err := shouldUseReadForIO(ctx, fd)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	devMinor, err := vfsObj.GetAnonBlockDevMinor()
 	if err != nil {
 		return nil, nil, err
 	}
 
 	fs := &filesystem{
-		mopts:    opts.Data,
-		iopts:    iopts,
-		image:    image,
-		devMinor: devMinor,
-		mf:       imageMemmapFile{image: image},
+		mopts:        opts.Data,
+		iopts:        iopts,
+		image:        image,
+		devMinor:     devMinor,
+		useReadForIO: useReadForIO,
+		mf:           imageMemmapFile{image: image},
 	}
 	fs.vfsfs.Init(vfsObj, &fstype, fs)
 	cu.Add(func() { fs.vfsfs.DecRef(ctx) })
@@ -173,6 +186,19 @@ func getFDFromMountOptionsMap(ctx context.Context, mopts map[string]string) (int
 	return ifd, nil
 }
 
+func shouldUseReadForIO(ctx context.Context, imageFD int) (bool, error) {
+	var st unix.Statfs_t
+	if err := unix.Fstatfs(imageFD, &st); err != nil {
+		// Detection is best-effort; fall back to reading via the mapping.
+		ctx.Warningf("erofs.shouldUseReadForIO: fstatfs failed, defaulting to using mmap for reads: %v", err)
+		return false, nil
+	}
+	// Use read syscalls when the EROFS image file lives on FUSE filesystem.
+	// This avoids faulting the mapping in page by page and turning a large
+	// read into many small, largely serialized requests to the FUSE server.
+	return st.Type == linux.FUSE_SUPER_MAGIC, nil
+}
+
 // Release implements vfs.FilesystemImpl.Release.
 func (fs *filesystem) Release(ctx context.Context) {
 	// An extra reference was held by the filesystem on the root.

pkg/sentry/fsimpl/erofs/regular_file.go

@@ -25,6 +25,7 @@ import (
 	"gvisor.dev/gvisor/pkg/hostarch"
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/safemem"
+	"gvisor.dev/gvisor/pkg/sentry/hostfd"
 	"gvisor.dev/gvisor/pkg/sentry/memmap"
 	"gvisor.dev/gvisor/pkg/sentry/vfs"
 	"gvisor.dev/gvisor/pkg/usermem"
@@ -59,30 +60,53 @@ func (fd *regularFileFD) PRead(ctx context.Context, dst usermem.IOSequence, offs
 		return 0, nil
 	}
 
-	data, err := fd.inode().Data()
+	inode := fd.inode()
+	ranges, err := inode.DataRanges()
 	if err != nil {
 		return 0, err
 	}
 	r := &regularFileReader{
-		data: data,
-		off:  uint64(offset),
+		image:   inode.fs.image,
+		ranges:  ranges,
+		off:     uint64(offset),
+		useRead: inode.fs.useReadForIO,
 	}
 	return dst.CopyOutFrom(ctx, r)
 }
 
 type regularFileReader struct {
-	data safemem.BlockSeq
-	off  uint64
+	image   *erofs.Image
+	ranges  [2]erofs.FileRange
+	off     uint64
+	useRead bool
 }
 
 // ReadToBlocks implements safemem.Reader.ReadToBlocks.
 func (r *regularFileReader) ReadToBlocks(dsts safemem.BlockSeq) (uint64, error) {
-	if r.off >= r.data.NumBytes() {
-		return 0, io.EOF
+	// Find the range containing the current offset. A read must not span the
+	// discontinuity between ranges in the image, so it is clamped to the end
+	// of the containing range.
+	off := r.off
+	for _, rng := range r.ranges {
+		if off < rng.Size {
+			dsts = dsts.TakeFirst64(rng.Size - off)
+			imageOff := rng.Off + off
+			var (
+				n   uint64
+				err error
+			)
+			if r.useRead {
+				n, err = hostfd.Preadv2(int32(r.image.FD()), dsts, int64(imageOff), 0 /* flags */)
+			} else {
+				src := safemem.BlockSeqOf(safemem.BlockFromSafeSlice(rng.Bytes)).DropFirst64(off)
+				n, err = safemem.CopySeq(dsts, src)
+			}
+			r.off += n
+			return n, err
+		}
+		off -= rng.Size
 	}
-	cp, err := safemem.CopySeq(dsts, r.data.DropFirst(int(r.off)))
-	r.off += cp
-	return cp, err
+	return 0, io.EOF
 }
 
 // Read implements vfs.FileDescriptionImpl.Read.

runsc/boot/filter/config/config_main.go

@@ -61,7 +61,11 @@ var allowedSyscalls = seccomp.MakeSyscallRules(map[uintptr]seccomp.SyscallRule{
 			seccomp.EqualTo(unix.F_GETFD),
 		},
 	},
-	unix.SYS_FSTAT:     seccomp.MatchAll{},
+	unix.SYS_FSTAT: seccomp.MatchAll{},
+	unix.SYS_FSTATFS: seccomp.PerArg{
+		seccomp.NonNegativeFD{},
+		seccomp.AnyValue{},
+	},
 	unix.SYS_FSYNC:     seccomp.MatchAll{},
 	unix.SYS_FDATASYNC: seccomp.MatchAll{},
 	unix.SYS_FTRUNCATE: seccomp.MatchAll{},
@@ -430,10 +434,6 @@ func hostFilesystemFilters() seccomp.SyscallRules {
 			seccomp.NonNegativeFD{},
 			seccomp.AnyValue{},
 		},
-		unix.SYS_FSTATFS: seccomp.PerArg{
-			seccomp.NonNegativeFD{},
-			seccomp.AnyValue{},
-		},
 		unix.SYS_READLINKAT: seccomp.PerArg{
 			seccomp.NonNegativeFD{},
 			seccomp.AnyValue{},