Implement filesystem-only checkpointing.

- `runsc fscheckpoint` saves a filesystem checkpoint, analogous to `runsc
    checkpoint`. `-direct` and `-leave-running` flags are supported for
    compatibility with `runsc checkpoint`. `-save-restore-exec*` is currently
    not supported due to lack of a use case. `-compression` support is not
    planned (specific files in filesystem checkpoint images may use compression
    as appropriate, but this is an implementation detail).
    `-exclude-committed-zero-pages` is inapplicable.

  - `runsc create -fs-restore-image-path` restores a filesystem checkpoint. The
    `-fs-restore-direct` flag has the same effect as `runsc restore -direct`.
    The behavior of `runsc restore -background` is always in effect for
    filesystem checkpoints.

PiperOrigin-RevId: 885914969

g3doc/user_guide/BUILD

@@ -128,3 +128,11 @@ doc(
     permalink = "/docs/user_guide/rootfs_snapshot/",
     weight = "92",
 )
+
+doc(
+    name = "fs_snapshot",
+    src = "fs_snapshot.md",
+    category = "User Guide",
+    permalink = "/docs/user_guide/fs_snapshot/",
+    weight = "93",
+)

g3doc/user_guide/fs_snapshot.md

@@ -0,0 +1,57 @@
+# Filesystem Snapshots
+
+[TOC]
+
+gVisor has the ability to save changes made to root filesystems in a *filesystem
+snapshot*, and subsequently restore those changes in new sandboxes.
+
+Compared to [rootfs tar snapshots](rootfs_snapshot.md):
+
+*   Filesystem snapshots include root filesystem changes in all containers in a
+    sandbox, rather than a single container.
+
+*   Filesystem snapshots comprise multiple files in a directory.
+
+*   Save and restore throughput is higher for filesystem snapshots than for
+    rootfs tar snapshots, subject to disk throughput limits.
+
+*   Containers can begin execution while filesystem snapshot restore is in
+    progress, but must wait for rootfs tar snapshot restore to complete.
+
+*   Filesystem snapshots store sparse files more efficiently than rootfs tar
+    snapshots.
+
+## Prerequisites
+
+*   Container root filesystems must be an overlay whose upper layer is
+    disk-backed tmpfs. This behavior is enabled by default, via the default
+    value of the `runsc -overlay2` flag. The requirement that the tmpfs is
+    disk-backed may be removed in the future.
+
+*   Filesystem snapshots do not include non-root tmpfs upper layers created by
+    the `-overlay2` flag with mount specifier `all`; tmpfs mounts specified by a
+    container spec; tmpfs mounts inferred from Kubernetes `emptyDir` volumes; or
+    tmpfs mounts created by sandboxed applications. Options to include these may
+    be added in the future.
+
+*   Filesystem snapshots can only be restored by the same runsc binary that
+    produced the snapshot. This restriction may be removed in the future.
+
+## Usage
+
+To save a filesystem snapshot, use the `runsc fscheckpoint` command, passing the
+directory that will contain the snapshot using the `--image-path` flag:
+
+```bash
+runsc fscheckpoint --image-path=<path> <container ID>
+```
+
+By default, saving a filesystem snapshot causes the sandbox to exit. This
+behavior can be disabled with the `--leave-running` flag.
+
+To restore a filesystem snapshot, pass the directory containing the snapshot to
+`runsc create` or `runsc run` using the `--fs-restore-image-path` flag:
+
+```bash
+runsc create --fs-restore-image-path=<path> <container ID>
+```

g3doc/user_guide/rootfs_snapshot.md

@@ -1,4 +1,4 @@
-# Rootfs Snapshot
+# Rootfs Tar Snapshots
 
 [TOC]
 
@@ -6,6 +6,14 @@ gVisor allows users to snapshot changes made to the root filesystem and save
 them to a tar file. These changes in the tar file can then be applied to a new
 sandbox upon creation.
 
+Compared to [filesystem snapshots](fs_snapshot.md):
+
+*   Rootfs tar snapshots include root filesystem changes from a single
+    container, rather than a whole sandbox.
+
+*   Rootfs tar snapshots are in a standard format (tar files) that can be
+    manipulated by other software.
+
 ## Prerequisite
 
 *   Rootfs must be overlayfs whose upper layer is tmpfs (this is the default

pkg/fd/fd.go

@@ -269,3 +269,13 @@ func (f *FD) File() (*os.File, error) {
 func (f *FD) ReleaseToFile(name string) *os.File {
 	return os.NewFile(uintptr(f.Release()), name)
 }
+
+// ReleaseToFiles returns the result of calling ReleaseToFile on each FD in
+// the given slice.
+func ReleaseToFiles(fds []*FD, name string) []*os.File {
+	files := make([]*os.File, len(fds))
+	for i, fd := range fds {
+		files[i] = fd.ReleaseToFile(name)
+	}
+	return files
+}

pkg/hostarch/BUILD

@@ -23,6 +23,7 @@ go_test(
     srcs = [
         "addr_range_seq_test.go",
         "addr_test.go",
+        "hostarch_test.go",
     ],
     library = ":hostarch",
 )

pkg/hostarch/hostarch.go

@@ -4,5 +4,23 @@
 // license that can be found in the LICENSE file or at
 // https://developers.google.com/open-source/licenses/bsd.
 
-// Package hostarch contains host arch address operations for user memory.
+// Package hostarch contains machine architecture parameters.
 package hostarch
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+// EndianString returns "little" if the invoking process is little-endian and
+// "big" if the invoking process is big-endian.
+func EndianString() string {
+	switch val := binary.NativeEndian.Uint32([]byte{0x01, 0x02, 0x03, 0x04}); val {
+	case 0x01020304:
+		return "big"
+	case 0x04030201:
+		return "little"
+	default:
+		panic(fmt.Sprintf("unknown endianness: [01 02 03 04] => %#x", val))
+	}
+}

pkg/hostarch/hostarch_test.go

@@ -0,0 +1,38 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hostarch
+
+import (
+	"runtime"
+	"testing"
+)
+
+func TestEndianString(t *testing.T) {
+	got := EndianString()
+	switch runtime.GOARCH {
+	case "386", "amd64", "arm", "arm64", "mipsle", "mips64le", "ppc64le", "riscv64", "wasm":
+		if got != "little" {
+			t.Errorf("got %s, want little", got)
+		}
+	case "armbe", "arm64be", "mips", "mips64", "ppc64", "s390x":
+		if got != "big" {
+			t.Errorf("got %s, want big", got)
+		}
+	default:
+		// Unknown GOARCH, unknown endianness; at least EndianString() didn't
+		// panic.
+		t.Logf("GOARCH %v is %s-endian", runtime.GOARCH, got)
+	}
+}

pkg/sentry/fscheckpoint/BUILD

@@ -0,0 +1,15 @@
+load("//tools:defs.bzl", "go_library")
+
+package(
+    default_applicable_licenses = ["//:license"],
+    licenses = ["notice"],
+)
+
+go_library(
+    name = "fscheckpoint",
+    srcs = ["fscheckpoint.go"],
+    visibility = ["//pkg/sentry:internal"],
+    deps = [
+        "//pkg/sentry/checkpoint",
+    ],
+)

pkg/sentry/fscheckpoint/fscheckpoint.go

@@ -0,0 +1,122 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package fscheckpoint defines the format of a gVisor filesystem checkpoint.
+//
+// Filesystem checkpoints can only be taken in sandboxes with runsc flag
+// `-overlay2` enabled, which causes sandboxed filesystems to be overlaid with
+// a tmpfs "upper layer" that captures filesystem changes. Each filesystem
+// checkpoint contains the state of all upper layers in the sandbox.
+//
+// A filesystem checkpoint comprises four files:
+//
+//   - A "pages metadata" file and a "pages" file are generated by
+//     pgalloc.MemoryFile save/restore. These contain the state of MemoryFiles
+//     that store the contents of files in checkpointed tmpfs filesystems.
+//
+//   - A "multi-tar" file contains tar archives representing the filesystem
+//     structure of checkpointed tmpfs filesystems. For regular files, these
+//     tar archives store the MemoryFile offsets containing file data, avoiding
+//     duplication of MemoryFile content and ensuring a dense representation of
+//     possibly-sparse files.
+//
+//   - A "manifest" file contains information about each MemoryFile and tmpfs
+//     filesystem required to load state from other checkpoint files. See type
+//     definitions below.
+//
+// TODO: NOLINT - The tar file format involves a lot of padding to 512-byte
+// boundaries. Consider gzip/zstd compression of (each archive in) the
+// multi-tar file, as well as something simpler like run-length encoding of
+// zero bytes.
+package fscheckpoint
+
+import (
+	"gvisor.dev/gvisor/pkg/sentry/checkpoint"
+)
+
+// Manifest is the type of the JSON object stored in the manifest file.
+type Manifest struct {
+	// Version is the checkpoint format version.
+	//
+	// While Version is 0, filesystem checkpoint compatibility is not
+	// guaranteed between differing runsc binary versions, so RunscVersion is
+	// used to check compatibility. Version should not be incremented to 1
+	// until filesystem checkpoint compatibility is established.
+	Version uint64 `json:"version,omitzero"`
+
+	// RunscVersion is the version of the runsc binary that produced this
+	// checkpoint.
+	//
+	// TODO: NOLINT - Currently this must match during restore. Removing this
+	// requirement involves at least stabilizing pgalloc's checkpoint format.
+	RunscVersion string `json:"runsc_version,omitzero"`
+
+	// PageSize is hostarch.PageSize for the sandbox that produced this
+	// checkpoint.
+	//
+	// TODO: NOLINT - Currently this must match during restore. Technically,
+	// it's already permissible for PageSize to shrink (as long as both old and
+	// new sizes remain powers of 2). Allowing PageSize to increase may require
+	// being able to relocate pgalloc.MemoryFile pages during restore based on
+	// information provided by tmpfs.
+	PageSize uint64 `json:"page_size"`
+
+	// Endian is hostarch.EndianString() for the sandbox that produced this
+	// checkpoint.
+	//
+	// TODO: NOLINT - Currently this must match during restore. Removing this
+	// requirement requires fixing an endianness for
+	// tmpfs.fsckptRegularFileSegment, and for pgalloc checkpoints.
+	Endian string `json:"endian"`
+
+	// MemoryFiles contains information about pgalloc.MemoryFiles stored in the
+	// checkpoint, in order of increasing file offsets in the pages metadata
+	// and pages files.
+	MemoryFiles []MemoryFile `json:"memory_files"`
+
+	// Information about filesystems stored in the checkpoint, in order of
+	// increasing file offsets in the multi-tar file.
+	Tmpfs []Tmpfs `json:"tmpfs"`
+}
+
+// MemoryFile represents a pgalloc.MemoryFile stored in a filesystem
+// checkpoint.
+type MemoryFile struct {
+	// ResourceID is the value of pgalloc.MemoryFile.ResourceID().
+	ResourceID checkpoint.ResourceID `json:"resource_id"`
+
+	// PagesMetadataStart and PagesMetadataEnd are the offsets in the pages
+	// metadata file at which the pages metadata for this MemoryFile begin and
+	// end respectively.
+	PagesMetadataStart uint64 `json:"pages_metadata_start"`
+	PagesMetadataEnd   uint64 `json:"pages_metadata_end"`
+
+	// PagesStart is the offset in the pages file at which the pages for this
+	// MemoryFile begin.
+	PagesStart uint64 `json:"pages_start"`
+}
+
+// Tmpfs represents a tmpfs filesystem stored in a filesystem checkpoint.
+type Tmpfs struct {
+	// ResourceID is the value of tmpfs.filesystem.mf.ResourceID().
+	//
+	// TODO: NOLINT - Assign ResourceIDs to tmpfs filesystems directly so that
+	// we can also checkpoint filesystems using the main MemoryFile.
+	ResourceID checkpoint.ResourceID `json:"resource_id"`
+
+	// TarStart and TarEnd are the offsets in the multi-tar file at which the
+	// tar archive for this filesystem begin and end respectively.
+	TarStart uint64 `json:"tar_start"`
+	TarEnd   uint64 `json:"tar_end"`
+}

pkg/sentry/fsimpl/tmpfs/BUILD

@@ -85,6 +85,7 @@ go_library(
         "directory.go",
         "filesystem.go",
         "filesystem_mutex.go",
+        "fscheckpoint.go",
         "fstree.go",
         "inode_mutex.go",
         "inode_refs.go",
@@ -101,6 +102,7 @@ go_library(
     imports = [
         "gvisor.dev/gvisor/pkg/sentry/checkpoint",
     ],
+    marshal = True,
     visibility = ["//pkg/sentry:internal"],
     deps = [
         "//pkg/abi/linux",
@@ -112,6 +114,7 @@ go_library(
         "//pkg/fsutil",
         "//pkg/hostarch",
         "//pkg/log",
+        "//pkg/marshal",
         "//pkg/refs",
         "//pkg/safemem",
         "//pkg/sentry/arch",

pkg/sentry/fsimpl/tmpfs/fscheckpoint.go

@@ -0,0 +1,162 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tmpfs
+
+import (
+	"archive/tar"
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/sentry/memmap"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
+	"gvisor.dev/gvisor/pkg/sentry/vfs"
+)
+
+// MemoryFileOf returns the pgalloc.MemoryFile used by the given tmpfs
+// filesystem. If vfsfs is not a tmpfs filesystem, MemoryFileOf returns nil.
+func MemoryFileOf(vfsfs *vfs.Filesystem) *pgalloc.MemoryFile {
+	if fs, _ := vfsfs.Impl().(*filesystem); fs != nil {
+		return fs.mf
+	}
+	return nil
+}
+
+// FSCheckpointWrite serializes the given tmpfs filesystem to dst. The contents
+// of its regular files are not written to dst; instead, callers must
+// separately save and restore the contents of the filesystem's MemoryFile. If
+// vfsfs is not a tmpfs filesystem, FSCheckpointWrite returns an error.
+//
+// Preconditions: The Kernel must be paused and quiesced.
+func FSCheckpointWrite(ctx context.Context, vfsfs *vfs.Filesystem, dst io.Writer) error {
+	fs, _ := vfsfs.Impl().(*filesystem)
+	if fs == nil {
+		return fmt.Errorf("non-tmpfs filesystem: %T", vfsfs.Impl())
+	}
+	return fs.tarWrite(ctx, dst, &fsckptTarWriterCallbacks{
+		regularFiles: make(map[*regularFile]*fsckptRegularFile),
+	})
+}
+
+// fsckptTarWriterCallbacks implements tarWriterCallbacks by storing MemoryFile
+// offsets containing regular file data in the tar archive.
+type fsckptTarWriterCallbacks struct {
+	regularFiles map[*regularFile]*fsckptRegularFile
+}
+
+type fsckptRegularFile struct {
+	size uint64
+	data []fsckptRegularFileSegment
+}
+
+// fsckptRegularFileSegment is equivalent to fsutil.FileRangeFlatSegment, but
+// is marshalable.
+//
+// +marshal slice:CheckpointRegularFileSegmentSlice
+type fsckptRegularFileSegment struct {
+	Start uint64
+	End   uint64
+	Value uint64
+}
+
+func (cb *fsckptTarWriterCallbacks) checkpointRegularFile(rf *regularFile) *fsckptRegularFile {
+	crf := cb.regularFiles[rf]
+	if crf == nil {
+		rf.dataMu.RLock()
+		defer rf.dataMu.RUnlock()
+		crf = &fsckptRegularFile{
+			size: rf.size.RacyLoad(),
+		}
+		for seg := rf.data.FirstSegment(); seg.Ok(); seg = seg.NextSegment() {
+			crf.data = append(crf.data, fsckptRegularFileSegment{
+				Start: seg.Start(),
+				End:   seg.End(),
+				Value: seg.Value(),
+			})
+		}
+		cb.regularFiles[rf] = crf
+	}
+	return crf
+}
+
+func (cb *fsckptTarWriterCallbacks) regularFileSize(rf *regularFile) int64 {
+	crf := cb.checkpointRegularFile(rf)
+	return 8 /* size */ + int64(len(crf.data))*int64((*fsckptRegularFileSegment)(nil).SizeBytes())
+}
+
+func (cb *fsckptTarWriterCallbacks) regularFileWrite(ctx context.Context, rf *regularFile, tw *tar.Writer) error {
+	crf := cb.checkpointRegularFile(rf)
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], crf.size)
+	if _, err := tw.Write(buf[:]); err != nil {
+		return fmt.Errorf("failed to write file size to tar: %w", err)
+	}
+	if _, err := WriteCheckpointRegularFileSegmentSlice(tw, crf.data); err != nil {
+		return fmt.Errorf("failed to write file segments to tar: %w", err)
+	}
+	return nil
+}
+
+// fsckptTarReaderCallbacks implements tarReaderCallbacks by storing MemoryFile
+// offsets containing regular file data in the tar archive.
+type fsckptTarReaderCallbacks struct {
+	fs           *filesystem
+	regularFiles map[*tar.Header]*fsckptRegularFile
+}
+
+func (cb *fsckptTarReaderCallbacks) regularFileRead(ctx context.Context, hdr *tar.Header, tr *tar.Reader) error {
+	if hdr.Size < 8 {
+		return fmt.Errorf("header size %d too small for regular file size", hdr.Size)
+	}
+	remSize := hdr.Size - 8
+	segSize := int64((*fsckptRegularFileSegment)(nil).SizeBytes())
+	if remSize%segSize != 0 {
+		return fmt.Errorf("header size %d is not 8 + integer multiple of %d", hdr.Size, segSize)
+	}
+	var buf [8]byte
+	if _, err := io.ReadFull(tr, buf[:]); err != nil {
+		return fmt.Errorf("failed to read file size from tar: %w", err)
+	}
+	crf := &fsckptRegularFile{
+		size: binary.LittleEndian.Uint64(buf[:]),
+		data: make([]fsckptRegularFileSegment, remSize/segSize),
+	}
+	if _, err := ReadCheckpointRegularFileSegmentSlice(tr, crf.data); err != nil {
+		return fmt.Errorf("failed to read file segments from tar: %w", err)
+	}
+	cb.regularFiles[hdr] = crf
+	return nil
+}
+
+func (cb *fsckptTarReaderCallbacks) regularFileSetContents(ctx context.Context, hdr *tar.Header, rf *regularFile) error {
+	crf := cb.regularFiles[hdr]
+	rf.inode.mu.Lock()
+	defer rf.inode.mu.Unlock()
+	rf.dataMu.Lock()
+	defer rf.dataMu.Unlock()
+	rf.size.Store(uint64(crf.size))
+	gap := rf.data.FirstGap()
+	n := uint64(0)
+	for _, rfseg := range crf.data {
+		gap = rf.data.Insert(gap, memmap.MappableRange{rfseg.Start, rfseg.End}, rfseg.Value).NextGap()
+		n += (rfseg.End - rfseg.Start) / hostarch.PageSize
+	}
+	if !cb.fs.accountPages(n) {
+		return fmt.Errorf("restored filesystem would exceed size limit of %d pages", cb.fs.maxSizeInPages)
+	}
+	return nil
+}

pkg/sentry/fsimpl/tmpfs/tar.go

@@ -32,15 +32,15 @@ import (
 	"gvisor.dev/gvisor/pkg/usermem"
 )
 
-// UntarUpperLayer creates the corresponding dentry and its children from the
-// given snapshot tar file.
-func (fs *filesystem) UntarUpperLayer(ctx context.Context, inFile *os.File) error {
-	tr := tar.NewReader(inFile)
+// tarRead creates the corresponding dentry and its children from the given
+// snapshot tar file.
+func (fs *filesystem) tarRead(ctx context.Context, src io.Reader, cb tarReaderCallbacks) error {
+	tr := tar.NewReader(src)
 
 	fs.mu.Lock()
 	defer fs.mu.Unlock()
 
-	return fs.readFromTar(ctx, tr)
+	return fs.readFromTar(ctx, tr, cb)
 }
 
 // readFromTar creates the corresponding dentry and its children from the given
@@ -48,13 +48,12 @@ func (fs *filesystem) UntarUpperLayer(ctx context.Context, inFile *os.File) erro
 //
 // Preconditions:
 //   - filesystem.mu must be locked.
-func (fs *filesystem) readFromTar(ctx context.Context, tr *tar.Reader) error {
+func (fs *filesystem) readFromTar(ctx context.Context, tr *tar.Reader, cb tarReaderCallbacks) error {
 	pathToInode := map[string]*inode{}
 	directoryToHeader := map[string]*tar.Header{}
 	fileToHeader := map[string]*tar.Header{}
 	symlinkToHeader := map[string]*tar.Header{}
 	linkToHeader := map[string]*tar.Header{}
-	fileToContent := map[string]*bytes.Buffer{}
 	for {
 		header, err := tr.Next()
 		if err != nil {
@@ -68,16 +67,8 @@ func (fs *filesystem) readFromTar(ctx context.Context, tr *tar.Reader) error {
 		case tar.TypeDir:
 			directoryToHeader[header.Name] = header
 		case tar.TypeReg:
-			var buffer bytes.Buffer
-			n, err := io.Copy(&buffer, tr)
-			if err != nil {
-				return fmt.Errorf("failed to read file content: %w", err)
-			}
-			if n != header.Size {
-				return fmt.Errorf("failed to read all file content, got %d bytes, want %d", n, header.Size)
-			}
-			if header.Size > 0 {
-				fileToContent[header.Name] = &buffer
+			if err := cb.regularFileRead(ctx, header, tr); err != nil {
+				return err
 			}
 			fileToHeader[header.Name] = header
 		case tar.TypeFifo, tar.TypeBlock, tar.TypeChar:
@@ -98,7 +89,7 @@ func (fs *filesystem) readFromTar(ctx context.Context, tr *tar.Reader) error {
 	}
 	// Re-create all regular files, FIFOs, block devices, and character devices.
 	for path, hdr := range fileToHeader {
-		if err := fs.mknodFromTar(ctx, hdr, pathToInode, fileToContent); err != nil {
+		if err := fs.mknodFromTar(ctx, hdr, pathToInode, cb); err != nil {
 			return fmt.Errorf("failed to make file %v: %w", path, err)
 		}
 	}
@@ -166,7 +157,7 @@ func (fs *filesystem) mkdirFromTar(hdr *tar.Header, pathToInode map[string]*inod
 // mknodFromTar creates a regular file,FIFO, block device, or character device file using
 // the provided header. It also writes the file content to the corresponding regular file if it
 // exists.
-func (fs *filesystem) mknodFromTar(ctx context.Context, hdr *tar.Header, pathToInode map[string]*inode, pathToContent map[string]*bytes.Buffer) error {
+func (fs *filesystem) mknodFromTar(ctx context.Context, hdr *tar.Header, pathToInode map[string]*inode, cb tarReaderCallbacks) error {
 	dir, name := filepath.Split(hdr.Name)
 	parentInode, ok := pathToInode[dir]
 	if !ok {
@@ -194,10 +185,10 @@ func (fs *filesystem) mknodFromTar(ctx context.Context, hdr *tar.Header, pathToI
 	parentDir.insertChildLocked(child, name)
 	pathToInode[hdr.Name] = childInode
 
-	// Write file contents to the corresponding regular files if they exist.
-	if buf, ok := pathToContent[hdr.Name]; ok {
-		if err := fs.writeTo(ctx, hdr.Name, pathToInode, int64(len(buf.Bytes())), buf); err != nil {
-			return fmt.Errorf("failed to write file content for %v: %w", hdr.Name, err)
+	// Write file contents to the corresponding regular files.
+	if rf, _ := childInode.impl.(*regularFile); rf != nil {
+		if err := cb.regularFileSetContents(ctx, hdr, rf); err != nil {
+			return err
 		}
 	}
 
@@ -236,9 +227,6 @@ func (fs *filesystem) symlinkFromTar(hdr *tar.Header, pathToInode map[string]*in
 	if !ok {
 		return fmt.Errorf("parent directory %v does not exist", dir)
 	}
-	if len(hdr.Linkname) >= shortSymlinkLen {
-		return fmt.Errorf("symlink %v is too long", hdr.Linkname)
-	}
 	parentDir, ok := parentInode.impl.(*directory)
 	if !ok {
 		return fmt.Errorf("%v is not a directory", dir)
@@ -250,12 +238,42 @@ func (fs *filesystem) symlinkFromTar(hdr *tar.Header, pathToInode map[string]*in
 	return nil
 }
 
-func (fs *filesystem) writeTo(ctx context.Context, path string, pathToInode map[string]*inode, size int64, buf *bytes.Buffer) error {
-	i, ok := pathToInode[path]
+type tarReaderCallbacks interface {
+	// regularFileRead reads information about the regular file with header hdr
+	// from tr.
+	regularFileRead(ctx context.Context, hdr *tar.Header, tr *tar.Reader) error
+
+	// regularFileSetContents sets the contents and size of rf, using what was
+	// previously read for hdr.
+	regularFileSetContents(ctx context.Context, hdr *tar.Header, rf *regularFile) error
+}
+
+// tarDefaultReaderCallbacks implements tarReaderCallbacks by reading regular
+// file contents from the tar archive.
+type tarDefaultReaderCallbacks struct {
+	headerToContent map[*tar.Header]*bytes.Buffer
+}
+
+func (cb *tarDefaultReaderCallbacks) regularFileRead(ctx context.Context, hdr *tar.Header, tr *tar.Reader) error {
+	var buf bytes.Buffer
+	n, err := io.Copy(&buf, tr)
+	if err != nil {
+		return fmt.Errorf("failed to read file content: %w", err)
+	}
+	if n != hdr.Size {
+		return fmt.Errorf("failed to read all file content, got %d bytes, want %d", n, hdr.Size)
+	}
+	if hdr.Size > 0 {
+		cb.headerToContent[hdr] = &buf
+	}
+	return nil
+}
+
+func (cb *tarDefaultReaderCallbacks) regularFileSetContents(ctx context.Context, hdr *tar.Header, rf *regularFile) error {
+	buf, ok := cb.headerToContent[hdr]
 	if !ok {
-		return fmt.Errorf("failed to find inode for %v", path)
+		return nil
 	}
-	rf := i.impl.(*regularFile)
 	rf.inode.mu.Lock()
 	defer rf.inode.mu.Unlock()
 	src := usermem.BytesIOSequence(buf.Bytes())
@@ -264,8 +282,8 @@ func (fs *filesystem) writeTo(ctx context.Context, path string, pathToInode map[
 	if err != nil {
 		return fmt.Errorf("failed to write file content: %w", err)
 	}
-	if n != size {
-		return fmt.Errorf("failed to write all file content to %v, got %d bytes, want %d", path, n, size)
+	if size := int64(len(buf.Bytes())); n != size {
+		return fmt.Errorf("failed to write all file content to %v, got %d bytes, want %d", hdr.Name, n, size)
 	}
 	putRegularFileReadWriter(rw)
 	return nil
@@ -273,12 +291,16 @@ func (fs *filesystem) writeTo(ctx context.Context, path string, pathToInode map[
 
 // TarUpperLayer implements vfs.TarSerializer.TarUpperLayer.
 func (fs *filesystem) TarUpperLayer(ctx context.Context, outFD *os.File) error {
-	tw := tar.NewWriter(outFD)
+	return fs.tarWrite(ctx, outFD, tarDefaultWriterCallbacks{})
+}
+
+func (fs *filesystem) tarWrite(ctx context.Context, dst io.Writer, cb tarWriterCallbacks) error {
+	tw := tar.NewWriter(dst)
 
 	fs.mu.RLock()
 	defer fs.mu.RUnlock()
 
-	err := fs.root.writeToTar(ctx, tw, ".", make(map[uint64]string))
+	err := fs.root.writeToTar(ctx, tw, ".", make(map[uint64]string), cb)
 	if err != nil {
 		return fmt.Errorf("failed to write dentry to tar: %w", err)
 	}
@@ -291,12 +313,12 @@ func (fs *filesystem) TarUpperLayer(ctx context.Context, outFD *os.File) error {
 }
 
 // writeToTar recursively writes a dentry and its children to the tar archive.
-func (d *dentry) writeToTar(ctx context.Context, tw *tar.Writer, baseDir string, inoToPath map[uint64]string) error {
+func (d *dentry) writeToTar(ctx context.Context, tw *tar.Writer, baseDir string, inoToPath map[uint64]string, cb tarWriterCallbacks) error {
 	path := baseDir
 	if d.name != "" {
 		path = path + "/" + d.name
 	}
-	header, err := d.createTarHeader(path, inoToPath)
+	header, err := d.createTarHeader(path, inoToPath, cb)
 	if err != nil {
 		return fmt.Errorf("failed to create tar header for %q: %w", path, err)
 	}
@@ -316,12 +338,12 @@ func (d *dentry) writeToTar(ctx context.Context, tw *tar.Writer, baseDir string,
 	switch impl := d.inode.impl.(type) {
 	case *directory:
 		for _, child := range impl.childMap {
-			if err := child.writeToTar(ctx, tw, path, inoToPath); err != nil {
+			if err := child.writeToTar(ctx, tw, path, inoToPath, cb); err != nil {
 				return err
 			}
 		}
 	case *regularFile:
-		if err := impl.writeToTar(ctx, tw); err != nil {
+		if err := cb.regularFileWrite(ctx, impl, tw); err != nil {
 			return fmt.Errorf("failed to write file content for %q: %w", path, err)
 		}
 	}
@@ -330,7 +352,7 @@ func (d *dentry) writeToTar(ctx context.Context, tw *tar.Writer, baseDir string,
 }
 
 // createTarHeader creates a tar header for the given dentry.
-func (d *dentry) createTarHeader(path string, inoToPath map[uint64]string) (*tar.Header, error) {
+func (d *dentry) createTarHeader(path string, inoToPath map[uint64]string, cb tarWriterCallbacks) (*tar.Header, error) {
 	if d.isSelfFilestoreWhiteout() {
 		// Skip the self filestore whiteout.
 		return nil, nil
@@ -356,7 +378,7 @@ func (d *dentry) createTarHeader(path string, inoToPath map[uint64]string) (*tar
 		header.Name += "/"
 	case *regularFile:
 		header.Typeflag = tar.TypeReg
-		header.Size = int64(impl.size.Load())
+		header.Size = cb.regularFileSize(impl)
 	case *symlink:
 		header.Typeflag = tar.TypeSymlink
 		header.Linkname = impl.target
@@ -381,8 +403,24 @@ func (d *dentry) createTarHeader(path string, inoToPath map[uint64]string) (*tar
 	return header, nil
 }
 
-// writeToTar writes the content of a regular file to the tar archive.
-func (rf *regularFile) writeToTar(ctx context.Context, tw *tar.Writer) error {
+type tarWriterCallbacks interface {
+	// regularFileSize returns the size of the given regular file as written to
+	// the tar archive.
+	regularFileSize(rf *regularFile) int64
+
+	// regularFileWrite writes the contents of the given regular file to tw.
+	regularFileWrite(ctx context.Context, rf *regularFile, tw *tar.Writer) error
+}
+
+// tarDefaultWriterCallbacks implements tarWriterCallbacks by writing regular
+// file contents to the tar archive.
+type tarDefaultWriterCallbacks struct{}
+
+func (tarDefaultWriterCallbacks) regularFileSize(rf *regularFile) int64 {
+	return int64(rf.size.Load())
+}
+
+func (tarDefaultWriterCallbacks) regularFileWrite(ctx context.Context, rf *regularFile, tw *tar.Writer) error {
 	// Note that regularFileReadWriter.ReadToBlocks() does not lock inode.mu. So
 	// it is safe to lock here to ensure no concurrent writes occur.
 	rf.inode.mu.Lock()

pkg/sentry/fsimpl/tmpfs/tmpfs.go

@@ -30,12 +30,15 @@
 package tmpfs
 
 import (
+	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
 	"math"
-	"os"
 	"strconv"
 	"strings"
 	"sync/atomic"
+	"time"
 
 	"gvisor.dev/gvisor/pkg/abi/linux"
 	"gvisor.dev/gvisor/pkg/atomicbitops"
@@ -155,9 +158,14 @@ type FilesystemOpts struct {
 	// tmpfs mount will allow.
 	AllowXattrPrefix []string
 
-	// SourceTarFD is the file descriptor of the source tar file to be untarred
-	// into the tmpfs.
-	SourceTarFile *os.File
+	// SourceTar is the source tar file to be untarred into the tmpfs.
+	// Ownership is transferred to GetFilesystem, whether or not it returns an
+	// error. If SourceTarFSCheckpoint is true, SourceTar was produced by a
+	// call to FSCheckpoint, and the caller is responsible for restoring
+	// MemoryFile's contents; otherwise, SourceTar was produced by a call to
+	// TarWrite.
+	SourceTar             io.ReadCloser
+	SourceTarFSCheckpoint bool
 }
 
 // Default size limit mount option. It is immutable after initialization.
@@ -217,6 +225,9 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 		for _, xattr := range tmpfsOpts.AllowXattrPrefix {
 			allowXattrPrefix[xattr] = struct{}{}
 		}
+		if tmpfsOpts.SourceTar != nil {
+			defer tmpfsOpts.SourceTar.Close()
+		}
 	}
 
 	mopts := vfs.GenericParseMountOptions(opts.Data)
@@ -327,13 +338,24 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt
 	}
 	fs.root = root
 
-	if tmpfsOptsOk && tmpfsOpts.SourceTarFile != nil {
-		defer tmpfsOpts.SourceTarFile.Close()
-
-		if err := fs.UntarUpperLayer(ctx, tmpfsOpts.SourceTarFile); err != nil {
+	if tmpfsOptsOk && tmpfsOpts.SourceTar != nil {
+		var cb tarReaderCallbacks
+		if tmpfsOpts.SourceTarFSCheckpoint {
+			cb = &fsckptTarReaderCallbacks{
+				fs:           &fs,
+				regularFiles: make(map[*tar.Header]*fsckptRegularFile),
+			}
+		} else {
+			cb = &tarDefaultReaderCallbacks{
+				headerToContent: make(map[*tar.Header]*bytes.Buffer),
+			}
+		}
+		timeUntarStart := time.Now()
+		if err := fs.tarRead(ctx, tmpfsOpts.SourceTar, cb); err != nil {
 			fs.vfsfs.DecRef(ctx)
 			return nil, nil, err
 		}
+		ctx.Infof("tmpfs.filesystem: loaded SourceTar in %s", time.Since(timeUntarStart))
 	}
 
 	return &fs.vfsfs, &root.vfsd, nil

pkg/sentry/kernel/BUILD

@@ -254,6 +254,7 @@ go_library(
         "fs_context.go",
         "fs_context_mutex.go",
         "fs_context_refs.go",
+        "fscheckpoint.go",
         "ipc_namespace.go",
         "kcov.go",
         "kcov_unsafe.go",
@@ -363,6 +364,7 @@ go_library(
         "//pkg/sentry/checkpoint",
         "//pkg/sentry/devices/nvproxy/nvconf",
         "//pkg/sentry/fdcollector",
+        "//pkg/sentry/fscheckpoint",
         "//pkg/sentry/fsimpl/kernfs",
         "//pkg/sentry/fsimpl/lock",
         "//pkg/sentry/fsimpl/mqfs",

pkg/sentry/kernel/fscheckpoint.go

@@ -0,0 +1,241 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kernel
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"gvisor.dev/gvisor/pkg/abi/linux"
+	"gvisor.dev/gvisor/pkg/cleanup"
+	"gvisor.dev/gvisor/pkg/context"
+	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/checkpoint"
+	"gvisor.dev/gvisor/pkg/sentry/fscheckpoint"
+	"gvisor.dev/gvisor/pkg/sentry/fsimpl/tmpfs"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
+	"gvisor.dev/gvisor/pkg/sentry/state/stateio"
+	"gvisor.dev/gvisor/pkg/sentry/vfs"
+	"gvisor.dev/gvisor/pkg/sync"
+)
+
+// FSSaveOpts holds options to Kernel.FSSave.
+type FSSaveOpts struct {
+	// These correspond to files specified by the fscheckpoint package, and are
+	// all required.
+	ManifestFile      io.WriteCloser
+	MultiTarFile      io.WriteCloser
+	PagesMetadataFile io.WriteCloser
+	PagesFile         stateio.AsyncWriter
+
+	// RunscVersion is the runsc binary version.
+	RunscVersion string
+
+	// If ExitAfterSaving is true, all processes exit with status 0 before
+	// FSSave returns, whether or not it returns a non-nil error.
+	ExitAfterSaving bool
+}
+
+// FSSave collects a filesystem checkpoint as specified by the fscheckpoint
+// package. FSSave takes ownership of resources in opts.
+func (k *Kernel) FSSave(ctx context.Context, opts *FSSaveOpts) error {
+	defer func() {
+		if opts.ManifestFile != nil {
+			opts.ManifestFile.Close()
+			opts.ManifestFile = nil
+		}
+		if opts.MultiTarFile != nil {
+			opts.MultiTarFile.Close()
+			opts.MultiTarFile = nil
+		}
+		if opts.PagesMetadataFile != nil {
+			opts.PagesMetadataFile.Close()
+			opts.PagesMetadataFile = nil
+		}
+		if opts.PagesFile != nil {
+			opts.PagesFile.Close()
+			opts.PagesFile = nil
+		}
+	}()
+
+	k.Pause()
+	defer k.Unpause()
+	if opts.ExitAfterSaving {
+		defer k.Kill(linux.WaitStatusExit(0)) // consistent with sentry/state.SaveOpts.Save
+	}
+	return k.quiescePausedAnd(ctx, func() error {
+		var (
+			asyncPageSaveWg  sync.WaitGroup
+			asyncPageSaveErr error
+		)
+		asyncPageSaveWg.Add(1)
+		apfs, err := pgalloc.StartAsyncPagesFileSave(opts.PagesFile /* transfers ownership */, func(err error) {
+			defer asyncPageSaveWg.Done()
+			asyncPageSaveErr = err
+		})
+		opts.PagesFile = nil
+		if err != nil {
+			return fmt.Errorf("failed to start async page saving: %w", err)
+		}
+		asyncPageSaveCleanup := cleanup.Make(func() {
+			apfs.MemoryFilesDone()
+			asyncPageSaveWg.Wait()
+		})
+		defer asyncPageSaveCleanup.Clean()
+		mfOpts := pgalloc.SaveOpts{
+			PagesFile: apfs,
+		}
+
+		manifest := fscheckpoint.Manifest{
+			RunscVersion: opts.RunscVersion,
+			PageSize:     hostarch.PageSize,
+			Endian:       hostarch.EndianString(),
+		}
+		type tmpfsAndPrivateMemoryFile struct {
+			fs *vfs.Filesystem
+			mf *pgalloc.MemoryFile
+		}
+		resourceIDs := make(map[checkpoint.ResourceID]tmpfsAndPrivateMemoryFile)
+		multiTarWriter := &countingWriter{w: opts.MultiTarFile}
+		pagesMetadataWriter := &countingWriter{w: opts.PagesMetadataFile}
+		prevTarOffset := uint64(0)
+		prevPagesMetadataOffset := uint64(0)
+		prevPagesOffset := uint64(0)
+		fss := k.vfs.GetFilesystems()
+		defer func() {
+			for _, fs := range fss {
+				fs.DecRef(ctx)
+			}
+		}()
+		// TODO: NOLINT - fss is obtained by iterating a map, so its order -
+		// and thus the order in which filesystems will be saved - is
+		// effectively random. pgalloc.MemoryFile async page loading biases
+		// toward MemoryFiles stored at lower offsets in the pages file. We
+		// should save both filesystems and MemoryFiles in the order that they
+		// were created, which is most likely to be the order in which they are
+		// created after restore; in particular, this will be the case when the
+		// same Kubernetes Pod spec is reused after restore.
+		for _, fs := range fss {
+			mf := tmpfs.MemoryFileOf(fs)
+			if mf == nil {
+				continue
+			}
+			resourceID := mf.ResourceID()
+			if resourceID.Path != "/" {
+				// This excludes:
+				// - MemoryFiles with no ResourceID, i.e. the main MemoryFile.
+				//   This currently needs to be excluded since we have no way
+				//   to save only the contents of the main MemoryFile owned by
+				//   a checkpointed filesystem, and don't want to save all
+				//   application memory.
+				// - Disk-backed MemoryFiles for non-rootfs tmpfs filesystems.
+				//   This is currently needed to exclude disk-backed Kubernetes
+				//   emptyDir volumes, which is in turn for consistency with
+				//   memory-backed emptyDir volumes, which are excluded by the
+				//   above. (This confusion isn't considered a problem for
+				//   rootfs because rootfs tmpfs is typically disk-backed via
+				//   the default value of the runsc -overlay2 flag.) This also
+				//   has the side effect of excluding non-rootfs tmpfs
+				//   filesystems created by runsc -overlay2=all:self.
+				//
+				// TODO: NOLINT - Provide an option to relax this requirement
+				// by implementing the ability to save a subset of a
+				// pgalloc.MemoryFile, and using it to save only the subset of
+				// each MemoryFile used by any checkpointed tmpfs filesystem.
+				// This would also avoid saving MemoryFile pages that are
+				// referenced by e.g. a previous MM.Pin() but no longer owned
+				// by a regularFile; in such cases, the holder of the extra
+				// reference won't be restored by filesystem checkpointing,
+				// causing the referenced pages to be leaked. (As of this
+				// writing, this leak is unlikely to be an issue in practice,
+				// since the primary user of MM.Pin() is nvproxy, and the
+				// Nvidia driver appears to reject pinning mappings of
+				// disk-backed files.)
+				continue
+			}
+			if old, ok := resourceIDs[resourceID]; ok {
+				return fmt.Errorf("pgalloc.MemoryFile restore ID %q is used by both tmpfs filesystem %p (MemoryFile %p) and tmpfs filesystem %p (MemoryFile %p)", resourceID, old.fs, old.mf, fs, mf)
+			}
+			resourceIDs[resourceID] = tmpfsAndPrivateMemoryFile{fs, mf}
+			if log.IsLogging(log.Debug) {
+				log.Debugf("Filesystem checkpoint saving tmpfs with resourceID %s", resourceID)
+			}
+			if err := mf.SaveTo(ctx, pagesMetadataWriter, &mfOpts); err != nil {
+				return fmt.Errorf("failed to save MemoryFile with resourceID %s: %w", resourceID, err)
+			}
+			manifest.MemoryFiles = append(manifest.MemoryFiles, fscheckpoint.MemoryFile{
+				ResourceID:         resourceID,
+				PagesMetadataStart: prevPagesMetadataOffset,
+				PagesMetadataEnd:   pagesMetadataWriter.count,
+				PagesStart:         prevPagesOffset,
+			})
+			prevPagesMetadataOffset = pagesMetadataWriter.count
+			prevPagesOffset = apfs.PagesFileOffset()
+			if err := tmpfs.FSCheckpointWrite(ctx, fs, multiTarWriter); err != nil {
+				return fmt.Errorf("failed to write tmpfs with resourceID %s to multi-tar file: %w", resourceID, err)
+			}
+			manifest.Tmpfs = append(manifest.Tmpfs, fscheckpoint.Tmpfs{
+				ResourceID: resourceID,
+				TarStart:   prevTarOffset,
+				TarEnd:     multiTarWriter.count,
+			})
+			prevTarOffset = multiTarWriter.count
+		}
+		if len(resourceIDs) == 0 {
+			return fmt.Errorf("no checkpointable filesystems")
+		}
+		apfs.MemoryFilesDone()
+		if err := json.NewEncoder(opts.ManifestFile).Encode(manifest); err != nil {
+			return fmt.Errorf("failed to write manifest file: %w", err)
+		}
+		// Close other writers while MemoryFile saving is in progress to
+		// overlap their latencies.
+		err = opts.ManifestFile.Close()
+		opts.ManifestFile = nil
+		if err != nil {
+			return fmt.Errorf("failed to close manifest file: %w", err)
+		}
+		err = opts.MultiTarFile.Close()
+		opts.MultiTarFile = nil
+		if err != nil {
+			return fmt.Errorf("failed to close multi-tar file: %w", err)
+		}
+		err = opts.PagesMetadataFile.Close()
+		opts.PagesMetadataFile = nil
+		if err != nil {
+			return fmt.Errorf("failed to close pages metadata file: %w", err)
+		}
+		// Finally wait for MemoryFile saving to complete.
+		asyncPageSaveCleanup.Release()()
+		if asyncPageSaveErr != nil {
+			return fmt.Errorf("failed to save MemoryFile pages: %w", asyncPageSaveErr)
+		}
+		return nil
+	})
+}
+
+type countingWriter struct {
+	w     io.Writer
+	count uint64
+}
+
+// Write implements io.Writer.Write.
+func (cw *countingWriter) Write(src []byte) (int, error) {
+	n, err := cw.w.Write(src)
+	cw.count += uint64(n)
+	return n, err
+}

pkg/sentry/kernel/kernel.go

@@ -596,6 +596,26 @@ func (k *Kernel) Init(args InitKernelArgs) error {
 	return nil
 }
 
+func (k *Kernel) quiescePausedAnd(ctx context.Context, f func() error) error {
+	// Do not allow other Kernel methods to affect it while it's being saved.
+	k.extMu.Lock()
+	defer k.extMu.Unlock()
+
+	// Suspend fdnotifier notifications.
+	fdnotifier.Pause()
+	defer fdnotifier.Resume()
+
+	// Stop time.
+	k.pauseTimeLocked(ctx)
+	defer k.resumeTimeLocked(ctx)
+
+	// Evict all evictable MemoryFile allocations.
+	k.mf.StartEvictions()
+	k.mf.WaitForEvictions()
+
+	return f()
+}
+
 // +stateify savable
 type privateMemoryFileMetadata struct {
 	owners []checkpoint.ResourceID
@@ -643,120 +663,106 @@ func (k *Kernel) SaveTo(ctx context.Context, stateFile, pagesMetadata io.WriteCl
 	}
 	defer pagesCleanup.Clean()
 
-	// Do not allow other Kernel methods to affect it while it's being saved.
-	k.extMu.Lock()
-	defer k.extMu.Unlock()
+	return k.quiescePausedAnd(ctx, func() error {
+		// Discard unsavable mappings, such as those for host file descriptors.
+		if err := k.invalidateUnsavableMappings(ctx); err != nil {
+			return fmt.Errorf("failed to invalidate unsavable mappings: %v", err)
+		}
 
-	// Suspend fdnotifier notifications.
-	fdnotifier.Pause()
-	defer fdnotifier.Resume()
+		// Capture all private memory files.
+		mfsToSave := make(map[checkpoint.ResourceID]*pgalloc.MemoryFile)
+		vfsCtx := context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfsToSave)
+		// Prepare filesystems for saving. This must be done after
+		// invalidateUnsavableMappings(), since dropping memory mappings may
+		// affect filesystem state (e.g. page cache reference counts).
+		if err := k.vfs.PrepareSave(vfsCtx); err != nil {
+			return err
+		}
+		// Mark all to-be-saved MemoryFiles as savable to inform kernel save below.
+		k.mf.MarkSavable()
+		for _, mf := range mfsToSave {
+			mf.MarkSavable()
+		}
 
-	// Stop time.
-	k.pauseTimeLocked(ctx)
-	defer k.resumeTimeLocked(ctx)
+		var (
+			mfSaveWg  sync.WaitGroup
+			mfSaveErr error
+		)
+		if parallelMFSave {
+			// Parallelize MemoryFile save and kernel save. Both are independent.
+			mfSaveWg.Add(1)
+			go func() {
+				defer mfSaveWg.Done()
+				mfSaveErr = k.saveMemoryFiles(ctx, nil, pagesMetadata, pagesFile, mfsToSave, appMFExcludeCommittedZeroPages) // transfers ownership
+			}()
+			pagesCleanup.Release()
+			// Defer a Wait() so we wait for k.saveMemoryFiles() to complete even if we
+			// error out without reaching the other Wait() below.
+			defer mfSaveWg.Wait()
+		}
 
-	// Evict all evictable MemoryFile allocations.
-	k.mf.StartEvictions()
-	k.mf.WaitForEvictions()
+		// Save the CPUID FeatureSet before the rest of the kernel so we can
+		// verify its compatibility on restore before attempting to restore the
+		// entire kernel, which may fail on an incompatible machine.
+		//
+		// N.B. This will also be saved along with the full kernel save below.
+		cpuidStart := time.Now()
+		if _, err := state.Save(ctx, stateFile, &k.featureSet); err != nil {
+			return err
+		}
+		log.Infof("CPUID save took [%s].", time.Since(cpuidStart))
 
-	// Discard unsavable mappings, such as those for host file descriptors.
-	if err := k.invalidateUnsavableMappings(ctx); err != nil {
-		return fmt.Errorf("failed to invalidate unsavable mappings: %v", err)
-	}
+		// Save the timekeeper's state.
 
-	// Capture all private memory files.
-	mfsToSave := make(map[checkpoint.ResourceID]*pgalloc.MemoryFile)
-	vfsCtx := context.WithValue(ctx, pgalloc.CtxMemoryFileMap, mfsToSave)
-	// Prepare filesystems for saving. This must be done after
-	// invalidateUnsavableMappings(), since dropping memory mappings may
-	// affect filesystem state (e.g. page cache reference counts).
-	if err := k.vfs.PrepareSave(vfsCtx); err != nil {
-		return err
-	}
-	// Mark all to-be-saved MemoryFiles as savable to inform kernel save below.
-	k.mf.MarkSavable()
-	for _, mf := range mfsToSave {
-		mf.MarkSavable()
-	}
+		if rootNS := k.rootNetworkNamespace; rootNS != nil && rootNS.Stack() != nil {
+			// Pause the network stack.
+			netstackPauseStart := time.Now()
+			// Stack.removeConf should be true when resume=false and vice versa.
+			k.rootNetworkNamespace.Stack().SetRemoveConf(!resume)
+			log.Infof("Pausing root network namespace")
+			k.rootNetworkNamespace.Stack().Pause()
+			defer k.rootNetworkNamespace.Stack().Resume()
+			log.Infof("Pausing root network namespace took [%s].", time.Since(netstackPauseStart))
+		}
 
-	var (
-		mfSaveWg  sync.WaitGroup
-		mfSaveErr error
-	)
-	if parallelMFSave {
-		// Parallelize MemoryFile save and kernel save. Both are independent.
-		mfSaveWg.Add(1)
-		go func() {
-			defer mfSaveWg.Done()
-			mfSaveErr = k.saveMemoryFiles(ctx, nil, pagesMetadata, pagesFile, mfsToSave, appMFExcludeCommittedZeroPages) // transfers ownership
-		}()
-		pagesCleanup.Release()
-		// Defer a Wait() so we wait for k.saveMemoryFiles() to complete even if we
-		// error out without reaching the other Wait() below.
-		defer mfSaveWg.Wait()
-	}
-
-	// Save the CPUID FeatureSet before the rest of the kernel so we can
-	// verify its compatibility on restore before attempting to restore the
-	// entire kernel, which may fail on an incompatible machine.
-	//
-	// N.B. This will also be saved along with the full kernel save below.
-	cpuidStart := time.Now()
-	if _, err := state.Save(ctx, stateFile, &k.featureSet); err != nil {
-		return err
-	}
-	log.Infof("CPUID save took [%s].", time.Since(cpuidStart))
-
-	// Save the timekeeper's state.
-
-	if rootNS := k.rootNetworkNamespace; rootNS != nil && rootNS.Stack() != nil {
-		// Pause the network stack.
-		netstackPauseStart := time.Now()
-		// Stack.removeConf should be true when resume=false and vice versa.
-		k.rootNetworkNamespace.Stack().SetRemoveConf(!resume)
-		log.Infof("Pausing root network namespace")
-		k.rootNetworkNamespace.Stack().Pause()
-		defer k.rootNetworkNamespace.Stack().Resume()
-		log.Infof("Pausing root network namespace took [%s].", time.Since(netstackPauseStart))
-	}
-
-	// Save the kernel state.
-	kernelStart := time.Now()
-	stats, err := state.Save(ctx, stateFile, k)
-	if err != nil {
-		return err
-	}
-	log.Infof("Kernel save stats: %s", stats.String())
-	log.Infof("Kernel save took [%s].", time.Since(kernelStart))
-
-	if parallelMFSave {
-		// Close stateFile while MemoryFile saving is in progress to overlap
-		// their latencies.
-		err := stateFile.Close()
-		stateFileCleanup.Release()
+		// Save the kernel state.
+		kernelStart := time.Now()
+		stats, err := state.Save(ctx, stateFile, k)
 		if err != nil {
-			return fmt.Errorf("closing state file failed: %w", err)
+			return err
 		}
-		mfSaveWg.Wait()
-		if mfSaveErr != nil {
-			return mfSaveErr
-		}
-	} else {
-		mfSaveErr = k.saveMemoryFiles(ctx, stateFile, nil, nil, mfsToSave, appMFExcludeCommittedZeroPages)
-		if mfSaveErr != nil {
-			return mfSaveErr
-		}
-		// Can't close stateFile until k.saveMemoryFiles() finishes writing to
-		// it.
-		err := stateFile.Close()
-		stateFileCleanup.Release()
-		if err != nil {
-			return fmt.Errorf("closing state file failed: %w", err)
-		}
-	}
+		log.Infof("Kernel save stats: %s", stats.String())
+		log.Infof("Kernel save took [%s].", time.Since(kernelStart))
 
-	log.Infof("Overall save took [%s].", time.Since(saveStart))
-	return nil
+		if parallelMFSave {
+			// Close stateFile while MemoryFile saving is in progress to overlap
+			// their latencies.
+			err := stateFile.Close()
+			stateFileCleanup.Release()
+			if err != nil {
+				return fmt.Errorf("closing state file failed: %w", err)
+			}
+			mfSaveWg.Wait()
+			if mfSaveErr != nil {
+				return mfSaveErr
+			}
+		} else {
+			mfSaveErr = k.saveMemoryFiles(ctx, stateFile, nil, nil, mfsToSave, appMFExcludeCommittedZeroPages)
+			if mfSaveErr != nil {
+				return mfSaveErr
+			}
+			// Can't close stateFile until k.saveMemoryFiles() finishes writing to
+			// it.
+			err := stateFile.Close()
+			stateFileCleanup.Release()
+			if err != nil {
+				return fmt.Errorf("closing state file failed: %w", err)
+			}
+		}
+
+		log.Infof("Overall save took [%s].", time.Since(saveStart))
+		return nil
+	})
 }
 
 // BeforeResume is called before the kernel is resumed after save.

pkg/sentry/kernel/kernel_restore.go

@@ -34,6 +34,7 @@ import (
 type Saver interface {
 	SaveAsync() error
 	SpecEnviron(containerName string) []string
+	FSSave() error
 }
 
 // CheckpointGeneration stores information about the last checkpoint taken.
@@ -314,6 +315,8 @@ func (mfl *AsyncMFLoader) backgroundGoroutine(pagesMetadata io.ReadCloser, pages
 		PagesFile: apfl,
 		Timeline:  timeline,
 	}
+	// Note that we depend on opts.PagesFileOffset being carried between
+	// LoadFrom calls, so the same opts must be used for all calls.
 
 	timeline.Reached("loading mainMF")
 	log.Infof("Loading metadata for main MemoryFile: %p", mainMF)

pkg/sentry/pgalloc/save_restore.go

@@ -585,6 +585,12 @@ func (apfs *AsyncPagesFileSave) MemoryFilesDone() {
 	apfs.stStatus.Notify(apsSTDone)
 }
 
+// PagesFileOffset returns the offset into the pages file of the next page to
+// be written.
+func (apfs *AsyncPagesFileSave) PagesFileOffset() uint64 {
+	return apfs.saveOff
+}
+
 func (apfs *AsyncPagesFileSave) canEnqueue() bool {
 	return apfs.qavail > 0
 }
@@ -894,9 +900,14 @@ func (apfs *AsyncPagesFileSave) main() {
 
 // LoadOpts provides options to MemoryFile.LoadFrom().
 type LoadOpts struct {
-	// If PagesFile is not nil, then page contents will be read from PagesFile
-	// rather than from r.
-	PagesFile *AsyncPagesFileLoad
+	// If PagesFile is not nil, then page contents will be read from PagesFile,
+	// starting at offset PagesFileOffset, rather than from r. After LoadFrom
+	// returns, PagesFileOffset will be updated to the offset of the first byte
+	// in PagesFile after this MemoryFile's contents.
+	//
+	// Invariant: PagesFileOffset must be page-aligned.
+	PagesFile       *AsyncPagesFileLoad
+	PagesFileOffset uint64
 
 	// Optional timeline for the restore process.
 	// If async page loading is enabled, a forked timeline will be created, so
@@ -1040,10 +1051,10 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, opts *LoadOpts)
 			}
 			amfl.pf.mu.Lock()
 			amfl.unloaded.InsertRange(maFR, aplUnloadedInfo{
-				off: amfl.pf.loadOff,
+				off: opts.PagesFileOffset,
 			})
 			amfl.pf.mu.Unlock()
-			amfl.pf.loadOff += amount
+			opts.PagesFileOffset += amount
 			amfl.pf.lfStatus.Notify(aplLFPending)
 		} else {
 			// Verify header.
@@ -1092,12 +1103,6 @@ func (f *MemoryFile) LoadFrom(ctx context.Context, r io.Reader, opts *LoadOpts)
 
 // AsyncPagesFileLoad holds async page loading state for a single pages file.
 type AsyncPagesFileLoad struct {
-	// loadOff is the offset in the pages file from which the next page should
-	// be loaded. loadOff is not synchronized since it is only accessed by
-	// MemoryFile.LoadFrom(), which cannot be called concurrently using the
-	// same AsyncPagesFileLoad.
-	loadOff uint64
-
 	mu apflMutex
 
 	// If errVal is not nil, it is an error that has terminated asynchronous

pkg/sentry/state/checkpointfiles/checkpointfiles.go

@@ -16,11 +16,8 @@
 // checkpointed to multiple files in a directory rather than to an opaque FD.
 package checkpointfiles
 
+// Files common to both full and filesystem checkpoints:
 const (
-	// StateFileName is the file in an image-path directory which contains the
-	// sentry object graph.
-	StateFileName = "checkpoint.img"
-
 	// PagesMetadataFileName is the file in an image-path directory containing
 	// MemoryFile metadata.
 	PagesMetadataFileName = "pages_meta.img"
@@ -29,3 +26,17 @@ const (
 	// MemoryFile page contents.
 	PagesFileName = "pages.img"
 )
+
+// Files specific to full checkpoints:
+const (
+	// StateFileName is the file in an image-path directory which contains the
+	// sentry object graph.
+	StateFileName = "checkpoint.img"
+)
+
+// Files specific to filesystem checkpoints (see fscheckpoint package for
+// details):
+const (
+	FSCheckpointManifestFileName = "fscheckpoint.json"
+	FSCheckpointMultiTarFileName = "multitar.img"
+)

pkg/sentry/vfs/save_restore.go

@@ -65,14 +65,18 @@ type FilesystemImplSaveRestoreExtension interface {
 
 // PrepareSave prepares all filesystems for serialization.
 func (vfs *VirtualFilesystem) PrepareSave(ctx context.Context) error {
-	for fs := range vfs.getFilesystems() {
+	fss := vfs.GetFilesystems()
+	defer func() {
+		for _, fs := range fss {
+			fs.DecRef(ctx)
+		}
+	}()
+	for _, fs := range fss {
 		if ext, ok := fs.impl.(FilesystemImplSaveRestoreExtension); ok {
 			if err := ext.PrepareSave(ctx); err != nil {
-				fs.DecRef(ctx)
 				return err
 			}
 		}
-		fs.DecRef(ctx)
 	}
 	return nil
 }
@@ -80,7 +84,7 @@ func (vfs *VirtualFilesystem) PrepareSave(ctx context.Context) error {
 // BeforeResume is called before the kernel is resumed after save and allows
 // filesystems to clean up S/R state.
 func (vfs *VirtualFilesystem) BeforeResume(ctx context.Context) {
-	for fs := range vfs.getFilesystems() {
+	for _, fs := range vfs.GetFilesystems() {
 		if ext, ok := fs.impl.(FilesystemImplSaveRestoreExtension); ok {
 			ext.BeforeResume(ctx)
 		}
@@ -91,14 +95,18 @@ func (vfs *VirtualFilesystem) BeforeResume(ctx context.Context) {
 // CompleteRestore completes restoration from checkpoint for all filesystems
 // after deserialization.
 func (vfs *VirtualFilesystem) CompleteRestore(ctx context.Context, opts *CompleteRestoreOptions) error {
-	for fs := range vfs.getFilesystems() {
+	fss := vfs.GetFilesystems()
+	defer func() {
+		for _, fs := range fss {
+			fs.DecRef(ctx)
+		}
+	}()
+	for _, fs := range fss {
 		if ext, ok := fs.impl.(FilesystemImplSaveRestoreExtension); ok {
 			if err := ext.CompleteRestore(ctx, *opts); err != nil {
-				fs.DecRef(ctx)
 				return PrependErrMsg(fmt.Sprintf("failed to complete restore for filesystem type %q", fs.fsType.Name()), err)
 			}
 		}
-		fs.DecRef(ctx)
 	}
 	return nil
 }

pkg/sentry/vfs/vfs.go

@@ -848,7 +848,7 @@ func (vfs *VirtualFilesystem) RemoveXattrAt(ctx context.Context, creds *auth.Cre
 // SyncAllFilesystems has the semantics of Linux's sync(2).
 func (vfs *VirtualFilesystem) SyncAllFilesystems(ctx context.Context) error {
 	var retErr error
-	for fs := range vfs.getFilesystems() {
+	for _, fs := range vfs.GetFilesystems() {
 		if err := fs.impl.Sync(ctx); err != nil && retErr == nil {
 			retErr = err
 		}
@@ -857,15 +857,17 @@ func (vfs *VirtualFilesystem) SyncAllFilesystems(ctx context.Context) error {
 	return retErr
 }
 
-func (vfs *VirtualFilesystem) getFilesystems() map[*Filesystem]struct{} {
-	fss := make(map[*Filesystem]struct{})
+// GetFilesystems returns a slice containing all Filesystems. It takes a
+// reference on each Filesystem.
+func (vfs *VirtualFilesystem) GetFilesystems() []*Filesystem {
+	var fss []*Filesystem
 	vfs.filesystemsMu.Lock()
 	defer vfs.filesystemsMu.Unlock()
 	for fs := range vfs.filesystems {
 		if !fs.TryIncRef() {
 			continue
 		}
-		fss[fs] = struct{}{}
+		fss = append(fss, fs)
 	}
 	return fss
 }

runsc/boot/BUILD

@@ -15,6 +15,8 @@ go_library(
         "controller.go",
         "debug.go",
         "events.go",
+        "fscheckpoint.go",
+        "fscheckpoint_impl.go",
         "gofer_conf.go",
         "limits.go",
         "loader.go",
@@ -50,6 +52,7 @@ go_library(
         "//pkg/fspath",
         "//pkg/fsutil",
         "//pkg/gomaxprocs",
+        "//pkg/hostarch",
         "//pkg/log",
         "//pkg/memutil",
         "//pkg/metric",
@@ -67,6 +70,7 @@ go_library(
         "//pkg/sentry/devices/ttydev",
         "//pkg/sentry/devices/tundev",
         "//pkg/sentry/fdimport",
+        "//pkg/sentry/fscheckpoint",
         "//pkg/sentry/fsimpl/cgroupfs",
         "//pkg/sentry/fsimpl/dev",
         "//pkg/sentry/fsimpl/devpts",
@@ -103,6 +107,7 @@ go_library(
         "//pkg/sentry/socket/plugin",
         "//pkg/sentry/socket/unix",
         "//pkg/sentry/state",
+        "//pkg/sentry/state/checkpointfiles",
         "//pkg/sentry/state/stateio",
         "//pkg/sentry/strace",
         "//pkg/sentry/time",

runsc/boot/controller.go

@@ -66,6 +66,9 @@ const (
 	// ContMgrExecuteAsync executes a command in a container.
 	ContMgrExecuteAsync = "containerManager.ExecuteAsync"
 
+	// ContMgrFSSave saves a filesystem checkpoint.
+	ContMgrFSSave = "containerManager.FSSave"
+
 	// ContMgrGetSavings gets the savings for restored sandboxes.
 	ContMgrGetSavings = "containerManager.GetSavings"
 
@@ -396,6 +399,9 @@ func (cm *containerManager) StartSubcontainer(args *StartArgs, _ *struct{}) erro
 		expectedFDs += 3
 	}
 	if args.IsRootfsUpperTarFilePresent {
+		if cm.l.fsRestore != nil {
+			return fmt.Errorf("rootfs upper tar file is mutually exclusive with filesystem checkpoint restore")
+		}
 		expectedFDs++
 	}
 	if len(args.Files) < expectedFDs {
@@ -565,6 +571,9 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) (retErr error)
 			cm.onRestoreFailed(fmt.Errorf("Restore failed: %w", retErr))
 		}
 	}()
+	if cm.l.fsRestore != nil {
+		return fmt.Errorf("cannot restore sandbox when filesystem restore is enabled")
+	}
 	if len(o.Files) == 0 {
 		return fmt.Errorf("at least one file must be passed to Restore")
 	}
@@ -1075,6 +1084,31 @@ func (cm *containerManager) ContainerRuntimeState(cid *string, state *ContainerR
 	return nil
 }
 
+// FSSaveArgs holds arguments to FSSave.
+type FSSaveArgs struct {
+	// FilePayload contains the following fscheckpoint files in order:
+	// 1. manifest file
+	// 2. multi-tar file
+	// 3. pages metadata file
+	// 4. pages file
+	urpc.FilePayload
+
+	// Equivalent to kernel.FSSaveOpts fields.
+	ExitAfterSaving bool
+
+	FSSaveArgsExtra
+}
+
+// FSSave collects a filesystem checkpoint.
+func (cm *containerManager) FSSave(args *FSSaveArgs, _ *struct{}) error {
+	log.Debugf("containerManager.FSSave")
+	kopts, err := convertToKernelFSSaveOpts(args)
+	if err != nil {
+		return err
+	}
+	return cm.l.k.FSSave(cm.l.k.SupervisorContext(), &kopts)
+}
+
 // Savings holds the savings with restore.
 type Savings struct {
 	// CPUTimeSaved is the CPU time saved at restore.

runsc/boot/fscheckpoint.go

@@ -0,0 +1,270 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package boot
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"gvisor.dev/gvisor/pkg/hostarch"
+	"gvisor.dev/gvisor/pkg/log"
+	"gvisor.dev/gvisor/pkg/sentry/checkpoint"
+	"gvisor.dev/gvisor/pkg/sentry/fscheckpoint"
+	"gvisor.dev/gvisor/pkg/sentry/kernel"
+	"gvisor.dev/gvisor/pkg/sentry/pgalloc"
+	"gvisor.dev/gvisor/pkg/sentry/state/checkpointfiles"
+	"gvisor.dev/gvisor/pkg/sentry/state/stateio"
+	"gvisor.dev/gvisor/pkg/sync"
+	"gvisor.dev/gvisor/runsc/version"
+)
+
+func convertToKernelFSSaveOpts(args *FSSaveArgs) (kernel.FSSaveOpts, error) {
+	opts := kernel.FSSaveOpts{
+		RunscVersion:    version.Version(),
+		ExitAfterSaving: args.ExitAfterSaving,
+	}
+	if err := setKernelFSSaveOptsFilesImpl(args, &opts); err != nil {
+		return kernel.FSSaveOpts{}, err
+	}
+	return opts, nil
+}
+
+func setKernelFSSaveOptsFilesForLocalCheckpoint(args *FSSaveArgs, opts *kernel.FSSaveOpts) error {
+	if len(args.FilePayload.Files) != 4 {
+		return fmt.Errorf("got %d files, want 4", len(args.FilePayload.Files))
+	}
+	manifestFile, err := args.ReleaseFD(0)
+	if err != nil {
+		return err
+	}
+	multiTarFile, err := args.ReleaseFD(1)
+	if err != nil {
+		return err
+	}
+	pagesMetadataFile, err := args.ReleaseFD(2)
+	if err != nil {
+		return err
+	}
+	pagesFile, err := args.ReleaseFD(3)
+	if err != nil {
+		return err
+	}
+	opts.ManifestFile = stateio.NewBufioWriteCloser(manifestFile)
+	opts.MultiTarFile = stateio.NewBufioWriteCloser(multiTarFile)
+	opts.PagesMetadataFile = stateio.NewBufioWriteCloser(pagesMetadataFile)
+	opts.PagesFile = stateio.NewPagesFileFDWriterDefault(int32(pagesFile.Release()))
+	return nil
+}
+
+// fsRestore holds the state of a filesystem restore.
+type fsRestore struct {
+	wg sync.WaitGroup
+
+	// immutable
+	getPagesMetadata func() ([]byte, error)
+	getMultiTar      func() ([]byte, error)
+
+	// immutable after wg.Wait()
+	manifestErr error
+	apfl        *pgalloc.AsyncPagesFileLoad
+	mfs         map[checkpoint.ResourceID]*fscheckpoint.MemoryFile
+	tmpfs       map[checkpoint.ResourceID]*fscheckpoint.Tmpfs
+}
+
+// fsRestoreOpts holds options to startFSRestore.
+type fsRestoreOpts struct {
+	// These correspond to files specified by the fscheckpoint package, and are
+	// all required.
+	ManifestFile      io.ReadCloser
+	MultiTarFile      io.ReadCloser
+	PagesMetadataFile io.ReadCloser
+	PagesFile         stateio.AsyncReader
+}
+
+func makeFSRestoreOptsForLocalCheckpoint(args *Args) (fsRestoreOpts, error) {
+	if len(args.FSRestoreFDs) != 4 {
+		return fsRestoreOpts{}, fmt.Errorf("got %d files in -fs-restore-fds, want 4", len(args.FSRestoreFDs))
+	}
+	return fsRestoreOpts{
+		ManifestFile:      stateio.NewBufioReadCloser(args.FSRestoreFDs[0].ReleaseToFile(checkpointfiles.FSCheckpointManifestFileName)),
+		MultiTarFile:      stateio.NewBufioReadCloser(args.FSRestoreFDs[1].ReleaseToFile(checkpointfiles.FSCheckpointMultiTarFileName)),
+		PagesMetadataFile: stateio.NewBufioReadCloser(args.FSRestoreFDs[2].ReleaseToFile(checkpointfiles.PagesMetadataFileName)),
+		PagesFile:         stateio.NewPagesFileFDReaderDefault(int32(args.FSRestoreFDs[3].Release())),
+	}, nil
+}
+
+// startFSRestore takes ownership of resources in opts.
+func startFSRestore(opts *fsRestoreOpts) (*fsRestore, error) {
+	fsr := &fsRestore{
+		mfs:   make(map[checkpoint.ResourceID]*fscheckpoint.MemoryFile),
+		tmpfs: make(map[checkpoint.ResourceID]*fscheckpoint.Tmpfs),
+	}
+
+	// TODO: NOLINT - Currently we read the whole pages metadata file into a
+	// []byte, then pass pieces of that []byte to MemoryFile construction. This
+	// is necessary because opts.PagesMetadataFile is io.Reader (read
+	// sequentially), and tmpfs filesystems and their private MemoryFiles may
+	// be restored in a different order than checkpoint order (disk-backed
+	// filestore files are not available until container creation).
+	//
+	// We could make opts.PagesMetadataFile io.ReaderAt to avoid this copy.
+	// However, when the multi-tar file is accessed via stateio.AsyncReader,
+	// this requires an implementation of io.ReaderAt that wraps
+	// stateio.AsyncReader, akin to stateio.BufReader. AsyncReader already
+	// supports random reads, but has a fixed maximum parallelism per
+	// AsyncReader that would need to be shared between readers. Furthermore,
+	// BufReader asynchronously fills its buffer with reads to minimize
+	// latency; our io.ReaderAt implementation would need to do something
+	// comparable to avoid regressions.
+	//
+	// Alternatively, we could implement io.ReaderAt by asynchronously
+	// buffering the whole file in memory, which is probably better overall
+	// (equivalent to what we are doing now, but permits reading parts of the
+	// file that have been read before the whole file is read) but requires
+	// adding a stateio.AsyncReader method to get file size.
+	//
+	// All of the above also applies to the multi-tar file.
+	readOnce := func(desc string, optsR *io.ReadCloser) func() ([]byte, error) {
+		r := *optsR
+		*optsR = nil
+		f := sync.OnceValues(func() ([]byte, error) {
+			timeStart := time.Now()
+			data, err := io.ReadAll(r)
+			dur := time.Since(timeStart)
+			// Close r immediately to release any memory used for buffering.
+			r.Close()
+			r = nil
+			if err == nil {
+				log.Infof("Read filesystem checkpoint %s (%d bytes) in %s", desc, len(data), dur)
+			}
+			return data, err
+		})
+		// Start reading immediately.
+		go f()
+		return f
+	}
+	fsr.getPagesMetadata = readOnce("pages metadata file", &opts.PagesMetadataFile)
+	fsr.getMultiTar = readOnce("multi-tar file", &opts.MultiTarFile)
+
+	// Read and handle the manifest in parallel.
+	fsr.manifestErr = fmt.Errorf("loading manifest panicked")
+	fsr.wg.Go(func() {
+		defer func() {
+			if opts.ManifestFile != nil {
+				opts.ManifestFile.Close()
+				opts.ManifestFile = nil
+			}
+			if opts.PagesFile != nil {
+				opts.PagesFile.Close()
+				opts.PagesFile = nil
+			}
+		}()
+		fsr.manifestErr = func() error {
+			var manifest fscheckpoint.Manifest
+			timeStart := time.Now()
+			if err := json.NewDecoder(opts.ManifestFile).Decode(&manifest); err != nil {
+				return fmt.Errorf("failed to read manifest: %w", err)
+			}
+			log.Infof("Read filesystem checkpoint manifest in %s", time.Since(timeStart))
+			if manifest.Version != 0 {
+				return fmt.Errorf("unsupported filesystem checkpoint version: %d", manifest.Version)
+			}
+			if manifest.RunscVersion != version.Version() {
+				return fmt.Errorf("filesystem checkpoint runsc version %q does not match current runsc version %q", manifest.RunscVersion, version.Version())
+			}
+			if manifest.PageSize != hostarch.PageSize {
+				return fmt.Errorf("filesystem checkpoint page size %d does not match current page size %d", manifest.PageSize, hostarch.PageSize)
+			}
+			if endian := hostarch.EndianString(); manifest.Endian != endian {
+				return fmt.Errorf("filesystem checkpoint endianness %q does not match current endianness %q", manifest.Endian, endian)
+			}
+
+			apfl, err := pgalloc.StartAsyncPagesFileLoad(opts.PagesFile /* transfers ownership */, func(err error) {
+				if err != nil {
+					log.Warningf("Failed to load filesystem checkpoint pages: %v", err)
+				} else if log.IsLogging(log.Debug) {
+					log.Debugf("Finished loading filesystem checkpoint pages")
+				}
+			}, nil)
+			opts.PagesFile = nil
+			if err != nil {
+				return fmt.Errorf("failed to start async page loading: %w", err)
+			}
+			fsr.apfl = apfl
+			for i := range manifest.MemoryFiles {
+				mmf := &manifest.MemoryFiles[i]
+				fsr.mfs[mmf.ResourceID] = mmf
+			}
+			for i := range manifest.Tmpfs {
+				mt := &manifest.Tmpfs[i]
+				fsr.tmpfs[mt.ResourceID] = mt
+			}
+			// Note that apfl.MemoryFilesDone() will never be called. This is
+			// because if a container fails and is restarted, we also want to
+			// restore the filesystem of the restarted container, so we need to
+			// keep filesystem restore operational indefinitely.
+			return nil
+		}()
+	})
+
+	return fsr, nil
+}
+
+func (fsr *fsRestore) memoryFileLoadArgs(id checkpoint.ResourceID) (io.Reader, uint64, error) {
+	if fsr == nil {
+		return nil, 0, nil
+	}
+	fsr.wg.Wait()
+	if fsr.manifestErr != nil {
+		return nil, 0, fsr.manifestErr
+	}
+	mmf := fsr.mfs[id]
+	if mmf == nil {
+		return nil, 0, nil
+	}
+	pagesMetadata, err := fsr.getPagesMetadata()
+	if mmf.PagesMetadataEnd <= uint64(len(pagesMetadata)) {
+		return bytes.NewReader(pagesMetadata[mmf.PagesMetadataStart:mmf.PagesMetadataEnd]), mmf.PagesStart, nil
+	}
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to read pages metadata: %w", err)
+	}
+	return nil, 0, fmt.Errorf("MemoryFile %q has pages metadata range [%d, %d) beyond pages metadata file size %d", mmf.ResourceID, mmf.PagesMetadataStart, mmf.PagesMetadataEnd, len(pagesMetadata))
+}
+
+func (fsr *fsRestore) tmpfsSourceTar(id checkpoint.ResourceID) (io.ReadCloser, error) {
+	if fsr == nil {
+		return nil, nil
+	}
+	fsr.wg.Wait()
+	if fsr.manifestErr != nil {
+		return nil, fsr.manifestErr
+	}
+	mt := fsr.tmpfs[id]
+	if mt == nil {
+		return nil, nil
+	}
+	multiTar, err := fsr.getMultiTar()
+	if mt.TarEnd <= uint64(len(multiTar)) {
+		return io.NopCloser(bytes.NewReader(multiTar[mt.TarStart:mt.TarEnd])), nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read tar archive: %w", err)
+	}
+	return nil, fmt.Errorf("tmpfs %q has tar range [%d, %d) beyond multi-tar file size %d", mt.ResourceID, mt.TarStart, mt.TarEnd, len(multiTar))
+}

runsc/boot/fscheckpoint_impl.go

@@ -0,0 +1,32 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !false
+// +build !false
+
+package boot
+
+import (
+	"gvisor.dev/gvisor/pkg/sentry/kernel"
+)
+
+type FSSaveArgsExtra struct{}
+
+func setKernelFSSaveOptsFilesImpl(args *FSSaveArgs, opts *kernel.FSSaveOpts) error {
+	return setKernelFSSaveOptsFilesForLocalCheckpoint(args, opts)
+}
+
+func makeFSRestoreOptsImpl(args *Args) (fsRestoreOpts, error) {
+	return makeFSRestoreOptsForLocalCheckpoint(args)
+}

runsc/boot/loader.go

@@ -309,6 +309,8 @@ type Loader struct {
 	// +checklocks:mu
 	savings Savings
 
+	fsRestore *fsRestore
+
 	LoaderExtra
 }
 
@@ -413,7 +415,8 @@ type Args struct {
 	// HostTHP contains host transparent hugepage settings.
 	HostTHP HostTHP
 
-	SaveFDs []*fd.FD
+	SaveFDs      []*fd.FD
+	FSRestoreFDs []*fd.FD
 
 	ArgsExtra
 
@@ -519,6 +522,20 @@ func New(args Args) (*Loader, error) {
 	log.Infof("CPUs: %d", args.NumCPU)
 	gomaxprocs.SetBase(args.NumCPU)
 
+	// Start filesystem checkpoint restore as soon as possible to maximize
+	// parallel loading.
+	if len(args.FSRestoreFDs) != 0 {
+		fsrOpts, err := makeFSRestoreOptsImpl(&args)
+		if err != nil {
+			return nil, fmt.Errorf("failed to set up filesystem checkpoint restore: %w", err)
+		}
+		fsr, err := startFSRestore(&fsrOpts)
+		if err != nil {
+			return nil, fmt.Errorf("failed to start filesystem checkpoint restore: %w", err)
+		}
+		l.fsRestore = fsr
+	}
+
 	containerName := l.registerContainer(args.Spec, args.ID)
 	l.root = containerInfo{
 		cid:                args.ID,
@@ -573,6 +590,9 @@ func New(args Args) (*Loader, error) {
 	}
 
 	if args.RootfsUpperTarFD >= 0 {
+		if l.fsRestore != nil {
+			return nil, fmt.Errorf("rootfs upper tar file is mutually exclusive with filesystem checkpoint restore")
+		}
 		l.root.rootfsUpperTarFD = fd.New(args.RootfsUpperTarFD)
 	}
 
@@ -1315,7 +1335,7 @@ func (l *Loader) createContainerProcess(info *containerInfo) (*kernel.ThreadGrou
 	}
 	// We can share l.sharedMounts with containerMounter since l.mu is locked.
 	// Hence, mntr must only be used within this function (while l.mu is locked).
-	mntr := newContainerMounter(info, l.k, l.mountHints, l.sharedMounts, l.productName, l.sandboxID)
+	mntr := l.newContainerMounter(info)
 	if err := setupContainerVFS(ctx, info, mntr, &info.procArgs); err != nil {
 		return nil, nil, err
 	}

runsc/boot/loader_test.go

@@ -478,7 +478,7 @@ func TestCreateMountNamespace(t *testing.T) {
 
 			l.mu.Lock()
 			defer l.mu.Unlock()
-			mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, "", l.sandboxID)
+			mntr := l.newContainerMounter(&l.root)
 			ctx := l.k.SupervisorContext()
 			creds := auth.NewRootCredentials(l.root.procArgs.Credentials.UserNamespace)
 			mns, err := mntr.mountAll(ctx, creds, l.root.spec, l.root.conf, &l.root.procArgs)
@@ -577,7 +577,7 @@ func TestCreateMountPoint(t *testing.T) {
 
 	l.mu.Lock()
 	defer l.mu.Unlock()
-	mntr := newContainerMounter(&l.root, l.k, l.mountHints, l.sharedMounts, "", l.sandboxID)
+	mntr := l.newContainerMounter(&l.root)
 	ctx := l.k.SupervisorContext()
 	creds := auth.NewRootCredentials(l.root.procArgs.Credentials.UserNamespace)
 	mns, err := mntr.mountAll(ctx, creds, l.root.spec, l.root.conf, &l.root.procArgs)

runsc/boot/restore.go

@@ -246,7 +246,7 @@ func (r *restorer) restore(l *Loader) error {
 	mfmap := make(map[checkpoint.ResourceID]*pgalloc.MemoryFile)
 	for _, cont := range r.containers {
 		// TODO(b/298078576): Need to process hints here probably
-		mntr := newContainerMounter(cont, l.k, l.mountHints, l.sharedMounts, l.productName, cont.cid)
+		mntr := l.newContainerMounter(cont)
 		if err = mntr.configureRestore(fdmap, mfmap); err != nil {
 			return fmt.Errorf("configuring filesystem restore: %v", err)
 		}

runsc/boot/vfs.go

@@ -170,7 +170,7 @@ func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *container
 	rootProcArgs.Credentials = rootCreds
 	rootProcArgs.Umask = 0022
 	rootProcArgs.MaxSymlinkTraversals = linux.MaxSymlinkTraversals
-	rootCtx := rootProcArgs.NewContext(mntr.k)
+	rootCtx := rootProcArgs.NewContext(mntr.l.k)
 
 	mns, err := mntr.mountAll(rootCtx, rootCreds, info.spec, info.conf, &rootProcArgs)
 	if err != nil {
@@ -181,7 +181,7 @@ func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *container
 	// If cgroups are mounted, then only check for the cgroup mounts per
 	// container. Otherwise the root cgroups will be enabled.
 	if mntr.cgroupsMounted {
-		cgroupRegistry := mntr.k.CgroupRegistry()
+		cgroupRegistry := mntr.l.k.CgroupRegistry()
 		for _, ctrl := range kernel.CgroupCtrls {
 			cg, err := cgroupRegistry.FindCgroup(ctx, ctrl, "/"+mntr.containerID)
 			if err != nil {
@@ -197,11 +197,11 @@ func setupContainerVFS(ctx context.Context, info *containerInfo, mntr *container
 	mnsRoot := mns.Root(rootCtx)
 	defer mnsRoot.DecRef(rootCtx)
 
-	if err := createDeviceFiles(rootCtx, rootCreds, info, mntr.k.VFS(), mnsRoot); err != nil {
+	if err := createDeviceFiles(rootCtx, rootCreds, info, mntr.l.k.VFS(), mnsRoot); err != nil {
 		return fmt.Errorf("failed to create device files: %w", err)
 	}
 
-	if err := mntr.k.VFS().MkdirAllAt(
+	if err := mntr.l.k.VFS().MkdirAllAt(
 		ctx, procArgs.WorkingDirectory, mnsRoot, rootCreds,
 		&vfs.MkdirOptions{Mode: 0755}, true, /* mustBeDir */
 	); err != nil {
@@ -361,6 +361,8 @@ func (f *fdDispenser) empty() bool {
 }
 
 type containerMounter struct {
+	l *Loader
+
 	root *specs.Root
 
 	// mounts is the set of submounts for the container. It's a copy from the spec
@@ -382,24 +384,15 @@ type containerMounter struct {
 	// for bind mounts in Spec.Mounts (in the same order).
 	goferMountConfs []GoferMountConf
 
-	k *kernel.Kernel
-
-	// hints is the set of pod mount hints for the sandbox.
-	hints *PodMountHints
-
 	// sharedMounts is a map of shared mounts that can be reused across
 	// containers.
+	//
+	// FIXME: This is equivalent to l.sharedMounts, but lacks the checklocks
+	// annotation.
 	sharedMounts map[string]*vfs.Mount
 
-	// productName is the value to show in
-	// /sys/devices/virtual/dmi/id/product_name.
-	productName string
-
 	// containerID is the ID for the container.
-	containerID string
-
-	// sandboxID is the ID for the whole sandbox.
-	sandboxID     string
+	containerID   string
 	containerName string
 
 	// cgroupsMounted indicates if cgroups are mounted in the container.
@@ -412,20 +405,18 @@ type containerMounter struct {
 	rootfsUpperTarFD *fd.FD
 }
 
-func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *PodMountHints, sharedMounts map[string]*vfs.Mount, productName string, sandboxID string) *containerMounter {
+// +checklocks:l.mu
+func (l *Loader) newContainerMounter(info *containerInfo) *containerMounter {
 	return &containerMounter{
+		l:                 l,
 		root:              info.spec.Root,
 		mounts:            compileMounts(info.spec, info.conf, info.procArgs.ContainerID),
 		goferFDs:          fdDispenser{fds: info.goferFDs},
 		goferFilestoreFDs: fdDispenser{fds: info.goferFilestoreFDs},
 		devGoferFD:        info.devGoferFD,
 		goferMountConfs:   info.goferMountConfs,
-		k:                 k,
-		hints:             hints,
-		sharedMounts:      sharedMounts,
-		productName:       productName,
+		sharedMounts:      l.sharedMounts,
 		containerID:       info.cid,
-		sandboxID:         sandboxID,
 		containerName:     info.containerName,
 		rootfsUpperTarFD:  info.rootfsUpperTarFD,
 	}
@@ -464,12 +455,12 @@ func (c *containerMounter) mountAll(rootCtx context.Context, rootCreds *auth.Cre
 	defer root.DecRef(rootCtx)
 	if root.Mount().ReadOnly() {
 		// Switch to ReadWrite while we setup submounts.
-		if err := c.k.VFS().SetMountReadOnly(root.Mount(), false); err != nil {
+		if err := c.l.k.VFS().SetMountReadOnly(root.Mount(), false); err != nil {
 			return nil, fmt.Errorf(`failed to set mount at "/" readwrite: %w`, err)
 		}
 		// Restore back to ReadOnly at the end.
 		defer func() {
-			if err := c.k.VFS().SetMountReadOnly(root.Mount(), true); err != nil {
+			if err := c.l.k.VFS().SetMountReadOnly(root.Mount(), true); err != nil {
 				panic(fmt.Sprintf(`failed to restore mount at "/" back to readonly: %v`, err))
 			}
 		}()
@@ -567,14 +558,14 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 	// The namespace root mount can't be changed, so let's mount a dummy
 	// read-only tmpfs here. It simplifies creation of containers without
 	// leaking the root file system.
-	mns, err := c.k.VFS().NewMountNamespace(ctx, creds, "rootfs", "tmpfs",
-		&vfs.MountOptions{ReadOnly: true, Locked: true}, c.k)
+	mns, err := c.l.k.VFS().NewMountNamespace(ctx, creds, "rootfs", "tmpfs",
+		&vfs.MountOptions{ReadOnly: true, Locked: true}, c.l.k)
 	if err != nil {
 		return nil, fmt.Errorf("setting up mount namespace: %w", err)
 	}
 	defer mns.DecRef(ctx)
 
-	mnt, err := c.k.VFS().MountDisconnected(ctx, creds, "root", fsName, opts)
+	mnt, err := c.l.k.VFS().MountDisconnected(ctx, creds, "root", fsName, opts)
 	if err != nil {
 		return nil, fmt.Errorf("creating root file system: %w", err)
 	}
@@ -585,7 +576,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
 		Root:  root,
 		Start: root,
 	}
-	if err := c.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
+	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
 		return nil, fmt.Errorf("mounting root file system: %w", err)
 	}
 
@@ -613,7 +604,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 
 	// All writes go to the upper layer, be paranoid and make lower readonly.
 	lowerOpts.ReadOnly = true
-	lower, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, lowerFSName, lowerOpts)
+	lower, err := c.l.k.VFS().MountDisconnected(ctx, creds, "" /* source */, lowerFSName, lowerOpts)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -622,7 +613,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 
 	// Determine the lower layer's root's type.
 	lowerRootVD := vfs.MakeVirtualDentry(lower, lower.Root())
-	stat, err := c.k.VFS().StatAt(ctx, creds, &vfs.PathOperation{
+	stat, err := c.l.k.VFS().StatAt(ctx, creds, &vfs.PathOperation{
 		Root:  lowerRootVD,
 		Start: lowerRootVD,
 	}, &vfs.StatOptions{
@@ -648,19 +639,30 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 	}
 	if filestoreFD != nil {
 		// Create memory file for disk-backed overlays.
-		mf, err := createPrivateMemoryFile(filestoreFD.ReleaseToFile("overlay-filestore"), checkpoint.ResourceID{ContainerName: c.containerName, Path: dst})
+		resourceID := checkpoint.ResourceID{ContainerName: c.containerName, Path: dst}
+		mf, err := createPrivateMemoryFile(filestoreFD.ReleaseToFile("overlay-filestore"), resourceID, c.l.fsRestore)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create memory file for overlay: %v", err)
 		}
 		tmpfsOpts.MemoryFile = mf
+		sourceTar, err := c.l.fsRestore.tmpfsSourceTar(resourceID)
+		if err != nil {
+			mf.Destroy()
+			return nil, nil, fmt.Errorf("failed to get tar archive from filesystem checkpoint: %w", err)
+		}
+		if sourceTar != nil {
+			log.Infof("Loading filesystem checkpoint tree for %q", resourceID)
+			tmpfsOpts.SourceTar = sourceTar
+			tmpfsOpts.SourceTarFSCheckpoint = true
+		}
 	}
 	// If the rootfs upper tar file is provided, it will be applied to the
 	// tmpfs which is on the upper layer of the root's overlay fs.
 	if rootfsUpperTarFD != nil {
-		tmpfsOpts.SourceTarFile = rootfsUpperTarFD.ReleaseToFile("rootfs-upper-tar-fd")
+		tmpfsOpts.SourceTar = rootfsUpperTarFD.ReleaseToFile("rootfs-upper-tar-fd")
 	}
 	upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts
-	upper, err := c.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts)
+	upper, err := c.l.k.VFS().MountDisconnected(ctx, creds, "" /* source */, tmpfs.Name, &upperOpts)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to create upper layer for overlay, opts: %+v: %v", upperOpts, err)
 	}
@@ -671,7 +673,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 	// layer file will take precedence.
 	upperRootVD := vfs.MakeVirtualDentry(upper, upper.Root())
 	if rootType == linux.S_IFREG {
-		lowerFD, err := c.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
+		lowerFD, err := c.l.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
 			Root:  lowerRootVD,
 			Start: lowerRootVD,
 		}, &vfs.OpenOptions{
@@ -681,7 +683,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 			return nil, nil, fmt.Errorf("failed to open lower layer root for copying: %v", err)
 		}
 		defer lowerFD.DecRef(ctx)
-		upperFD, err := c.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
+		upperFD, err := c.l.k.VFS().OpenAt(ctx, creds, &vfs.PathOperation{
 			Root:  upperRootVD,
 			Start: upperRootVD,
 		}, &vfs.OpenOptions{
@@ -698,10 +700,10 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 
 	// We need to hide the filestore from the containerized application.
 	if mountConf.IsSelfBacked() {
-		if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
+		if err := overlay.CreateWhiteout(ctx, c.l.k.VFS(), creds, &vfs.PathOperation{
 			Root:  upperRootVD,
 			Start: upperRootVD,
-			Path:  fspath.Parse(selfFilestoreName(c.sandboxID)),
+			Path:  fspath.Parse(selfFilestoreName(c.l.sandboxID)),
 		}); err != nil {
 			return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err)
 		}
@@ -709,7 +711,7 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
 
 	// Propagate the lower layer's root's owner, group, and mode to the upper
 	// layer's root for consistency with VFS1.
-	err = c.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{
+	err = c.l.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{
 		Root:  upperRootVD,
 		Start: upperRootVD,
 	}, &vfs.SetStatOptions{
@@ -769,12 +771,12 @@ func (c *containerMounter) mountSubmounts(ctx context.Context, spec *specs.Spec,
 
 		if mnt != nil && mnt.ReadOnly() {
 			// Switch to ReadWrite while we setup submounts.
-			if err := c.k.VFS().SetMountReadOnly(mnt, false); err != nil {
+			if err := c.l.k.VFS().SetMountReadOnly(mnt, false); err != nil {
 				return fmt.Errorf("failed to set mount at %q readwrite: %w", submount.mount.Destination, err)
 			}
 			// Restore back to ReadOnly at the end.
 			defer func() {
-				if err := c.k.VFS().SetMountReadOnly(mnt, true); err != nil {
+				if err := c.l.k.VFS().SetMountReadOnly(mnt, true); err != nil {
 					panic(fmt.Sprintf("failed to restore mount at %q back to readonly: %v", submount.mount.Destination, err))
 				}
 			}()
@@ -798,7 +800,7 @@ type mountInfo struct {
 func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 	// If device gofer exists, connect to it.
 	if c.devGoferFD != nil {
-		if err := c.k.AddDevGofer(c.containerName, c.devGoferFD.Release()); err != nil {
+		if err := c.l.k.AddDevGofer(c.containerName, c.devGoferFD.Release()); err != nil {
 			return nil, err
 		}
 	}
@@ -810,7 +812,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 	for i := range c.mounts {
 		info := mountInfo{
 			mount: &c.mounts[i],
-			hint:  c.hints.FindMount(c.mounts[i].Source),
+			hint:  c.l.mountHints.FindMount(c.mounts[i].Source),
 		}
 		specutils.MaybeConvertToBindMount(info.mount)
 		if specutils.HasMountConfig(*info.mount) {
@@ -842,7 +844,7 @@ func (c *containerMounter) prepareMounts() ([]mountInfo, error) {
 
 // getPathMode returns file mode for the specified path.
 func (c *containerMounter) getPathMode(ctx context.Context, creds *auth.Credentials, path *vfs.PathOperation) (linux.FileMode, error) {
-	stat, err := c.k.VFS().StatAt(ctx, creds, path, &vfs.StatOptions{
+	stat, err := c.l.k.VFS().StatAt(ctx, creds, path, &vfs.StatOptions{
 		Mask: linux.STATX_TYPE,
 	})
 	if err != nil {
@@ -855,7 +857,7 @@ func (c *containerMounter) getPathMode(ctx context.Context, creds *auth.Credenti
 }
 
 func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec, conf *config.Config, mns *vfs.MountNamespace, creds *auth.Credentials, submount *mountInfo) (*vfs.Mount, error) {
-	fsName, opts, err := getMountNameAndOptions(spec, conf, submount, c.productName, c.containerName)
+	fsName, opts, err := getMountNameAndOptions(spec, conf, submount, c.l.productName, c.containerName, c.l.fsRestore)
 	if err != nil {
 		return nil, fmt.Errorf("mountOptions failed: %w", err)
 	}
@@ -883,7 +885,7 @@ func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec,
 		Start: root,
 		Path:  fspath.Parse(mount.Destination),
 	}
-	mnt, err := c.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
+	mnt, err := c.l.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
 	if err != nil {
 		return nil, fmt.Errorf("mounting %q (type:%s, dest: %q): %w, opts: %v",
 			mount.Source, mount.Type, mount.Destination, err, opts)
@@ -906,7 +908,7 @@ func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec,
 
 	// Avoid mounting on top of symlinks. The mount syscall on Linux always follows symlinks.
 	target.FollowFinalSymlink = true
-	if err := c.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
+	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, mnt, target); err != nil {
 		return nil, fmt.Errorf("attaching %q to %q (type: %s): %w, opts: %v",
 			mount.Source, mount.Destination, mount.Type, err, opts)
 	}
@@ -918,7 +920,7 @@ func (c *containerMounter) mountSubmount(ctx context.Context, spec *specs.Spec,
 
 // getMountNameAndOptions retrieves the fsName, opts, and useOverlay values
 // used for mounts.
-func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, productName, containerName string) (string, *vfs.MountOptions, error) {
+func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo, productName, containerName string, fsr *fsRestore) (string, *vfs.MountOptions, error) {
 	fsName := m.mount.Type
 	var (
 		mopts        = m.mount.Options
@@ -951,16 +953,27 @@ func getMountNameAndOptions(spec *specs.Spec, conf *config.Config, m *mountInfo,
 			return "", nil, err
 		}
 		if m.filestoreFD != nil {
-			mf, err := createPrivateMemoryFile(m.filestoreFD.ReleaseToFile("tmpfs-filestore"), checkpoint.ResourceID{ContainerName: containerName, Path: m.mount.Destination})
+			resourceID := checkpoint.ResourceID{ContainerName: containerName, Path: m.mount.Destination}
+			mf, err := createPrivateMemoryFile(m.filestoreFD.ReleaseToFile("tmpfs-filestore"), resourceID, fsr)
 			if err != nil {
-				return "", nil, fmt.Errorf("failed to create memory file for tmpfs: %v", err)
+				return "", nil, fmt.Errorf("failed to create memory file for tmpfs: %w", err)
 			}
-			internalData = tmpfs.FilesystemOpts{
+			tmpfsOpts := tmpfs.FilesystemOpts{
 				MemoryFile: mf,
 				// If a mount is being overlaid with tmpfs, it should not be limited by
 				// the default tmpfs size limit.
 				DisableDefaultSizeLimit: true,
 			}
+			sourceTar, err := fsr.tmpfsSourceTar(resourceID)
+			if err != nil {
+				return "", nil, fmt.Errorf("failed to get tar archive from filesystem checkpoint: %w", err)
+			}
+			if sourceTar != nil {
+				log.Infof("Loading filesystem checkpoint tree for %q", resourceID)
+				tmpfsOpts.SourceTar = sourceTar
+				tmpfsOpts.SourceTarFSCheckpoint = true
+			}
+			internalData = tmpfsOpts
 		}
 
 	case Bind:
@@ -1053,7 +1066,11 @@ func parseKeyValue(s string) (string, string, bool) {
 	return strings.TrimSpace(tokens[0]), strings.TrimSpace(tokens[1]), true
 }
 
-func createPrivateMemoryFile(file *os.File, resourceID checkpoint.ResourceID) (*pgalloc.MemoryFile, error) {
+func createPrivateMemoryFile(file *os.File, resourceID checkpoint.ResourceID, fsr *fsRestore) (*pgalloc.MemoryFile, error) {
+	pagesMetadataReader, pagesFileOffset, err := fsr.memoryFileLoadArgs(resourceID)
+	if err != nil {
+		return nil, err
+	}
 	mfOpts := pgalloc.MemoryFileOpts{
 		// Private memory files are usually backed by files on disk. Ideally we
 		// would confirm with fstatfs(2) but that is prohibited by seccomp.
@@ -1067,7 +1084,21 @@ func createPrivateMemoryFile(file *os.File, resourceID checkpoint.ResourceID) (*
 		// Private memory files need to be restored correctly using this ID.
 		ResourceID: resourceID,
 	}
-	return pgalloc.NewMemoryFile(file, mfOpts)
+	mf, err := pgalloc.NewMemoryFile(file, mfOpts)
+	if err != nil {
+		return mf, err
+	}
+	if pagesMetadataReader != nil {
+		log.Infof("Loading filesystem checkpoint data for %q", resourceID)
+		if err := mf.LoadFrom(context.Background(), pagesMetadataReader, &pgalloc.LoadOpts{
+			PagesFile:       fsr.apfl,
+			PagesFileOffset: pagesFileOffset,
+		}); err != nil {
+			mf.Destroy()
+			return nil, err
+		}
+	}
+	return mf, nil
 }
 
 // mountTmp mounts an internal tmpfs at '/tmp' if it's safe to do so.
@@ -1095,7 +1126,7 @@ func (c *containerMounter) mountTmp(ctx context.Context, spec *specs.Spec, conf
 		Start: root,
 		Path:  fspath.Parse("/tmp"),
 	}
-	fd, err := c.k.VFS().OpenAt(ctx, creds, &pop, &vfs.OpenOptions{Flags: linux.O_RDONLY | linux.O_DIRECTORY})
+	fd, err := c.l.k.VFS().OpenAt(ctx, creds, &pop, &vfs.OpenOptions{Flags: linux.O_RDONLY | linux.O_DIRECTORY})
 	switch {
 	case err == nil:
 		defer fd.DecRef(ctx)
@@ -1210,12 +1241,12 @@ func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs
 	}
 	if mnt != nil && mnt.ReadOnly() {
 		// Switch to ReadWrite while we setup submounts.
-		if err := c.k.VFS().SetMountReadOnly(mnt, false); err != nil {
+		if err := c.l.k.VFS().SetMountReadOnly(mnt, false); err != nil {
 			return fmt.Errorf("failed to set mount at %q readwrite: %w", submount.mount.Destination, err)
 		}
 		// Restore back to ReadOnly at the end.
 		defer func() {
-			if err := c.k.VFS().SetMountReadOnly(mnt, true); err != nil {
+			if err := c.l.k.VFS().SetMountReadOnly(mnt, true); err != nil {
 				panic(fmt.Sprintf("failed to restore mount at %q back to readonly: %v", submount.mount.Destination, err))
 			}
 		}()
@@ -1225,7 +1256,7 @@ func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs
 	mountCtx := vfs.WithRoot(vfs.WithMountNamespace(ctx, mns), root)
 	for _, ctrl := range kernel.CgroupCtrls {
 		ctrlName := string(ctrl)
-		cgroupMnt := c.k.GetCgroupMount(ctrlName)
+		cgroupMnt := c.l.k.GetCgroupMount(ctrlName)
 		if cgroupMnt == nil {
 			return fmt.Errorf("cgroup mount for controller %s not found", ctrlName)
 		}
@@ -1237,7 +1268,7 @@ func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs
 			// Use the containerID as the cgroup path.
 			Path: fspath.Parse(c.containerID),
 		}
-		if err := c.k.VFS().MkdirAt(mountCtx, creds, &sourcePop, &vfs.MkdirOptions{
+		if err := c.l.k.VFS().MkdirAt(mountCtx, creds, &sourcePop, &vfs.MkdirOptions{
 			Mode: 0755,
 		}); err != nil {
 			log.Infof("error in creating directory %v", err)
@@ -1246,7 +1277,7 @@ func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs
 
 		// Bind mount the new cgroup directory into the container's mount namespace.
 		destination := "/sys/fs/cgroup/" + ctrlName
-		if err := c.k.VFS().MakeSyntheticMountpoint(mountCtx, destination, root, creds); err != nil {
+		if err := c.l.k.VFS().MakeSyntheticMountpoint(mountCtx, destination, root, creds); err != nil {
 			// Log a warning, but attempt the mount anyway.
 			log.Warningf("Failed to create mount point %q: %v", destination, err)
 		}
@@ -1256,7 +1287,7 @@ func (c *containerMounter) mountCgroupSubmounts(ctx context.Context, spec *specs
 			Start: root,
 			Path:  fspath.Parse(destination),
 		}
-		if err := c.k.VFS().BindAt(mountCtx, creds, &sourcePop, target, false); err != nil {
+		if err := c.l.k.VFS().BindAt(mountCtx, creds, &sourcePop, target, false); err != nil {
 			log.Infof("error in bind mounting %v", err)
 			return err
 		}
@@ -1271,7 +1302,7 @@ func (c *containerMounter) mountSharedMaster(ctx context.Context, spec *specs.Sp
 	// Mount the master using the options from the hint (mount annotations).
 	origOpts := mntInfo.mount.Options
 	mntInfo.mount.Options = mntInfo.hint.Mount.Options
-	fsName, opts, err := getMountNameAndOptions(spec, conf, mntInfo, c.productName, c.containerName)
+	fsName, opts, err := getMountNameAndOptions(spec, conf, mntInfo, c.l.productName, c.containerName, c.l.fsRestore)
 	mntInfo.mount.Options = origOpts
 	if err != nil {
 		return nil, err
@@ -1279,7 +1310,7 @@ func (c *containerMounter) mountSharedMaster(ctx context.Context, spec *specs.Sp
 	if len(fsName) == 0 {
 		return nil, fmt.Errorf("mount type not supported %q", mntInfo.hint.Mount.Type)
 	}
-	return c.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
+	return c.l.k.VFS().MountDisconnected(ctx, creds, "", fsName, opts)
 }
 
 // mountSharedSubmount binds mount to a previously mounted volume that is shared
@@ -1291,7 +1322,7 @@ func (c *containerMounter) mountSharedSubmount(ctx context.Context, conf *config
 
 	// Generate mount point specific opts using mntInfo.mount.
 	opts := ParseMountOptions(mntInfo.mount.Options)
-	newMnt := c.k.VFS().NewDisconnectedMount(sharedMount.Filesystem(), sharedMount.Root(), opts)
+	newMnt := c.l.k.VFS().NewDisconnectedMount(sharedMount.Filesystem(), sharedMount.Root(), opts)
 	defer newMnt.DecRef(ctx)
 
 	root := mns.Root(ctx)
@@ -1318,7 +1349,7 @@ func (c *containerMounter) mountSharedSubmount(ctx context.Context, conf *config
 
 	// Avoid mounting on top of symlinks. The mount syscall on Linux always follows symlinks.
 	target.FollowFinalSymlink = true
-	if err := c.k.VFS().ConnectMountAt(ctx, creds, newMnt, target); err != nil {
+	if err := c.l.k.VFS().ConnectMountAt(ctx, creds, newMnt, target); err != nil {
 		return nil, err
 	}
 	log.Infof("Mounted %q type shared bind to %q", mntInfo.mount.Destination, mntInfo.hint.Name)
@@ -1346,7 +1377,7 @@ func (c *containerMounter) makeMountPoint(
 		FollowFinalSymlink: true,
 	}
 
-	fs := c.k.VFS()
+	fs := c.l.k.VFS()
 
 	mode, err := c.getPathMode(ctx, creds, target)
 	// First check if mount point exists.
@@ -1400,7 +1431,7 @@ func (c *containerMounter) configureRestore(fdmap map[checkpoint.ResourceID]int,
 	fdmap[rootKey] = c.goferFDs.remove()
 
 	if rootfsConf := c.goferMountConfs[0]; rootfsConf.IsFilestorePresent() {
-		mf, err := createPrivateMemoryFile(c.goferFilestoreFDs.removeAsFD().ReleaseToFile("overlay-filestore"), rootKey)
+		mf, err := createPrivateMemoryFile(c.goferFilestoreFDs.removeAsFD().ReleaseToFile("overlay-filestore"), rootKey, nil)
 		if err != nil {
 			return fmt.Errorf("failed to create private memory file for mount rootfs: %w", err)
 		}
@@ -1419,7 +1450,7 @@ func (c *containerMounter) configureRestore(fdmap map[checkpoint.ResourceID]int,
 		}
 		if submount.filestoreFD != nil {
 			key := checkpoint.ResourceID{ContainerName: c.containerName, Path: submount.mount.Destination}
-			mf, err := createPrivateMemoryFile(submount.filestoreFD.ReleaseToFile("overlay-filestore"), key)
+			mf, err := createPrivateMemoryFile(submount.filestoreFD.ReleaseToFile("overlay-filestore"), key, nil)
 			if err != nil {
 				return fmt.Errorf("failed to create private memory file for mount %q: %w", submount.mount.Destination, err)
 			}

runsc/cli/maincli/maincli.go

@@ -38,26 +38,29 @@ func commands() (map[util.SubCommand]string, []subcommands.Command) {
 
 	return map[util.SubCommand]string{
 			// Register OCI user-facing runsc commands.
-			new(cmd.Checkpoint):  "",
-			new(cmd.Create):      "",
-			new(cmd.Delete):      "",
-			new(cmd.Do):          "",
-			new(cmd.Events):      "",
-			new(cmd.Exec):        "",
-			new(cmd.Kill):        "",
-			new(cmd.List):        "",
-			new(cmd.PS):          "",
-			new(cmd.Pause):       "",
-			new(cmd.PortForward): "",
-			new(cmd.Restore):     "",
-			new(cmd.Resume):      "",
-			new(cmd.Run):         "",
-			new(cmd.Spec):        "",
-			new(cmd.Start):       "",
-			new(cmd.State):       "",
-			new(cmd.Tar):         "",
-			new(cmd.Update):      "",
-			new(cmd.Wait):        "",
+			new(cmd.Checkpoint): "",
+			new(cmd.Create):     "",
+			new(cmd.Delete):     "",
+			new(cmd.Events):     "",
+			new(cmd.Exec):       "",
+			new(cmd.Kill):       "",
+			new(cmd.List):       "",
+			new(cmd.PS):         "",
+			new(cmd.Pause):      "",
+			new(cmd.Restore):    "",
+			new(cmd.Resume):     "",
+			new(cmd.Run):        "",
+			new(cmd.Spec):       "",
+			new(cmd.Start):      "",
+			new(cmd.State):      "",
+			new(cmd.Update):     "",
+			new(cmd.Wait):       "",
+
+			// Non-OCI user-facing runsc commands.
+			new(cmd.Do):           "",
+			new(cmd.FSCheckpoint): "",
+			new(cmd.PortForward):  "",
+			new(cmd.Tar):          "",
 
 			// Helpers.
 			new(cmd.Install):     helperGroup,

runsc/cmd/BUILD

@@ -48,6 +48,7 @@ go_library(
         "events.go",
         "exec.go",
         "fd_mapping.go",
+        "fscheckpoint.go",
         "gofer.go",
         "install.go",
         "kill.go",

runsc/cmd/boot.go

@@ -155,6 +155,8 @@ type Boot struct {
 
 	saveFDs intFlags
 
+	fsRestoreFDs intFlags
+
 	// attached is set to true to kill the sandbox process when the parent process
 	// terminates. This flag is set when the command execve's itself because
 	// parent death signal doesn't propagate through execve when uid/gid changes.
@@ -202,10 +204,10 @@ type Boot struct {
 	uid int
 	gid int
 
-	bootExtra
-
 	// rootfsUpperTarFD is the file descriptor to a tar file that has rootfs change at startup.
 	rootfsUpperTarFD int
+
+	bootExtra
 }
 
 // Name implements subcommands.Command.Name.
@@ -257,6 +259,7 @@ func (b *Boot) SetFlags(f *flag.FlagSet) {
 	f.IntVar(&b.podInitConfigFD, "pod-init-config-fd", -1, "file descriptor to the pod init configuration file.")
 	f.Var(&b.sinkFDs, "sink-fds", "ordered list of file descriptors to be used by the sinks defined in --pod-init-config.")
 	f.Var(&b.saveFDs, "save-fds", "ordered list of file descriptors to be used save checkpoints. Order: kernel state, page metadata, page file")
+	f.Var(&b.fsRestoreFDs, "fs-restore-fds", "ordered list of file descriptors for filesystem checkpoint restore")
 	f.IntVar(&b.rootfsUpperTarFD, "rootfs-upper-tar-fd", -1, "file descriptor to the tar file containing the rootfs upper layer changes.")
 
 	// Profiling flags.
@@ -583,6 +586,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		},
 		HostTHP:          b.hostTHP,
 		SaveFDs:          b.saveFDs.GetFDs(),
+		FSRestoreFDs:     b.fsRestoreFDs.GetFDs(),
 		RootfsUpperTarFD: b.rootfsUpperTarFD,
 	}
 	b.setBootArgsExtra(&bootArgs)

runsc/cmd/create.go

@@ -51,6 +51,12 @@ type Create struct {
 	// consumed by developers.
 	userLog string
 
+	// fsRestoreImagePath is the path to the filesystem checkpoint to restore.
+	// fsRestoreDirect is true if files making up the filesystem checkpoint should
+	// be opened with O_DIRECT.
+	fsRestoreImagePath string
+	fsRestoreDirect    bool
+
 	// spec is the cached OCI spec.
 	spec *specs.Spec
 }
@@ -76,6 +82,8 @@ func (c *Create) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&c.consoleSocket, "console-socket", "", "path to an AF_UNIX socket which will receive a file descriptor referencing the master end of the console's pseudoterminal")
 	f.StringVar(&c.pidFile, "pid-file", "", "filename that the container pid will be written to")
 	f.StringVar(&c.userLog, "user-log", "", "filename to send user-visible logs to. Empty means no logging.")
+	f.StringVar(&c.fsRestoreImagePath, "fs-restore-image-path", "", "path to filesystem checkpoint to restore")
+	f.BoolVar(&c.fsRestoreDirect, "fs-restore-direct", false, "open files in fs-restore-image-path with O_DIRECT")
 }
 
 // FetchSpec implements util.SubCommand.FetchSpec.
@@ -121,12 +129,14 @@ func (c *Create) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcom
 	// container unless the metadata specifies that it should be run in an
 	// existing container.
 	contArgs := container.Args{
-		ID:            id,
-		Spec:          spec,
-		BundleDir:     c.bundleDir,
-		ConsoleSocket: c.consoleSocket,
-		PIDFile:       c.pidFile,
-		UserLog:       c.userLog,
+		ID:                 id,
+		Spec:               spec,
+		BundleDir:          c.bundleDir,
+		ConsoleSocket:      c.consoleSocket,
+		PIDFile:            c.pidFile,
+		UserLog:            c.userLog,
+		FSRestoreImagePath: c.fsRestoreImagePath,
+		FSRestoreDirect:    c.fsRestoreDirect,
 	}
 	if _, err := container.New(conf, contArgs); err != nil {
 		return util.Errorf("creating container: %v", err)

runsc/cmd/fscheckpoint.go

@@ -0,0 +1,102 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/google/subcommands"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"gvisor.dev/gvisor/runsc/cmd/util"
+	"gvisor.dev/gvisor/runsc/config"
+	"gvisor.dev/gvisor/runsc/container"
+	"gvisor.dev/gvisor/runsc/flag"
+	"gvisor.dev/gvisor/runsc/sandbox"
+)
+
+// FSCheckpoint implements subcommands.Command for the "fscheckpoint" command.
+type FSCheckpoint struct {
+	containerLoader
+	imagePath    string
+	leaveRunning bool
+	direct       bool
+}
+
+// Name implements subcommands.Command.Name.
+func (*FSCheckpoint) Name() string {
+	return "fscheckpoint"
+}
+
+// Synopsis implements subcommands.Command.Synopsis.
+func (*FSCheckpoint) Synopsis() string {
+	return "checkpoint container filesystems (experimental)"
+}
+
+// Usage implements subcommands.Command.Usage.
+func (*FSCheckpoint) Usage() string {
+	return `fscheckpoint [flags] <container id>:
+	Saves sandbox filesystem checkpoint to -image-path directory. Restore checkpoints using "runsc create -fs-restore-image-path=<path>".
+`
+}
+
+// SetFlags implements subcommands.Command.SetFlags.
+func (c *FSCheckpoint) SetFlags(f *flag.FlagSet) {
+	f.StringVar(&c.imagePath, "image-path", "", "directory path to saved filesystem checkpoint")
+	f.BoolVar(&c.leaveRunning, "leave-running", false, "if true, resume containers after checkpointing; if false, containers exit with status 0 after checkpointing")
+	f.BoolVar(&c.direct, "direct", false, "use O_DIRECT for writing checkpoint files")
+}
+
+// FetchSpec implements util.SubCommand.FetchSpec.
+func (c *FSCheckpoint) FetchSpec(conf *config.Config, f *flag.FlagSet) (string, *specs.Spec, error) {
+	cont, err := c.loadContainer(conf, f, container.LoadOpts{})
+	if err != nil {
+		return "", nil, fmt.Errorf("loading container: %w", err)
+	}
+	return cont.ID, cont.Spec, nil
+}
+
+// Execute implements subcommands.Command.Execute.
+func (c *FSCheckpoint) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus {
+	if f.NArg() != 1 {
+		f.Usage()
+		return subcommands.ExitUsageError
+	}
+
+	conf := args[0].(*config.Config)
+
+	cont, err := c.loadContainer(conf, f, container.LoadOpts{})
+	if err != nil {
+		util.Fatalf("loading container: %v", err)
+	}
+
+	if c.imagePath == "" {
+		util.Fatalf("image-path flag must be provided")
+	}
+
+	if err := os.MkdirAll(c.imagePath, 0755); err != nil {
+		util.Fatalf("making directories at path provided: %v", err)
+	}
+
+	if err := cont.FSSave(conf, c.imagePath, sandbox.FSSaveOpts{
+		Direct:          c.direct,
+		ExitAfterSaving: !c.leaveRunning,
+	}); err != nil {
+		util.Fatalf("filesystem checkpoint saving failed: %v", err)
+	}
+
+	return subcommands.ExitSuccess
+}

runsc/cmd/run.go

@@ -157,15 +157,17 @@ func (r *Run) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomman
 	}()
 
 	runArgs := container.Args{
-		ID:            id,
-		Spec:          spec,
-		BundleDir:     r.bundleDir,
-		ConsoleSocket: r.consoleSocket,
-		PIDFile:       r.pidFile,
-		UserLog:       r.userLog,
-		Attached:      !r.detach,
-		PassFiles:     fdMap,
-		ExecFile:      execFile,
+		ID:                 id,
+		Spec:               spec,
+		BundleDir:          r.bundleDir,
+		ConsoleSocket:      r.consoleSocket,
+		PIDFile:            r.pidFile,
+		UserLog:            r.userLog,
+		Attached:           !r.detach,
+		PassFiles:          fdMap,
+		ExecFile:           execFile,
+		FSRestoreImagePath: r.fsRestoreImagePath,
+		FSRestoreDirect:    r.fsRestoreDirect,
 	}
 	ws, err := container.Run(conf, runArgs)
 	if err != nil {

runsc/container/container.go

@@ -192,6 +192,12 @@ type Args struct {
 
 	// ExecFile is the host file used for program execution.
 	ExecFile *os.File
+
+	// If FSRestoreImagePath is non-empty, it is a path to a filesystem
+	// checkpoint that should be restored. FSRestoreImagePath may only be set
+	// for containers in a new Sandbox process.
+	FSRestoreImagePath string
+	FSRestoreDirect    bool
 }
 
 // New creates the container in a new Sandbox process, unless the metadata
@@ -218,6 +224,9 @@ func New(conf *config.Config, args Args) (*Container, error) {
 		if !ok {
 			return nil, fmt.Errorf("no sandbox ID found when creating container")
 		}
+		if args.FSRestoreImagePath != "" {
+			return nil, fmt.Errorf("cannot set FSRestoreImagePath when creating container in existing sandbox")
+		}
 	}
 
 	c := &Container{
@@ -322,6 +331,8 @@ func New(conf *config.Config, args Args) (*Container, error) {
 				MountHints:          mountHints,
 				PassFiles:           args.PassFiles,
 				ExecFile:            args.ExecFile,
+				FSRestoreImagePath:  args.FSRestoreImagePath,
+				FSRestoreDirect:     args.FSRestoreDirect,
 			}
 			sand, err := sandbox.New(conf, sandArgs)
 			if err != nil {
@@ -776,6 +787,15 @@ func (c *Container) Checkpoint(conf *config.Config, imagePath string, opts sandb
 	return c.Sandbox.Checkpoint(conf, c.ID, imagePath, opts)
 }
 
+// FSSave sends the filesystem checkpointing call to the container.
+func (c *Container) FSSave(conf *config.Config, imagePath string, opts sandbox.FSSaveOpts) error {
+	log.Debugf("Checkpoint container filesystem, cid: %s", c.ID)
+	if err := c.requireStatus("filesystem checkpoint", Created, Running, Paused); err != nil {
+		return err
+	}
+	return c.Sandbox.FSSave(conf, c.ID, imagePath, opts)
+}
+
 // Pause suspends the container and its kernel.
 // The call only succeeds if the container's status is created or running.
 func (c *Container) Pause() error {

runsc/container/multi_container_test.go

@@ -17,6 +17,7 @@ package container
 import (
 	"fmt"
 	"io"
+	"math/rand/v2"
 	"os"
 	"os/exec"
 	"path"
@@ -68,6 +69,10 @@ func createSpecs(cmds ...[]string) ([]*specs.Spec, []string) {
 }
 
 func startContainers(conf *config.Config, specs []*specs.Spec, ids []string) ([]*Container, func(), error) {
+	return startContainersWithArgs(conf, specs, ids, nil)
+}
+
+func startContainersWithArgs(conf *config.Config, specs []*specs.Spec, ids []string, mutateArgs func(int, *Args)) ([]*Container, func(), error) {
 	if len(conf.RootDir) == 0 {
 		panic("conf.RootDir not set. Call testutil.SetupRootDir() to set.")
 	}
@@ -88,6 +93,9 @@ func startContainers(conf *config.Config, specs []*specs.Spec, ids []string) ([]
 			Spec:      spec,
 			BundleDir: bundleDir,
 		}
+		if mutateArgs != nil {
+			mutateArgs(i, &args)
+		}
 		cont, err := New(conf, args)
 		if err != nil {
 			return nil, nil, fmt.Errorf("error creating container: %v", err)
@@ -3142,3 +3150,134 @@ func TestMultiContainerCgroupsMemoryUsage(t *testing.T) {
 		})
 	}
 }
+
+func TestFSCheckpointCommand(t *testing.T) {
+	conf := testutil.TestConfig(t)
+
+	rootDir, cleanupRoot, err := testutil.SetupRootDir()
+	if err != nil {
+		t.Fatalf("Error creating root dir: %v", err)
+	}
+	defer cleanupRoot()
+	conf.RootDir = rootDir
+
+	// Enable disk-backed overlay as required for filesystem checkpointing.
+	conf.Overlay2.Set("root:self")
+
+	// Containers are matched between save and restore by their names. If no
+	// name is specified, runsc auto-assigns container names based on ordering,
+	// but if a container dies and is externally restarted, runsc doesn't know
+	// that the old and new containers are related and will assign the new
+	// container a new name. So for restoring to work after container restart,
+	// we need to assign a name explicitly.
+	const (
+		rootName = "root-container"
+		subName  = "sub-container"
+		initName = "init-container"
+	)
+
+	// Each container must use a distinct writable temporary directory as its
+	// filesystem root, to hold the filestore file used by disk-backed overlay.
+	appSrc, err := testutil.FindFile("test/cmd/test_app/test_app")
+	if err != nil {
+		t.Fatal("Error finding test_app:", err)
+	}
+	setupSpecRoots := func(specs []*specs.Spec, ids []string) (func(), error) {
+		var cleanupSpecRoots cleanup.Cleanup
+		defer cleanupSpecRoots.Clean()
+		for i, spec := range specs {
+			contRootPath, err := os.MkdirTemp(testutil.TmpDir(), fmt.Sprintf("%s-root", ids[i]))
+			if err != nil {
+				return nil, fmt.Errorf("error creating root directory for container %d: %v", i, err)
+			}
+			cleanupSpecRoots.Add(func() { os.RemoveAll(contRootPath) })
+			spec.Root.Path = contRootPath
+			spec.Root.Readonly = false
+			// Copy test_app to "/app" inside the container.
+			appDst := filepath.Join(contRootPath, "app")
+			if err := copyFile(appSrc, appDst); err != nil {
+				return nil, fmt.Errorf("error copying app binary from %q to %q: %v", appSrc, appDst, err)
+			}
+		}
+		return cleanupSpecRoots.Release(), nil
+	}
+
+	// Start two containers which sleep.
+	var testAppSleepArgv = []string{"/app", "reaper"}
+	specs, ids := createSpecs(testAppSleepArgv, testAppSleepArgv)
+	specs[0].Annotations[specutils.ContainerdContainerNameAnnotation] = rootName
+	specs[1].Annotations[specutils.ContainerdContainerNameAnnotation] = subName
+	cleanupRootsOld, err := setupSpecRoots(specs, ids)
+	if err != nil {
+		t.Fatalf("Error setting up container roots: %v", err)
+	}
+	defer cleanupRootsOld()
+	conts, cleanupContsOld, err := startContainers(conf, specs, ids)
+	if err != nil {
+		t.Fatalf("Error starting containers: %v", err)
+	}
+	defer cleanupContsOld()
+
+	// Populate container filesystems.
+	fsTreeCommonArgs := []string{"--depth=10", "--file-per-level=10", "--file-size=65537", "--create-symlink", "--add-empty-files"}
+	fsTreeArgs := make([][]string, len(conts))
+	for i := range conts {
+		fsTreeArgs[i] = append(fsTreeCommonArgs, fmt.Sprintf("seed=%d", rand.Uint64()))
+		if ws, err := execute(conf, conts[i], "/app", append([]string{"fsTreeCreate"}, fsTreeArgs[i]...)...); err != nil || ws != 0 {
+			t.Fatalf("Error populating filesystem for container %d, ws: %v, err: %v", i, ws, err)
+		}
+	}
+
+	// Save a filesystem checkpoint and kill the sandbox.
+	imagePath, err := os.MkdirTemp(testutil.TmpDir(), "fscheckpoint-image")
+	if err != nil {
+		t.Fatalf("Error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(imagePath)
+	if err := conts[0].FSSave(conf, imagePath, sandbox.FSSaveOpts{
+		ExitAfterSaving: true,
+	}); err != nil {
+		t.Fatalf("Error saving filesystem checkpoint: %v", err)
+	}
+
+	// Start three containers which sleep, two of which restore from the
+	// filesystem checkpoint.
+	specs, ids = createSpecs(testAppSleepArgv, testAppSleepArgv, testAppSleepArgv)
+	specs[0].Annotations[specutils.ContainerdContainerNameAnnotation] = rootName
+	specs[1].Annotations[specutils.ContainerdContainerNameAnnotation] = subName
+	specs[2].Annotations[specutils.ContainerdContainerNameAnnotation] = initName
+	cleanupRootsNew, err := setupSpecRoots(specs, ids)
+	if err != nil {
+		t.Fatalf("Error setting up container roots: %v", err)
+	}
+	defer cleanupRootsNew()
+	conts, cleanupContsNew, err := startContainersWithArgs(conf, specs, ids, func(i int, contArgs *Args) {
+		if i == 0 {
+			contArgs.FSRestoreImagePath = imagePath
+		}
+	})
+	if err != nil {
+		t.Fatalf("Error starting containers: %v", err)
+	}
+	defer cleanupContsNew()
+
+	// Verify container filesystems restored from checkpoint.
+	for i := range conts[:2] {
+		if ws, err := execute(conf, conts[i], "/app", append([]string{"fsTreeVerify"}, fsTreeArgs[i]...)...); err != nil || ws != 0 {
+			t.Fatalf("Error verifying filesystem for container %d, ws: %v, err: %v", i, ws, err)
+		}
+	}
+
+	// Restart the second container and verify that its filesystem is restored
+	// again.
+	conts[1].Destroy()
+	restartID := testutil.RandomContainerID()
+	contsRestart, cleanupContsRestart, err := startContainers(conf, specs[1:2], []string{restartID})
+	if err != nil {
+		t.Fatalf("Error starting container: %v", err)
+	}
+	defer cleanupContsRestart()
+	if ws, err := execute(conf, contsRestart[0], "/app", append([]string{"fsTreeVerify"}, fsTreeArgs[1]...)...); err != nil || ws != 0 {
+		t.Fatalf("Error verifying filesystem for restarted container, ws: %v, err: %v", ws, err)
+	}
+}

runsc/sandbox/sandbox.go

@@ -303,6 +303,12 @@ type Args struct {
 
 	// ExecFile is the file from the host used for program execution.
 	ExecFile *os.File
+
+	// If FSRestoreImagePath is non-empty, it is a path to a filesystem
+	// checkpoint that should be restored. If FSRestoreDirect is true,
+	// open filesystem checkpoint files using O_DIRECT.
+	FSRestoreImagePath string
+	FSRestoreDirect    bool
 }
 
 // New creates the sandbox process. The caller must call Destroy() on the
@@ -982,6 +988,14 @@ func (s *Sandbox) createSandboxProcess(conf *config.Config, args *Args, startSyn
 		donations.DonateAndClose("save-fds", files...)
 	}
 
+	if args.FSRestoreImagePath != "" {
+		files, err := s.openFSRestoreFilesImpl(conf, args.FSRestoreImagePath, args.FSRestoreDirect, cmd)
+		if err != nil {
+			return fmt.Errorf("failed to open filesystem checkpoint files: %w", err)
+		}
+		donations.DonateAndClose("fs-restore-fds", files...)
+	}
+
 	if err := s.createSandboxProcessExtra(conf, args, cmd, &donations); err != nil {
 		return err
 	}
@@ -1599,6 +1613,121 @@ func createSaveFiles(path string, direct bool, compression statefile.Compression
 	return files, nil
 }
 
+func openFSRestoreFilesForLocalCheckpoint(imagePath string, direct bool) ([]*os.File, error) {
+	return openFSCheckpointLocalFiles(imagePath, os.O_RDONLY, direct)
+}
+
+// FSSaveOpts holds options to FSSave.
+type FSSaveOpts struct {
+	// If Direct is true, open checkpoint files with O_DIRECT where possible.
+	Direct bool
+
+	// If ExitAfterSaving is true, all sandboxed processes immediately exit
+	// with status 0 after filesystem checkpoint saving, whether or not saving
+	// is successful. This is equivalent to !CheckpointOpts.Resume, and is
+	// provided for parity with that feature.
+	ExitAfterSaving bool
+}
+
+// FSSave sends the filesystem checkpointing call to the sandbox.
+func (s *Sandbox) FSSave(conf *config.Config, cid string, imagePath string, opts FSSaveOpts) error {
+	log.Debugf("Checkpoint filesystem for sandbox %q, imagePath %q, opts %+v", s.ID, imagePath, opts)
+
+	args := boot.FSSaveArgs{
+		ExitAfterSaving: opts.ExitAfterSaving,
+	}
+	defer func() {
+		for _, f := range args.FilePayload.Files {
+			_ = f.Close()
+		}
+	}()
+	if err := s.setFSSaveArgsImpl(conf, imagePath, opts.Direct, &args); err != nil {
+		return err
+	}
+
+	if err := s.call(boot.ContMgrFSSave, &args, nil); err != nil {
+		return fmt.Errorf("checkpointing filesystem for container %q: %w", cid, err)
+	}
+	return nil
+}
+
+func setFSSaveArgsForLocalCheckpointFiles(conf *config.Config, imagePath string, direct bool, args *boot.FSSaveArgs) error {
+	files, err := openFSCheckpointLocalFiles(imagePath, os.O_CREATE|os.O_EXCL|os.O_RDWR, direct)
+	if err != nil {
+		return err
+	}
+	args.FilePayload.Files = files
+	return nil
+}
+
+func openFSCheckpointLocalFiles(imagePath string, openFlags int, direct bool) ([]*os.File, error) {
+	var files [4]*os.File
+	closeCleanup := cleanup.Make(func() {
+		for _, f := range files {
+			if f != nil {
+				f.Close()
+			}
+		}
+	})
+	defer closeCleanup.Clean()
+
+	// Use of O_DIRECT ~requires that I/O operations are aligned to some
+	// unspecified [1] granularity that (as of this writing) is, in practice,
+	// less than or equal to page size. Performance and memory pressure
+	// benefits of O_DIRECT where applicable (avoiding page cache when it won't
+	// be used) scale linearly with file size.
+	// - Manifest file: no alignment guarantee, very small size
+	// - Multi-tar files: no alignment guarantee, relatively small size
+	// - Pages metadata file: no alignment guarantee, relatively small size
+	// - Pages file: page-aligned I/O, relatively large size
+	// Thus currently O_DIRECT is only used for pages file.
+	//
+	// [1] Obtainable via STATX_DIOALIGN / STATX_DIO_READ_ALIGN; "support
+	// varies by filesystem". But we don't need to deal with this unless we
+	// start having checkpoints stored on devices where the direct I/O size is
+	// greater than page size.
+	//
+	// TODO: NOLINT - If multi-tar or pages metadata files get big, we can
+	// guarantee alignment and enable async I/O by using
+	// stateio.BufReader/Writer(stateio.FDReader/Writer) rather than
+	// bufio.Reader/Writer(os.File) in the sentry.
+	maybeODirect := 0
+	if direct {
+		maybeODirect = unix.O_DIRECT
+	}
+
+	manifestFilePath := filepath.Join(imagePath, checkpointfiles.FSCheckpointManifestFileName)
+	manifestFile, err := os.OpenFile(manifestFilePath, openFlags, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("opening manifest file %q: %w", manifestFilePath, err)
+	}
+	files[0] = manifestFile
+
+	multiTarFilePath := filepath.Join(imagePath, checkpointfiles.FSCheckpointMultiTarFileName)
+	multiTarFile, err := os.OpenFile(multiTarFilePath, openFlags, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("opening multi-tar file %q: %w", multiTarFilePath, err)
+	}
+	files[1] = multiTarFile
+
+	pagesMetadataFilePath := filepath.Join(imagePath, checkpointfiles.PagesMetadataFileName)
+	pagesMetadataFile, err := os.OpenFile(pagesMetadataFilePath, openFlags, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("opening pages metadata file %q: %w", pagesMetadataFilePath, err)
+	}
+	files[2] = pagesMetadataFile
+
+	pagesFilePath := filepath.Join(imagePath, checkpointfiles.PagesFileName)
+	pagesFileFD, err := unix.Open(pagesFilePath, openFlags|maybeODirect, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("opening pages metadata file %q: %w", pagesFilePath, err)
+	}
+	files[3] = os.NewFile(uintptr(pagesFileFD), pagesFilePath)
+
+	closeCleanup.Release()
+	return files[:], nil
+}
+
 // Pause sends the pause call for a container in the sandbox.
 func (s *Sandbox) Pause(cid string) error {
 	log.Debugf("Pause sandbox %q", s.ID)

runsc/sandbox/sandbox_impl.go

@@ -18,6 +18,7 @@
 package sandbox
 
 import (
+	"os"
 	"os/exec"
 
 	"gvisor.dev/gvisor/pkg/sentry/control"
@@ -39,3 +40,11 @@ func (s *Sandbox) setCheckpointOptsImpl(conf *config.Config, imagePath string, o
 func (s *Sandbox) setRestoreOptsImpl(conf *config.Config, imagePath string, direct bool, opt *boot.RestoreOpts) error {
 	return s.setRestoreOptsForLocalCheckpointFiles(conf, imagePath, direct, opt)
 }
+
+func (s *Sandbox) setFSSaveArgsImpl(conf *config.Config, imagePath string, direct bool, args *boot.FSSaveArgs) error {
+	return setFSSaveArgsForLocalCheckpointFiles(conf, imagePath, direct, args)
+}
+
+func (s *Sandbox) openFSRestoreFilesImpl(conf *config.Config, imagePath string, direct bool, cmd *exec.Cmd) ([]*os.File, error) {
+	return openFSRestoreFilesForLocalCheckpoint(imagePath, direct)
+}

test/cmd/test_app/BUILD

@@ -11,6 +11,7 @@ go_binary(
     srcs = [
         "fds.go",
         "main.go",
+        "main_impl.go",
         "zombies.go",
     ],
     features = ["fully_static_link"],
@@ -20,7 +21,6 @@ go_binary(
     ],
     deps = [
         "//pkg/gvisordetect",
-        "//pkg/rand",
         "//pkg/test/testutil",
         "//pkg/unet",
         "//runsc/flag",

test/cmd/test_app/main.go

@@ -17,10 +17,13 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"log"
+	"math/rand/v2"
 	"net"
 	"os"
 	"os/exec"
@@ -34,7 +37,6 @@ import (
 	"github.com/creack/pty"
 	"github.com/google/subcommands"
 	"gvisor.dev/gvisor/pkg/gvisordetect"
-	gvisorrand "gvisor.dev/gvisor/pkg/rand"
 	"gvisor.dev/gvisor/pkg/test/testutil"
 	"gvisor.dev/gvisor/runsc/flag"
 )
@@ -47,6 +49,7 @@ func main() {
 	subcommands.Register(new(fdSender), "")
 	subcommands.Register(new(forkBomb), "")
 	subcommands.Register(new(fsTreeCreator), "")
+	subcommands.Register(new(fsTreeVerify), "")
 	subcommands.Register(new(gvisorDetect), "")
 	subcommands.Register(new(ptyRunner), "")
 	subcommands.Register(new(reaper), "")
@@ -54,6 +57,7 @@ func main() {
 	subcommands.Register(new(taskTree), "")
 	subcommands.Register(new(uds), "")
 	subcommands.Register(new(zombieTest), "")
+	registerSubcommandsExtra()
 
 	flag.Parse()
 
@@ -62,12 +66,17 @@ func main() {
 }
 
 type fsTreeCreator struct {
+	fsTreeCommon
+}
+
+type fsTreeCommon struct {
 	depth            uint
 	numFilesPerLevel uint
 	fileSize         uint
 	targetDir        string
 	createSymlink    bool
 	addEmptyFiles    bool
+	seed             uint64
 }
 
 // Name implements subcommands.Command.Name.
@@ -86,13 +95,14 @@ func (*fsTreeCreator) Usage() string {
 }
 
 // SetFlags implements subcommands.Command.SetFlags.
-func (c *fsTreeCreator) SetFlags(f *flag.FlagSet) {
+func (c *fsTreeCommon) SetFlags(f *flag.FlagSet) {
 	f.UintVar(&c.depth, "depth", 10, "number of levels to create")
 	f.UintVar(&c.numFilesPerLevel, "file-per-level", 10, "number of files to create per level")
 	f.UintVar(&c.fileSize, "file-size", 4096, "size of each file")
 	f.StringVar(&c.targetDir, "target-dir", "/", "directory under which to create the filesystem tree")
 	f.BoolVar(&c.createSymlink, "create-symlink", false, "create symlinks other than the first file per level")
 	f.BoolVar(&c.addEmptyFiles, "add-empty-files", false, "add empty file to each level")
+	f.Uint64Var(&c.seed, "seed", 0, "seed used for randomization")
 }
 
 // Execute implements subcommands.Command.Execute.
@@ -105,7 +115,7 @@ func (c *fsTreeCreator) Execute(ctx context.Context, f *flag.FlagSet, args ...an
 	}
 
 	data := make([]byte, c.fileSize)
-	gvisorrand.Read(data)
+	randRead(c.seed, data)
 	for i := uint(0); i < c.depth; i++ {
 		for j := uint(0); j < c.numFilesPerLevel; j++ {
 			filePath := filepath.Join(curDir, fmt.Sprintf("file%d", j))
@@ -134,6 +144,87 @@ func (c *fsTreeCreator) Execute(ctx context.Context, f *flag.FlagSet, args ...an
 	return subcommands.ExitSuccess
 }
 
+func randRead(seed uint64, dst []byte) {
+	src := rand.NewPCG(seed, 0)
+	var buf [8]byte
+	for len(dst) != 0 {
+		u64 := src.Uint64()
+		binary.NativeEndian.PutUint64(buf[:], u64)
+		n := copy(dst, buf[:])
+		dst = dst[n:]
+	}
+}
+
+type fsTreeVerify struct {
+	fsTreeCommon
+}
+
+// Name implements subcommands.Command.Name.
+func (*fsTreeVerify) Name() string {
+	return "fsTreeVerify"
+}
+
+// Synopsis implements subcommands.Command.Synopsys.
+func (*fsTreeVerify) Synopsis() string {
+	return "verifies a filesystem tree created by fsTreeCreate with the same arguments"
+}
+
+// Usage implements subcommands.Command.Usage.
+func (*fsTreeVerify) Usage() string {
+	return "fsTreeVerify <flags>"
+}
+
+// Execute implements subcommands.Command.Execute.
+func (c *fsTreeVerify) Execute(ctx context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus {
+	curDir := c.targetDir
+	if stat, err := os.Lstat(curDir); err != nil {
+		log.Fatalf("Failed to stat directory %q: %v", curDir, err)
+	} else if !stat.IsDir() {
+		log.Fatalf("Directory %q has type %s", curDir, stat.Mode().Type())
+	}
+
+	dataWant := make([]byte, c.fileSize)
+	randRead(c.seed, dataWant)
+	for i := uint(0); i < c.depth; i++ {
+		for j := uint(0); j < c.numFilesPerLevel; j++ {
+			filePath := filepath.Join(curDir, fmt.Sprintf("file%d", j))
+			if c.createSymlink && j > 0 {
+				if stat, err := os.Lstat(filePath); err != nil {
+					log.Fatalf("Failed to stat file %q: %v", filePath, err)
+				} else if stat.Mode().Type() != os.ModeSymlink {
+					log.Fatalf("File %q has type %s, want symlink", filePath, stat.Mode().Type())
+				} else if target, err := os.Readlink(filePath); err != nil || target != "file0" {
+					log.Fatalf("Symlink %q: got target=%q, err=%v; want target=\"file0\"", filePath, target, err)
+				}
+			} else {
+				if data, err := os.ReadFile(filePath); err != nil {
+					log.Fatalf("Error reading file %q: %v", filePath, err)
+				} else if !bytes.Equal(data, dataWant) {
+					log.Fatalf("file %q has content %q, want %q", filePath, data, dataWant)
+				}
+			}
+		}
+		if c.addEmptyFiles {
+			emptyPath := filepath.Join(curDir, fmt.Sprintf("empty%d", i))
+			if stat, err := os.Lstat(emptyPath); err != nil {
+				log.Fatalf("Failed to stat file %q: %v", emptyPath, err)
+			} else if !stat.Mode().IsRegular() {
+				log.Fatalf("File %q has type %s, want regular file", emptyPath, stat.Mode().Type())
+			} else if stat.Size() != 0 {
+				log.Fatalf("File %q has size %d, want 0", emptyPath, stat.Size())
+			}
+		}
+		nextDir := filepath.Join(curDir, "dir")
+		if stat, err := os.Lstat(nextDir); err != nil {
+			log.Fatalf("Failed to stat directory %q: %v", nextDir, err)
+		} else if !stat.IsDir() {
+			log.Fatalf("Directory %q has type %s", nextDir, stat.Mode().Type())
+		}
+		curDir = nextDir
+	}
+	return subcommands.ExitSuccess
+}
+
 type uds struct {
 	fileName   string
 	socketPath string

test/cmd/test_app/main_impl.go

@@ -0,0 +1,21 @@
+// Copyright 2026 The gVisor Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !false
+// +build !false
+
+package main
+
+func registerSubcommandsExtra() {
+}

tools/go_marshal/gomarshal/generator_interfaces_struct.go

@@ -654,4 +654,106 @@ func (g *interfaceGenerator) emitMarshallableSliceForStruct(st *ast.StructType,
 		}
 	})
 	g.emit("}\n\n")
+
+	g.emit("// Read%s reads a []%s. It returns the number of bytes read\n", slice.ident, g.typeName())
+	g.recordUsedImport("io")
+	g.emit("func Read%s(src io.Reader, dst []%s) (int, error) {\n", slice.ident, g.typeName())
+	g.inIndent(func() {
+		g.emit("count := len(dst)\n")
+		g.emit("if count == 0 {\n")
+		g.inIndent(func() {
+			g.emit("return 0, nil\n")
+		})
+		g.emit("}\n")
+		g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName())
+
+		fallback := func() {
+			g.emit("// Type %s doesn't have a packed layout in memory, fall back to UnmarshalBytes.\n", g.typeName())
+			g.emit("buf := make([]byte, size)\n")
+			g.emit("length := 0\n")
+			g.emit("for idx := 0; idx < count; idx++ {\n")
+			g.inIndent(func() {
+				g.emit("n, err := io.ReadFull(src, buf)\n")
+				g.emit("length += n\n")
+				g.emit("if err != nil {\n")
+				g.inIndent(func() {
+					g.emit("return length, err\n")
+				})
+				g.emit("}\n")
+				g.emit("dst[idx].UnmarshalBytes(buf)\n")
+			})
+			g.emit("}\n")
+			g.emit("return length, nil\n")
+		}
+		if thisPacked {
+			g.recordUsedImport("reflect")
+			g.recordUsedImport("runtime")
+			g.recordUsedImport("unsafe")
+			if _, ok := g.areFieldsPackedExpression(); ok {
+				g.emit("if !dst[0].Packed() {\n")
+				g.inIndent(fallback)
+				g.emit("}\n\n")
+			}
+			// Fast serialization.
+			g.emitCastSliceToByteSlice("&dst", "buf", "size * count")
+
+			g.emit("length, err := io.ReadFull(src, buf)\n")
+			g.emitKeepAlive("dst")
+			g.emit("return length, err\n")
+		} else {
+			fallback()
+		}
+	})
+	g.emit("}\n\n")
+
+	g.emit("// Write%s is like %s.WriteTo, but for a []%s.\n", slice.ident, g.typeName(), g.typeName())
+	g.recordUsedImport("io")
+	g.emit("func Write%s(dst io.Writer, src []%s) (int, error) {\n", slice.ident, g.typeName())
+	g.inIndent(func() {
+		g.emit("count := len(src)\n")
+		g.emit("if count == 0 {\n")
+		g.inIndent(func() {
+			g.emit("return 0, nil\n")
+		})
+		g.emit("}\n")
+		g.emit("size := (*%s)(nil).SizeBytes()\n\n", g.typeName())
+
+		fallback := func() {
+			g.emit("// Type %s doesn't have a packed layout in memory, fall back to MarshalBytes.\n", g.typeName())
+			g.emit("buf := make([]byte, size)\n")
+			g.emit("length := 0\n")
+			g.emit("for idx := 0; idx < count; idx++ {\n")
+			g.inIndent(func() {
+				g.emit("src[idx].MarshalBytes(buf)\n")
+				g.emit("n, err := dst.Write(buf)\n")
+				g.emit("length += n\n")
+				g.emit("if err != nil {\n")
+				g.inIndent(func() {
+					g.emit("return length, err\n")
+				})
+				g.emit("}\n")
+			})
+			g.emit("}\n")
+			g.emit("return length, nil\n")
+		}
+		if thisPacked {
+			g.recordUsedImport("reflect")
+			g.recordUsedImport("runtime")
+			g.recordUsedImport("unsafe")
+			if _, ok := g.areFieldsPackedExpression(); ok {
+				g.emit("if !src[0].Packed() {\n")
+				g.inIndent(fallback)
+				g.emit("}\n\n")
+			}
+			// Fast serialization.
+			g.emitCastSliceToByteSlice("&src", "buf", "size * count")
+
+			g.emit("length, err := dst.Write(buf)\n")
+			g.emitKeepAlive("src")
+			g.emit("return length, err\n")
+		} else {
+			fallback()
+		}
+	})
+	g.emit("}\n\n")
 }

website/BUILD

@@ -159,6 +159,7 @@ docs(
         "//g3doc/user_guide:compatibility",
         "//g3doc/user_guide:debugging",
         "//g3doc/user_guide:filesystem",
+        "//g3doc/user_guide:fs_snapshot",
         "//g3doc/user_guide:gpu",
         "//g3doc/user_guide:install",
         "//g3doc/user_guide:networking",