Files
Tan Yifeng 8411c4028d fsimpl: Fix Linux rmdir inotify event ordering
Linux does not emit rmdir inotify events with a single unconditional
ordering.

If the removed directory has no extra references, the child receives
IN_DELETE_SELF/IN_IGNORED before the parent's IN_DELETE|IN_ISDIR.

If an extra reference remains, such as an open directory FD, Linux still
emits the parent's IN_DELETE|IN_ISDIR at rmdir() time and defers the
child's IN_DELETE_SELF/IN_IGNORED until the final reference is dropped.

Match that behavior across overlay, tmpfs, gofer, and kernfs by eagerly
sending child deletion notifications only in the no-extra-ref case, and
keeping the parent delete notification on the rmdir path.

Add syscall coverage for both the immediate and deferred cases.

Signed-off-by: Tan Yifeng <[email protected]>
2026-04-25 10:04:48 +08:00

573 lines
18 KiB
Go

// Copyright 2019 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kernfs_test
import (
"bytes"
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
"gvisor.dev/gvisor/pkg/abi/linux"
"gvisor.dev/gvisor/pkg/context"
"gvisor.dev/gvisor/pkg/errors/linuxerr"
"gvisor.dev/gvisor/pkg/fspath"
"gvisor.dev/gvisor/pkg/hostarch"
"gvisor.dev/gvisor/pkg/sentry/contexttest"
"gvisor.dev/gvisor/pkg/sentry/fsimpl/kernfs"
"gvisor.dev/gvisor/pkg/sentry/fsimpl/testutil"
"gvisor.dev/gvisor/pkg/sentry/kernel/auth"
"gvisor.dev/gvisor/pkg/sentry/vfs"
"gvisor.dev/gvisor/pkg/usermem"
)
const defaultMode linux.FileMode = 01777
const staticFileContent = "This is sample content for a static test file."
// RootDentryFn is a generator function for creating the root dentry of a test
// filesystem. See newTestSystem.
type RootDentryFn func(context.Context, *auth.Credentials, *filesystem) kernfs.Inode
// newTestSystem sets up a minimal environment for running a test, including an
// instance of a test filesystem. Tests can control the contents of the
// filesystem by providing an appropriate rootFn, which should return a
// pre-populated root dentry.
func newTestSystem(t *testing.T, rootFn RootDentryFn) *testutil.System {
ctx := contexttest.Context(t)
creds := auth.CredentialsFromContext(ctx)
v := &vfs.VirtualFilesystem{}
if err := v.Init(ctx); err != nil {
t.Fatalf("VFS init: %v", err)
}
v.MustRegisterFilesystemType("testfs", &fsType{rootFn: rootFn}, &vfs.RegisterFilesystemTypeOptions{
AllowUserMount: true,
})
mns, err := v.NewMountNamespace(ctx, creds, "", "testfs", &vfs.MountOptions{}, nil)
if err != nil {
t.Fatalf("Failed to create testfs root mount: %v", err)
}
return testutil.NewSystem(ctx, t, v, mns)
}
type fsType struct {
rootFn RootDentryFn
}
type filesystem struct {
kernfs.Filesystem
}
// MountOptions implements vfs.FilesystemImpl.MountOptions.
func (fs *filesystem) MountOptions() string {
return ""
}
type file struct {
kernfs.DynamicBytesFile
content string
}
func (fs *filesystem) newFile(ctx context.Context, creds *auth.Credentials, content string) kernfs.Inode {
f := &file{}
f.content = content
f.DynamicBytesFile.Init(ctx, creds, 0 /* devMajor */, 0 /* devMinor */, fs.NextIno(), f, 0777)
return f
}
func (f *file) Generate(ctx context.Context, buf *bytes.Buffer) error {
fmt.Fprintf(buf, "%s", f.content)
return nil
}
type attrs struct {
kernfs.InodeAttrs
}
func (*attrs) SetStat(context.Context, *vfs.Filesystem, *auth.Credentials, vfs.SetStatOptions) error {
return linuxerr.EPERM
}
type readonlyDir struct {
readonlyDirRefs
attrs
kernfs.InodeAlwaysValid
kernfs.InodeDirectoryNoNewChildren
kernfs.InodeNoStatFS
kernfs.InodeNotAnonymous
kernfs.InodeNotSymlink
kernfs.InodeTemporary
kernfs.InodeWatches
kernfs.OrderedChildren
kernfs.InodeFSOwned
locks vfs.FileLocks
}
func (fs *filesystem) newReadonlyDir(ctx context.Context, creds *auth.Credentials, mode linux.FileMode, contents map[string]kernfs.Inode) kernfs.Inode {
dir := &readonlyDir{}
dir.attrs.Init(ctx, creds, 0 /* devMajor */, 0 /* devMinor */, fs.NextIno(), linux.ModeDirectory|mode)
dir.OrderedChildren.Init(kernfs.OrderedChildrenOptions{})
dir.InitRefs()
dir.IncLinks(dir.OrderedChildren.Populate(contents))
return dir
}
func (d *readonlyDir) Open(ctx context.Context, rp *vfs.ResolvingPath, kd *kernfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
fd, err := kernfs.NewGenericDirectoryFD(rp.Mount(), kd, rp.Credentials(), &d.OrderedChildren, &d.locks, &opts, kernfs.GenericDirectoryFDOptions{
SeekEnd: kernfs.SeekEndStaticEntries,
})
if err != nil {
return nil, err
}
return fd.VFSFileDescription(), nil
}
func (d *readonlyDir) DecRef(ctx context.Context) {
d.readonlyDirRefs.DecRef(func() { d.Destroy(ctx) })
}
type dir struct {
dirRefs
attrs
kernfs.InodeAlwaysValid
kernfs.InodeNoStatFS
kernfs.InodeNotAnonymous
kernfs.InodeNotSymlink
kernfs.InodeTemporary
kernfs.InodeWatches
kernfs.OrderedChildren
kernfs.InodeFSOwned
locks vfs.FileLocks
fs *filesystem
}
func (fs *filesystem) newDir(ctx context.Context, creds *auth.Credentials, mode linux.FileMode, contents map[string]kernfs.Inode) kernfs.Inode {
dir := &dir{}
dir.fs = fs
dir.attrs.Init(ctx, creds, 0 /* devMajor */, 0 /* devMinor */, fs.NextIno(), linux.ModeDirectory|mode)
dir.OrderedChildren.Init(kernfs.OrderedChildrenOptions{Writable: true})
dir.InitRefs()
dir.IncLinks(dir.OrderedChildren.Populate(contents))
return dir
}
func (d *dir) Open(ctx context.Context, rp *vfs.ResolvingPath, kd *kernfs.Dentry, opts vfs.OpenOptions) (*vfs.FileDescription, error) {
fd, err := kernfs.NewGenericDirectoryFD(rp.Mount(), kd, rp.Credentials(), &d.OrderedChildren, &d.locks, &opts, kernfs.GenericDirectoryFDOptions{
SeekEnd: kernfs.SeekEndStaticEntries,
})
if err != nil {
return nil, err
}
return fd.VFSFileDescription(), nil
}
func (d *dir) DecRef(ctx context.Context) {
d.dirRefs.DecRef(func() { d.Destroy(ctx) })
}
func (d *dir) NewDir(ctx context.Context, name string, opts vfs.MkdirOptions) (kernfs.Inode, error) {
creds := auth.CredentialsFromContext(ctx)
dir := d.fs.newDir(ctx, creds, opts.Mode, nil)
if err := d.OrderedChildren.Insert(name, dir); err != nil {
dir.DecRef(ctx)
return nil, err
}
d.TouchCMtime(ctx)
d.IncLinks(1)
return dir, nil
}
func (d *dir) NewFile(ctx context.Context, name string, opts vfs.OpenOptions) (kernfs.Inode, error) {
creds := auth.CredentialsFromContext(ctx)
f := d.fs.newFile(ctx, creds, "")
if err := d.OrderedChildren.Insert(name, f); err != nil {
f.DecRef(ctx)
return nil, err
}
d.TouchCMtime(ctx)
return f, nil
}
type keptDir struct {
dir
}
func (fs *filesystem) newKeptDir(ctx context.Context, creds *auth.Credentials, mode linux.FileMode, contents map[string]kernfs.Inode) kernfs.Inode {
kdir := &keptDir{}
kdir.dir.fs = fs
kdir.dir.attrs.Init(ctx, creds, 0 /* devMajor */, 0 /* devMinor */, fs.NextIno(), linux.ModeDirectory|mode)
kdir.dir.OrderedChildren.Init(kernfs.OrderedChildrenOptions{Writable: true})
kdir.dir.InitRefs()
for name, child := range contents {
if err := kdir.dir.OrderedChildren.Insert(name, child); err != nil {
panic(fmt.Sprintf("newKeptDir Insert(%q): %v", name, err))
}
kdir.dir.IncLinks(1)
}
return kdir
}
func (*keptDir) Keep() bool {
return true
}
func (*dir) NewLink(context.Context, string, kernfs.Inode) (kernfs.Inode, error) {
return nil, linuxerr.EPERM
}
func (*dir) NewSymlink(context.Context, string, string) (kernfs.Inode, error) {
return nil, linuxerr.EPERM
}
func (*dir) NewNode(context.Context, string, vfs.MknodOptions) (kernfs.Inode, error) {
return nil, linuxerr.EPERM
}
func (fsType) Name() string {
return "kernfs"
}
func (fsType) Release(ctx context.Context) {}
func (fst fsType) GetFilesystem(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, source string, opt vfs.GetFilesystemOptions) (*vfs.Filesystem, *vfs.Dentry, error) {
fs := &filesystem{}
fs.VFSFilesystem().Init(vfsObj, &fst, fs)
root := fst.rootFn(ctx, creds, fs)
var d kernfs.Dentry
d.Init(&fs.Filesystem, root)
return fs.VFSFilesystem(), d.VFSDentry(), nil
}
type inotifyEvent struct {
WD int32
Mask uint32
Name string
}
func readInotifyEvents(t *testing.T, ctx context.Context, fd *vfs.FileDescription) []inotifyEvent {
t.Helper()
buf := make([]byte, 4096)
n, err := fd.Read(ctx, usermem.BytesIOSequence(buf), vfs.ReadOptions{})
if err != nil {
t.Fatalf("inotify read failed: %v", err)
}
buf = buf[:n]
var events []inotifyEvent
for len(buf) != 0 {
if len(buf) < 16 {
t.Fatalf("short inotify event buffer: got %d bytes", len(buf))
}
nameLen := int(hostarch.ByteOrder.Uint32(buf[12:16]))
eventLen := 16 + nameLen
if len(buf) < eventLen {
t.Fatalf("truncated inotify event buffer: got %d bytes, need %d", len(buf), eventLen)
}
events = append(events, inotifyEvent{
WD: int32(hostarch.ByteOrder.Uint32(buf[0:4])),
Mask: hostarch.ByteOrder.Uint32(buf[4:8]),
Name: string(bytes.TrimRight(buf[16:eventLen], "\x00")),
})
buf = buf[eventLen:]
}
return events
}
// -------------------- Remainder of the file are test cases --------------------
func TestBasic(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, map[string]kernfs.Inode{
"file1": fs.newFile(ctx, creds, staticFileContent),
})
})
defer sys.Destroy()
sys.GetDentryOrDie(sys.PathOpAtRoot("file1")).DecRef(sys.Ctx)
}
func TestMkdirGetDentry(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, map[string]kernfs.Inode{
"dir1": fs.newDir(ctx, creds, 0755, nil),
})
})
defer sys.Destroy()
pop := sys.PathOpAtRoot("dir1/a new directory")
if err := sys.VFS.MkdirAt(sys.Ctx, sys.Creds, pop, &vfs.MkdirOptions{Mode: 0755}); err != nil {
t.Fatalf("MkdirAt for PathOperation %+v failed: %v", pop, err)
}
sys.GetDentryOrDie(pop).DecRef(sys.Ctx)
}
func TestReadStaticFile(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, map[string]kernfs.Inode{
"file1": fs.newFile(ctx, creds, staticFileContent),
})
})
defer sys.Destroy()
pop := sys.PathOpAtRoot("file1")
fd, err := sys.VFS.OpenAt(sys.Ctx, sys.Creds, pop, &vfs.OpenOptions{
Flags: linux.O_RDONLY,
})
if err != nil {
t.Fatalf("OpenAt for PathOperation %+v failed: %v", pop, err)
}
defer fd.DecRef(sys.Ctx)
content, err := sys.ReadToEnd(fd)
if err != nil {
t.Fatalf("Read failed: %v", err)
}
if diff := cmp.Diff(staticFileContent, content); diff != "" {
t.Fatalf("Read returned unexpected data:\n--- want\n+++ got\n%v", diff)
}
}
func TestCreateNewFileInStaticDir(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, map[string]kernfs.Inode{
"dir1": fs.newDir(ctx, creds, 0755, nil),
})
})
defer sys.Destroy()
pop := sys.PathOpAtRoot("dir1/newfile")
opts := &vfs.OpenOptions{Flags: linux.O_CREAT | linux.O_EXCL, Mode: defaultMode}
fd, err := sys.VFS.OpenAt(sys.Ctx, sys.Creds, pop, opts)
if err != nil {
t.Fatalf("OpenAt(pop:%+v, opts:%+v) failed: %v", pop, opts, err)
}
// Close the file. The file should persist.
fd.DecRef(sys.Ctx)
fd, err = sys.VFS.OpenAt(sys.Ctx, sys.Creds, pop, &vfs.OpenOptions{
Flags: linux.O_RDONLY,
})
if err != nil {
t.Fatalf("OpenAt(pop:%+v) = %+v failed: %v", pop, fd, err)
}
fd.DecRef(sys.Ctx)
}
func TestDirFDReadWrite(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, nil)
})
defer sys.Destroy()
pop := sys.PathOpAtRoot("/")
fd, err := sys.VFS.OpenAt(sys.Ctx, sys.Creds, pop, &vfs.OpenOptions{
Flags: linux.O_RDONLY,
})
if err != nil {
t.Fatalf("OpenAt for PathOperation %+v failed: %v", pop, err)
}
defer fd.DecRef(sys.Ctx)
// Read/Write should fail for directory FDs.
if _, err := fd.Read(sys.Ctx, usermem.BytesIOSequence([]byte{}), vfs.ReadOptions{}); !linuxerr.Equals(linuxerr.EISDIR, err) {
t.Fatalf("Read for directory FD failed with unexpected error: %v", err)
}
if _, err := fd.Write(sys.Ctx, usermem.BytesIOSequence([]byte{}), vfs.WriteOptions{}); !linuxerr.Equals(linuxerr.EBADF, err) {
t.Fatalf("Write for directory FD failed with unexpected error: %v", err)
}
}
func TestDirFDIterDirents(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newReadonlyDir(ctx, creds, 0755, map[string]kernfs.Inode{
// Fill root with nodes backed by various inode implementations.
"dir1": fs.newReadonlyDir(ctx, creds, 0755, nil),
"dir2": fs.newDir(ctx, creds, 0755, map[string]kernfs.Inode{
"dir3": fs.newDir(ctx, creds, 0755, nil),
}),
"file1": fs.newFile(ctx, creds, staticFileContent),
})
})
defer sys.Destroy()
pop := sys.PathOpAtRoot("/")
sys.AssertAllDirentTypes(sys.ListDirents(pop), map[string]testutil.DirentType{
"dir1": linux.DT_DIR,
"dir2": linux.DT_DIR,
"file1": linux.DT_REG,
})
}
func TestDirWalkDentryTree(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newDir(ctx, creds, 0755, map[string]kernfs.Inode{
"dir1": fs.newDir(ctx, creds, 0755, nil),
"dir2": fs.newDir(ctx, creds, 0755, map[string]kernfs.Inode{
"file1": fs.newFile(ctx, creds, staticFileContent),
"dir3": fs.newDir(ctx, creds, 0755, nil),
}),
})
})
defer sys.Destroy()
testWalk := func(from *kernfs.Dentry, getDentryPath, walkPath string, expectedErr error) {
var d *kernfs.Dentry
if getDentryPath != "" {
pop := sys.PathOpAtRoot(getDentryPath)
vd := sys.GetDentryOrDie(pop)
defer vd.DecRef(sys.Ctx)
d = vd.Dentry().Impl().(*kernfs.Dentry)
}
match, err := from.WalkDentryTree(sys.Ctx, sys.VFS, fspath.Parse(walkPath))
if err == nil {
defer match.DecRef(sys.Ctx)
}
if err != expectedErr {
t.Fatalf("WalkDentryTree from %q to %q (with expected error: %v) unexpected error, want: %v, got: %v", from.FSLocalPath(), walkPath, expectedErr, expectedErr, err)
}
if expectedErr != nil {
return
}
if d != match {
t.Fatalf("WalkDentryTree from %q to %q (with expected error: %v) found unexpected dentry; want: %v, got: %v", from.FSLocalPath(), walkPath, expectedErr, d, match)
}
}
rootD := sys.Root.Dentry().Impl().(*kernfs.Dentry)
testWalk(rootD, "dir1", "/dir1", nil)
testWalk(rootD, "", "/dir-non-existent", linuxerr.ENOENT)
testWalk(rootD, "", "/dir1/child-non-existent", linuxerr.ENOENT)
testWalk(rootD, "", "/dir2/inner-non-existent/dir3", linuxerr.ENOENT)
testWalk(rootD, "dir2/dir3", "/dir2/../dir2/dir3", nil)
testWalk(rootD, "dir2/dir3", "/dir2/././dir3", nil)
testWalk(rootD, "dir2/dir3", "/dir2/././dir3/.././dir3", nil)
pop := sys.PathOpAtRoot("dir2")
dir2VD := sys.GetDentryOrDie(pop)
defer dir2VD.DecRef(sys.Ctx)
dir2D := dir2VD.Dentry().Impl().(*kernfs.Dentry)
testWalk(dir2D, "dir2/dir3", "/dir3", nil)
testWalk(dir2D, "dir2/dir3", "/../../../dir3", nil)
testWalk(dir2D, "dir2/file1", "/file1", nil)
testWalk(dir2D, "dir2/file1", "file1", nil)
}
func TestRmdirInotifyDeleteSelfBeforeParentDelete(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newKeptDir(ctx, creds, 0755, map[string]kernfs.Inode{
"parent": fs.newKeptDir(ctx, creds, 0755, map[string]kernfs.Inode{
"child": fs.newKeptDir(ctx, creds, 0755, nil),
}),
})
})
defer sys.Destroy()
ctx := vfs.WithMountNamespace(sys.Ctx, sys.MntNs)
inotifyFD, err := vfs.NewInotifyFD(ctx, sys.VFS, linux.O_NONBLOCK)
if err != nil {
t.Fatalf("NewInotifyFD failed: %v", err)
}
defer inotifyFD.DecRef(ctx)
ino := inotifyFD.Impl().(*vfs.Inotify)
parentVD := sys.GetDentryOrDie(sys.PathOpAtRoot("parent"))
defer parentVD.DecRef(sys.Ctx)
childVD := sys.GetDentryOrDie(sys.PathOpAtRoot("parent/child"))
parentWD := ino.AddWatch(parentVD.Dentry(), linux.IN_ALL_EVENTS)
childWD := ino.AddWatch(childVD.Dentry(), linux.IN_ALL_EVENTS)
childVD.DecRef(sys.Ctx)
if err := sys.VFS.RmdirAt(ctx, sys.Creds, sys.PathOpAtRoot("parent/child")); err != nil {
t.Fatalf("RmdirAt failed: %v", err)
}
got := readInotifyEvents(t, ctx, inotifyFD)
want := []inotifyEvent{
{WD: childWD, Mask: linux.IN_DELETE_SELF},
{WD: childWD, Mask: linux.IN_IGNORED},
{WD: parentWD, Mask: linux.IN_DELETE | linux.IN_ISDIR, Name: "child"},
}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected inotify events after rmdir (-want +got):\n%s", diff)
}
}
func TestRmdirInotifyWithOpenFDDefersDeleteSelf(t *testing.T) {
sys := newTestSystem(t, func(ctx context.Context, creds *auth.Credentials, fs *filesystem) kernfs.Inode {
return fs.newKeptDir(ctx, creds, 0755, map[string]kernfs.Inode{
"parent": fs.newKeptDir(ctx, creds, 0755, map[string]kernfs.Inode{
"child": fs.newKeptDir(ctx, creds, 0755, nil),
}),
})
})
defer sys.Destroy()
ctx := vfs.WithMountNamespace(sys.Ctx, sys.MntNs)
inotifyFD, err := vfs.NewInotifyFD(ctx, sys.VFS, linux.O_NONBLOCK)
if err != nil {
t.Fatalf("NewInotifyFD failed: %v", err)
}
defer inotifyFD.DecRef(ctx)
ino := inotifyFD.Impl().(*vfs.Inotify)
parentVD := sys.GetDentryOrDie(sys.PathOpAtRoot("parent"))
defer parentVD.DecRef(sys.Ctx)
childVD := sys.GetDentryOrDie(sys.PathOpAtRoot("parent/child"))
parentWD := ino.AddWatch(parentVD.Dentry(), linux.IN_ALL_EVENTS)
childWD := ino.AddWatch(childVD.Dentry(), linux.IN_ALL_EVENTS)
childVD.DecRef(sys.Ctx)
childFD, err := sys.VFS.OpenAt(ctx, sys.Creds, sys.PathOpAtRoot("parent/child"), &vfs.OpenOptions{Flags: linux.O_RDONLY})
if err != nil {
t.Fatalf("OpenAt failed: %v", err)
}
// Opening the watched directory generates IN_OPEN events; they are not part
// of the rmdir ordering being validated below.
_ = readInotifyEvents(t, ctx, inotifyFD)
if err := sys.VFS.RmdirAt(ctx, sys.Creds, sys.PathOpAtRoot("parent/child")); err != nil {
t.Fatalf("RmdirAt failed: %v", err)
}
got := readInotifyEvents(t, ctx, inotifyFD)
want := []inotifyEvent{{WD: parentWD, Mask: linux.IN_DELETE | linux.IN_ISDIR, Name: "child"}}
if diff := cmp.Diff(want, got); diff != "" {
t.Fatalf("unexpected inotify events after rmdir with open fd (-want +got):\n%s", diff)
}
childFD.DecRef(ctx)
got = readInotifyEvents(t, ctx, inotifyFD)
if len(got) < 2 {
t.Fatalf("expected at least delete-self and ignored events after close, got: %+v", got)
}
wantTail := []inotifyEvent{
{WD: childWD, Mask: linux.IN_DELETE_SELF},
{WD: childWD, Mask: linux.IN_IGNORED},
}
if diff := cmp.Diff(wantTail, got[len(got)-2:]); diff != "" {
t.Fatalf("unexpected trailing inotify events after close (-want +got):\n%s\nall events: %+v", diff, got)
}
}