s3/weed/storage/volume_io_error_test.go

package storage

import (
	"errors"
	"fmt"
	"sync"
	"syscall"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
)

// eioBackend is a backend.BackendStorageFile that returns EIO from
// every ReadAt — used to verify the streaming read path threads
// errors through checkReadWriteError.
type eioBackend struct{}

func (eioBackend) ReadAt(p []byte, off int64) (int, error) { return 0, syscall.EIO }
func (eioBackend) WriteAt(p []byte, off int64) (int, error) {
	return len(p), nil
}
func (eioBackend) Truncate(int64) error                          { return nil }
func (eioBackend) Close() error                                  { return nil }
func (eioBackend) GetStat() (int64, time.Time, error)            { return 0, time.Time{}, nil }
func (eioBackend) Name() string                                  { return "eio" }
func (eioBackend) Sync() error                                   { return nil }

func TestCheckReadWriteErrorTracksConsecutiveEIO(t *testing.T) {
	v := &Volume{}

	// each EIO bumps the counter.
	for i := int32(1); i <= 5; i++ {
		v.checkReadWriteError(fmt.Errorf("disk failed: %w", syscall.EIO))
		_, count, _ := v.getIoErrorState()
		if count != i {
			t.Fatalf("after %d EIO(s): counter = %d, want %d", i, count, i)
		}
	}

	// a single success resets the streak (but not the quarantine flag,
	// which is not set here).
	v.checkReadWriteError(nil)
	if err, count, q := v.getIoErrorState(); err != nil || count != 0 || q {
		t.Fatalf("success did not reset state: err=%v count=%d quarantined=%v", err, count, q)
	}
}

func TestCheckReadWriteErrorNonEIOResetsStreak(t *testing.T) {
	v := &Volume{}

	v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	if _, count, _ := v.getIoErrorState(); count != 2 {
		t.Fatalf("expected count=2 after two EIOs, got %d", count)
	}

	// a non-EIO error breaks the streak — only sustained EIOs are
	// diagnostic of a failing disk.
	v.checkReadWriteError(fmt.Errorf("other: %w", syscall.ENOSPC))
	if err, count, _ := v.getIoErrorState(); err != nil || count != 0 {
		t.Fatalf("non-EIO did not reset streak: err=%v count=%d", err, count)
	}

	// a fresh EIO starts the streak from 1, not 3.
	v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	if _, count, _ := v.getIoErrorState(); count != 1 {
		t.Fatalf("EIO after non-EIO did not restart streak: count=%d, want 1", count)
	}
}

func TestCheckReadWriteErrorIgnoresPlainError(t *testing.T) {
	v := &Volume{}

	v.checkReadWriteError(errors.New("some other error"))
	if err, count, _ := v.getIoErrorState(); err != nil || count != 0 {
		t.Fatalf("non-EIO with no prior streak set state: err=%v count=%d", err, count)
	}
}

func TestIoErrorToleranceGate(t *testing.T) {
	v := &Volume{}

	for i := 0; i < IoErrorTolerance-1; i++ {
		v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	}
	if _, count, _ := v.getIoErrorState(); count >= IoErrorTolerance {
		t.Fatalf("counter %d already crossed tolerance %d after %d errors",
			count, IoErrorTolerance, IoErrorTolerance-1)
	}

	v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	if _, count, _ := v.getIoErrorState(); count < IoErrorTolerance {
		t.Fatalf("counter %d below tolerance %d after %d errors",
			count, IoErrorTolerance, IoErrorTolerance)
	}
}

// Once CollectHeartbeat marks a replica quarantined, a stray successful
// read must NOT silently put a known-bad disk back into rotation. Only
// MarkVolumeWritable (resetIoErrorState) clears the sticky bit.
func TestQuarantineIsSticky(t *testing.T) {
	v := &Volume{}

	for i := 0; i < IoErrorTolerance; i++ {
		v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
	}
	v.markIoQuarantined()

	if _, _, q := v.getIoErrorState(); !q {
		t.Fatalf("markIoQuarantined did not set the flag")
	}

	// A successful read clears the streak counter…
	v.checkReadWriteError(nil)
	if _, count, q := v.getIoErrorState(); count != 0 {
		t.Fatalf("success did not clear streak: count=%d", count)
	} else if !q {
		t.Fatalf("success cleared the sticky quarantine flag — operator-only recovery violated")
	}

	// …and a non-EIO error also clears the streak but not the flag.
	v.checkReadWriteError(fmt.Errorf("other: %w", syscall.ENOSPC))
	if _, _, q := v.getIoErrorState(); !q {
		t.Fatalf("non-EIO cleared the sticky quarantine flag")
	}

	// Only resetIoErrorState (used by MarkVolumeWritable) un-quarantines.
	v.resetIoErrorState()
	if err, count, q := v.getIoErrorState(); err != nil || count != 0 || q {
		t.Fatalf("resetIoErrorState left state: err=%v count=%d quarantined=%v", err, count, q)
	}
}

// Streaming/range reads (ReadNeedleBlob) used to bypass the EIO
// counter, so a failing disk taking range GETs all day would never
// trip IoErrorTolerance. ReadNeedleBlob now threads the backend error
// through checkReadWriteError; one EIO must bump the streak by one.
func TestReadNeedleBlobTracksEIO(t *testing.T) {
	v := &Volume{
		DataBackend: eioBackend{},
		SuperBlock:  super_block.SuperBlock{Version: needle.GetCurrentVersion()},
		volumeInfo:  &volume_server_pb.VolumeInfo{Version: uint32(needle.GetCurrentVersion())},
	}

	if _, err := v.ReadNeedleBlob(0, 1); !errors.Is(err, syscall.EIO) {
		t.Fatalf("expected EIO from fake backend, got %v", err)
	}
	if _, count, _ := v.getIoErrorState(); count != 1 {
		t.Fatalf("ReadNeedleBlob did not bump EIO streak: count=%d", count)
	}
}

func TestIoErrorStateIsRaceFree(t *testing.T) {
	// Relies on `go test -race` to detect any unprotected access on
	// lastIoError / lastIoErrorCount / ioErrorQuarantined.
	v := &Volume{}

	var wg sync.WaitGroup
	stop := make(chan struct{})

	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
				v.checkReadWriteError(fmt.Errorf("eio: %w", syscall.EIO))
			}
		}
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
				v.checkReadWriteError(nil)
			}
		}
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
				v.markIoQuarantined()
				v.resetIoErrorState()
			}
		}
	}()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 1000; i++ {
			v.getIoErrorState()
		}
		close(stop)
	}()
	wg.Wait()
}