s3/weed/mount/peer_registrar.go

package mount

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)

// PeerRegistrar maintains this mount's presence in every configured
// filer's mount registry and its local snapshot of the merged seed set.
// It runs two background tickers:
//
//   - MountRegister heartbeats, fanned out to every filer in parallel so
//     each filer keeps a fresh TTL entry for us. Mounts pointing at
//     different filers therefore still see each other once all filers
//     have been heartbeated.
//   - MountList polling, fanned out identically, merged by peer_addr
//     (newest LastSeenNs wins) so OwnerFor() has a view of the whole
//     fleet regardless of which filer each peer happens to heartbeat
//     through.
//
// An unreachable filer is tolerated: we log and continue as long as at
// least one filer succeeds. Entries cached on a permanently-gone filer
// fall out of the merged view on their own TTL.
type PeerRegistrar struct {
	filerAddrs       []pb.ServerAddress
	dialFiler        filerDialFn
	selfPeerAddr     string
	selfDc           string
	selfRack         string
	registerInterval time.Duration
	registerTTL      time.Duration
	listInterval     time.Duration

	mu    sync.RWMutex
	seeds []SeedPeer

	// stopCtx cancels when Stop() is called. Background RPCs scope their
	// deadline to this so unmount does not block on pending filer calls.
	stopCtx    context.Context
	stopCancel context.CancelFunc
	stopped    atomic.Bool
}

// filerDialFn is how the registrar reaches one configured filer. The
// production wiring is pb.WithGrpcFilerClient; tests inject a fake.
type filerDialFn func(ctx context.Context, addr pb.ServerAddress, fn func(client filer_pb.SeaweedFilerClient) error) error

// NewPeerRegistrar constructs the registrar; Start launches the background
// loops. Callers must supply the full filer set so heartbeats and list
// polls reach every filer — otherwise mounts talking to different filers
// never observe each other.
func NewPeerRegistrar(filers []pb.ServerAddress, dial filerDialFn, selfAddr, dc, rack string) *PeerRegistrar {
	ctx, cancel := context.WithCancel(context.Background())
	return &PeerRegistrar{
		filerAddrs:       filers,
		dialFiler:        dial,
		selfPeerAddr:     selfAddr,
		selfRack:         rack,
		selfDc:           dc,
		registerInterval: 30 * time.Second,
		registerTTL:      90 * time.Second,
		listInterval:     30 * time.Second,
		stopCtx:          ctx,
		stopCancel:       cancel,
	}
}

// Start does an initial register+list synchronously (so OwnerFor has a
// usable view immediately) and then kicks off the background loops.
func (r *PeerRegistrar) Start(ctx context.Context) error {
	if err := r.registerOnce(ctx); err != nil {
		glog.V(1).Infof("initial MountRegister: %v", err)
		// Do not fail startup — the mount must still serve reads even if
		// no filer yet knows about us.
	}
	if err := r.listOnce(ctx); err != nil {
		glog.V(1).Infof("initial MountList: %v", err)
	}
	go r.loopRegister()
	go r.loopList()
	return nil
}

// Stop halts the background loops and cancels any in-flight RPCs they
// may have launched. Safe to call multiple times.
func (r *PeerRegistrar) Stop() {
	if r.stopped.Swap(true) {
		return
	}
	r.stopCancel()
}

// Seeds returns the currently-known seed set. Callers MUST NOT mutate the
// returned slice — it is shared with concurrent listOnce readers. Because
// listOnce atomically swaps to a brand-new slice on every refresh rather
// than mutating in place, returning the slice header directly is safe and
// avoids a per-call allocation on the read-hot OwnerFor path.
func (r *PeerRegistrar) Seeds() []SeedPeer {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.seeds
}

// OwnerFor is a convenience wrapper that runs HRW against the current
// seed snapshot.
func (r *PeerRegistrar) OwnerFor(fid string) string {
	return OwnerFor(fid, r.Seeds())
}

// filerRPCTimeout bounds a single MountRegister / MountList call so a slow
// or partitioned filer can't wedge the background loops (which run on a
// 30 s cadence). 20 s gives room for TLS handshakes on cold connections
// while leaving headroom before the next tick.
const filerRPCTimeout = 20 * time.Second

// registerOnce fans a MountRegister out to every configured filer in
// parallel. Returns an error only if every filer failed; otherwise the
// best-effort semantics let the mount proceed when some filer is down.
func (r *PeerRegistrar) registerOnce(ctx context.Context) error {
	if len(r.filerAddrs) == 0 {
		return fmt.Errorf("no filers configured")
	}
	ctx, cancel := context.WithTimeout(ctx, filerRPCTimeout)
	defer cancel()
	req := &filer_pb.MountRegisterRequest{
		PeerAddr:   r.selfPeerAddr,
		Rack:       r.selfRack,
		DataCenter: r.selfDc,
		TtlSeconds: int32(r.registerTTL / time.Second),
	}
	var wg sync.WaitGroup
	var successes atomic.Int32
	for _, addr := range r.filerAddrs {
		wg.Add(1)
		go func(addr pb.ServerAddress) {
			defer wg.Done()
			err := r.dialFiler(ctx, addr, func(c filer_pb.SeaweedFilerClient) error {
				_, err := c.MountRegister(ctx, req)
				return err
			})
			if err != nil {
				glog.V(2).Infof("MountRegister %s: %v", addr, err)
				return
			}
			successes.Add(1)
		}(addr)
	}
	wg.Wait()
	if successes.Load() == 0 {
		return fmt.Errorf("MountRegister failed on all %d filer(s)", len(r.filerAddrs))
	}
	return nil
}

// listOnce polls MountList from every filer in parallel and merges the
// responses by peer_addr (newest LastSeenNs wins). This way two mounts
// heartbeating through different filers still end up in each other's
// seed view as soon as at least one filer has been listed on each side.
func (r *PeerRegistrar) listOnce(ctx context.Context) error {
	if len(r.filerAddrs) == 0 {
		r.mu.Lock()
		r.seeds = nil
		r.mu.Unlock()
		return nil
	}
	ctx, cancel := context.WithTimeout(ctx, filerRPCTimeout)
	defer cancel()
	var (
		mu     sync.Mutex
		merged = map[string]*filer_pb.MountInfo{}
		fails  int
	)
	var wg sync.WaitGroup
	for _, addr := range r.filerAddrs {
		wg.Add(1)
		go func(addr pb.ServerAddress) {
			defer wg.Done()
			err := r.dialFiler(ctx, addr, func(c filer_pb.SeaweedFilerClient) error {
				resp, err := c.MountList(ctx, &filer_pb.MountListRequest{})
				if err != nil {
					return err
				}
				mu.Lock()
				for _, m := range resp.Mounts {
					if prev, ok := merged[m.PeerAddr]; !ok || m.LastSeenNs > prev.LastSeenNs {
						merged[m.PeerAddr] = m
					}
				}
				mu.Unlock()
				return nil
			})
			if err != nil {
				mu.Lock()
				fails++
				mu.Unlock()
				glog.V(2).Infof("MountList %s: %v", addr, err)
			}
		}(addr)
	}
	wg.Wait()

	if fails == len(r.filerAddrs) {
		return fmt.Errorf("MountList failed on all %d filer(s)", len(r.filerAddrs))
	}

	next := make([]SeedPeer, 0, len(merged))
	for _, m := range merged {
		next = append(next, SeedPeer{PeerAddr: m.PeerAddr, DataCenter: m.DataCenter, Rack: m.Rack})
	}
	r.mu.Lock()
	r.seeds = next
	r.mu.Unlock()
	return nil
}

func (r *PeerRegistrar) loopRegister() {
	t := time.NewTicker(r.registerInterval)
	defer t.Stop()
	for {
		select {
		case <-r.stopCtx.Done():
			return
		case <-t.C:
			if err := r.registerOnce(r.stopCtx); err != nil {
				glog.V(2).Infof("MountRegister heartbeat: %v", err)
			}
		}
	}
}

func (r *PeerRegistrar) loopList() {
	t := time.NewTicker(r.listInterval)
	defer t.Stop()
	for {
		select {
		case <-r.stopCtx.Done():
			return
		case <-t.C:
			if err := r.listOnce(r.stopCtx); err != nil {
				glog.V(2).Infof("MountList refresh: %v", err)
			}
		}
	}
}