s3/weed/server/filer_grpc_server_stream_mutate.go

package weed_server

import (
	"context"
	"fmt"
	"io"
	"strings"
	"sync"
	"syscall"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// streamMutateConcurrency bounds the number of in-flight mutations processed
// concurrently per client stream. Set to the typical filer-store sweet spot
// so one noisy mount cannot exhaust filer resources.
const streamMutateConcurrency = 64

// streamMutatePendingLimit caps total outstanding requests per stream
// (admitted + waiting-for-admission). Prevents goroutine explosion when a
// client floods requests against a conflicted path while leaving enough
// headroom that cross-path requests never block Recv in practice.
const streamMutatePendingLimit = 1024

// syncStream wraps a bidi stream so that concurrent goroutines can Send
// without interleaving frames. gRPC requires that Send not be called
// concurrently; this mutex is the serialization point.
type syncStream struct {
	stream grpc.BidiStreamingServer[filer_pb.StreamMutateEntryRequest, filer_pb.StreamMutateEntryResponse]
	mu     sync.Mutex
}

func (s *syncStream) Send(r *filer_pb.StreamMutateEntryResponse) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.stream.Send(r)
}

func (fs *FilerServer) StreamMutateEntry(stream grpc.BidiStreamingServer[filer_pb.StreamMutateEntryRequest, filer_pb.StreamMutateEntryResponse]) error {
	ss := &syncStream{stream: stream}
	// Path-keyed admission + subtree barriers, adapted from filer.sync's
	// MetadataProcessor (weed/command/filer_sync_jobs.go). Admit blocks when
	// a new request conflicts with an in-flight one on the same path or with
	// a barrier directory at the same path / an ancestor.
	sched := newMutateScheduler(streamMutateConcurrency)
	// pendingSem caps goroutines-per-stream. The receive loop only blocks on
	// this sem (not on Admit), so one conflicted path cannot head-of-line
	// block receipt of unrelated paths — later distinct-path requests can be
	// spawned, admitted, and processed while the conflicted request waits.
	pendingSem := make(chan struct{}, streamMutatePendingLimit)
	var wg sync.WaitGroup
	// Track the first fatal send error across worker goroutines.
	var sendErrMu sync.Mutex
	var sendErr error
	setSendErr := func(e error) {
		sendErrMu.Lock()
		if sendErr == nil {
			sendErr = e
		}
		sendErrMu.Unlock()
	}
	getSendErr := func() error {
		sendErrMu.Lock()
		defer sendErrMu.Unlock()
		return sendErr
	}

	for {
		if e := getSendErr(); e != nil {
			wg.Wait()
			return e
		}
		req, err := stream.Recv()
		if err == io.EOF {
			wg.Wait()
			return getSendErr()
		}
		if err != nil {
			wg.Wait()
			return err
		}

		primary, secondary, kind := classifyMutation(req)
		pendingSem <- struct{}{}
		wg.Add(1)
		go func(req *filer_pb.StreamMutateEntryRequest, p, s util.FullPath, k mutateJobKind) {
			defer wg.Done()
			defer func() { <-pendingSem }()
			// Admission happens off the Recv loop so a conflicted path never
			// blocks receipt of unrelated requests.
			sched.Admit(p, s, k)
			defer sched.Done(p, s, k)
			if e := fs.handleStreamMutateRequest(ss, req); e != nil {
				setSendErr(e)
			}
		}(req, primary, secondary, kind)
	}
}

// handleStreamMutateRequest processes one request and sends exactly one
// (possibly multi-message) response via the shared sync stream. It returns
// a non-nil error only if Send fails — in which case the caller should
// tear down the stream.
func (fs *FilerServer) handleStreamMutateRequest(ss *syncStream, req *filer_pb.StreamMutateEntryRequest) error {
	switch r := req.Request.(type) {

	case *filer_pb.StreamMutateEntryRequest_CreateRequest:
		resp, createErr := fs.CreateEntry(ss.stream.Context(), r.CreateRequest)
		if createErr != nil {
			resp = &filer_pb.CreateEntryResponse{Error: createErr.Error()}
		}
		out := &filer_pb.StreamMutateEntryResponse{
			RequestId: req.RequestId,
			IsLast:    true,
			Response:  &filer_pb.StreamMutateEntryResponse_CreateResponse{CreateResponse: resp},
		}
		if resp.Error != "" {
			out.Error = resp.Error
			out.Errno = int32(syscall.EIO)
		}
		return ss.Send(out)

	case *filer_pb.StreamMutateEntryRequest_UpdateRequest:
		resp, updateErr := fs.UpdateEntry(ss.stream.Context(), r.UpdateRequest)
		if updateErr != nil {
			resp = &filer_pb.UpdateEntryResponse{}
		}
		out := &filer_pb.StreamMutateEntryResponse{
			RequestId: req.RequestId,
			IsLast:    true,
			Response:  &filer_pb.StreamMutateEntryResponse_UpdateResponse{UpdateResponse: resp},
		}
		if updateErr != nil {
			out.Error = updateErr.Error()
			out.Errno = int32(syscall.EIO)
		}
		return ss.Send(out)

	case *filer_pb.StreamMutateEntryRequest_DeleteRequest:
		resp, deleteErr := fs.DeleteEntry(ss.stream.Context(), r.DeleteRequest)
		if deleteErr != nil {
			resp = &filer_pb.DeleteEntryResponse{Error: deleteErr.Error()}
		}
		out := &filer_pb.StreamMutateEntryResponse{
			RequestId: req.RequestId,
			IsLast:    true,
			Response:  &filer_pb.StreamMutateEntryResponse_DeleteResponse{DeleteResponse: resp},
		}
		if resp.Error != "" {
			out.Error = resp.Error
			out.Errno = int32(syscall.EIO)
		}
		return ss.Send(out)

	case *filer_pb.StreamMutateEntryRequest_RenameRequest:
		return fs.handleStreamMutateRename(ss, req.RequestId, r.RenameRequest)

	default:
		// Send a terminal error response so the client's per-RequestId waiter
		// is released; returning nil here would leak the client-side waiter
		// forever when a future oneof variant or a malformed request arrives.
		glog.Warningf("StreamMutateEntry: unknown request type %T", req.Request)
		return ss.Send(&filer_pb.StreamMutateEntryResponse{
			RequestId: req.RequestId,
			IsLast:    true,
			Error:     "unknown request type",
			Errno:     int32(syscall.EINVAL),
		})
	}
}

// handleStreamMutateRename delegates to the existing StreamRenameEntry logic
// using a proxy stream that converts StreamRenameEntryResponse events into
// StreamMutateEntryResponse messages on the parent bidi stream.
func (fs *FilerServer) handleStreamMutateRename(
	parent *syncStream,
	requestId uint64,
	req *filer_pb.StreamRenameEntryRequest,
) error {
	proxy := &renameStreamProxy{parent: parent, requestId: requestId}
	renameErr := fs.StreamRenameEntry(req, proxy)
	// Always send a final is_last=true to signal rename completion.
	finalResp := &filer_pb.StreamMutateEntryResponse{
		RequestId: requestId,
		IsLast:    true,
		Response: &filer_pb.StreamMutateEntryResponse_RenameResponse{
			RenameResponse: &filer_pb.StreamRenameEntryResponse{},
		},
	}
	if renameErr != nil {
		finalResp.Error = renameErr.Error()
		finalResp.Errno = renameErrno(renameErr)
		glog.V(0).Infof("StreamMutateEntry rename: %v", renameErr)
	}
	return parent.Send(finalResp)
}

// renameStreamProxy adapts the bidi StreamMutateEntry stream to look like a
// SeaweedFiler_StreamRenameEntryServer, which is what StreamRenameEntry and
// moveEntry expect. Each Send() call forwards the response as a non-final
// StreamMutateEntryResponse.
type renameStreamProxy struct {
	parent    *syncStream
	requestId uint64
}

func (p *renameStreamProxy) Send(resp *filer_pb.StreamRenameEntryResponse) error {
	return p.parent.Send(&filer_pb.StreamMutateEntryResponse{
		RequestId: p.requestId,
		IsLast:    false,
		Response:  &filer_pb.StreamMutateEntryResponse_RenameResponse{RenameResponse: resp},
	})
}

func (p *renameStreamProxy) Context() context.Context {
	return p.parent.stream.Context()
}

// SendMsg routes through Send so the payload is wrapped into a
// StreamMutateEntryResponse and goes through the syncStream mutex. Calling
// SendMsg with anything other than *filer_pb.StreamRenameEntryResponse would
// emit the wrong protobuf type on this RPC, so reject other shapes.
func (p *renameStreamProxy) SendMsg(m any) error {
	resp, ok := m.(*filer_pb.StreamRenameEntryResponse)
	if !ok {
		return fmt.Errorf("renameStreamProxy.SendMsg: unexpected type %T", m)
	}
	return p.Send(resp)
}

// RecvMsg on the proxy would race with the outer StreamMutateEntry recv
// loop and could steal unrelated mutation requests. The rename logic never
// calls RecvMsg (it is strictly a server-push stream), so fail loudly if it
// ever does.
func (p *renameStreamProxy) RecvMsg(m any) error {
	return fmt.Errorf("renameStreamProxy.RecvMsg is not supported")
}

func (p *renameStreamProxy) SetHeader(md metadata.MD) error  { return p.parent.stream.SetHeader(md) }
func (p *renameStreamProxy) SendHeader(md metadata.MD) error { return p.parent.stream.SendHeader(md) }
func (p *renameStreamProxy) SetTrailer(md metadata.MD)       { p.parent.stream.SetTrailer(md) }

// renameErrno maps a rename error to a POSIX errno for the client.
func renameErrno(err error) int32 {
	msg := err.Error()
	switch {
	case strings.Contains(msg, "not found"):
		return int32(syscall.ENOENT)
	case strings.Contains(msg, "not empty"):
		return int32(syscall.ENOTEMPTY)
	case strings.Contains(msg, "not directory"):
		return int32(syscall.ENOTDIR)
	default:
		return int32(syscall.EIO)
	}
}