s3/weed/operation/delete_content.go

package operation

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"strings"
	"sync"

	"github.com/seaweedfs/seaweedfs/weed/pb"
	"google.golang.org/grpc"

	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)

type DeleteResult struct {
	Fid    string `json:"fid"`
	Size   int    `json:"size"`
	Status int    `json:"status"`
	Error  string `json:"error,omitempty"`
}

func ParseFileId(fid string) (vid string, key_cookie string, err error) {
	commaIndex := strings.Index(fid, ",")
	if commaIndex <= 0 {
		return "", "", errors.New("Wrong fid format.")
	}
	return fid[:commaIndex], fid[commaIndex+1:], nil
}

// DeleteFileIds batch deletes a list of fileIds
// Returns individual results for each file ID. Check result.Error for per-file failures.
func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) []*volume_server_pb.DeleteResult {

	lookupFunc := func(vids []string) (results map[string]*LookupResult, err error) {
		results, err = LookupVolumeIds(masterFn, grpcDialOption, vids)
		if err == nil && usePublicUrl {
			for _, result := range results {
				for _, loc := range result.Locations {
					loc.Url = loc.PublicUrl
				}
			}
		}
		return
	}

	return DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)

}

func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {

	var ret []*volume_server_pb.DeleteResult

	vid_to_fileIds := make(map[string][]string)
	var vids []string
	for _, fileId := range fileIds {
		vid, _, err := ParseFileId(fileId)
		if err != nil {
			ret = append(ret, &volume_server_pb.DeleteResult{
				FileId: fileId,
				Status: http.StatusBadRequest,
				Error:  err.Error()},
			)
			continue
		}
		if _, ok := vid_to_fileIds[vid]; !ok {
			vid_to_fileIds[vid] = make([]string, 0)
			vids = append(vids, vid)
		}
		vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
	}

	lookupResults, err := lookupFunc(vids)
	if err != nil {
		// Lookup failed - return error results for all file IDs that passed parsing
		for _, fids := range vid_to_fileIds {
			for _, fileId := range fids {
				ret = append(ret, &volume_server_pb.DeleteResult{
					FileId: fileId,
					Status: http.StatusInternalServerError,
					Error:  fmt.Sprintf("lookup error: %v", err),
				})
			}
		}
		return ret
	}

	server_to_fileIds := make(map[pb.ServerAddress][]string)
	for vid, result := range lookupResults {
		if result.Error != "" {
			// Lookup error for this volume - mark all its files as failed
			for _, fileId := range vid_to_fileIds[vid] {
				ret = append(ret, &volume_server_pb.DeleteResult{
					FileId: fileId,
					Status: http.StatusBadRequest,
					Error:  result.Error},
				)
			}
			continue
		}
		for _, location := range result.Locations {
			serverAddress := location.ServerAddress()
			if _, ok := server_to_fileIds[serverAddress]; !ok {
				server_to_fileIds[serverAddress] = make([]string, 0)
			}
			server_to_fileIds[serverAddress] = append(
				server_to_fileIds[serverAddress], vid_to_fileIds[vid]...)
		}
	}

	resultChan := make(chan []*volume_server_pb.DeleteResult, len(server_to_fileIds))
	var wg sync.WaitGroup
	for server, fidList := range server_to_fileIds {
		wg.Add(1)
		go func(server pb.ServerAddress, fidList []string) {
			defer wg.Done()

			resultChan <- DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false)

		}(server, fidList)
	}
	wg.Wait()
	close(resultChan)

	for result := range resultChan {
		ret = append(ret, result...)
	}

	return ret
}

// DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
// Returns individual results for each file ID. Check result.Error for per-file failures.
func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {

	var ret []*volume_server_pb.DeleteResult

	err := WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

		req := &volume_server_pb.BatchDeleteRequest{
			FileIds:         fileIds,
			SkipCookieCheck: !includeCookie,
		}

		resp, err := volumeServerClient.BatchDelete(context.Background(), req)

		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)

		if err != nil {
			return err
		}

		ret = append(ret, resp.Results...)

		return nil
	})

	if err != nil {
		// Connection or communication error - return error results for all files
		ret = make([]*volume_server_pb.DeleteResult, 0, len(fileIds))
		for _, fileId := range fileIds {
			ret = append(ret, &volume_server_pb.DeleteResult{
				FileId: fileId,
				Status: http.StatusInternalServerError,
				Error:  err.Error(),
			})
		}
	}

	return ret

}