mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zeekay
e1e0356e75
filer, master, iam, plugin and worker no longer use google.golang.org/grpc on
the client path.
- gRPC-free client interfaces: the generated *_grpc.pb.go client interfaces drop
"opts ...grpc.CallOption" (no caller ever passed one) and their streaming
methods return the shared seam types in s3/pb/rpc (ServerStream[T] /
BidiStream[Req,Resp]) instead of grpc.ServerStreamingClient/BidiStreamingClient.
The gRPC server scaffolding (Unimplemented*Server, Register*Server, ServiceDesc,
handler trampolines) is deleted; servers serve over the ZAP wire Dispatch.
- The ZAP adapters (filerzap/masterzap/...) implement the gRPC-free interfaces;
grpc.DialOption becomes pb.DialOption (the dial is the ZAP transport).
- Unary calls honor context deadlines: the wire client threads ctx to
transport.Conn.CallContext (v1.8.1), so a call to an unresponsive peer aborts
on the caller's deadline instead of blocking forever.
- Error semantics: the filer tags each failure with its code name ("NotFound: …",
"Internal: …") in the message, which crosses the ZAP wire and is classified by
mount/error_classifier.go — replacing grpc status/codes.
- master ZAP listener port: grpcPort+10000 could exceed 65535; made overflow-safe.
- test doubles updated to the gRPC-free interfaces (0 tests skipped).
- dead cmd/s3-db and cmd/s3-sql (postgres-wire / SQL-engine entry points whose
backends were already ripped) deleted; finished the backend-rip import cleanup.
go build ./... green; go test ./s3/... green (heavy stream benches run under
their existing -short gates).
Still on gRPC (next wave): volume_server + mq (reverted — servers/adapters not
ready), the remaining ~23 plumbing files, and the go.mod dependency itself.
128 lines
3.8 KiB
Go
128 lines
3.8 KiB
Go
package erasure_coding_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/hanzoai/s3/s3/pb"
|
|
|
|
"github.com/hanzoai/s3/s3/pb/master_pb"
|
|
"github.com/hanzoai/s3/s3/pb/plugin_pb"
|
|
pluginworker "github.com/hanzoai/s3/s3/plugin/worker"
|
|
"github.com/hanzoai/s3/s3/worker/tasks/erasure_coding"
|
|
pluginworkers "github.com/hanzoai/s3/test/plugin_workers"
|
|
)
|
|
|
|
func TestErasureCodingDetectionLargeTopology(t *testing.T) {
|
|
const (
|
|
rackCount = 100
|
|
serverCount = 1000
|
|
volumesPerNode = 300
|
|
volumeSizeLimit = uint64(100)
|
|
)
|
|
|
|
if serverCount%rackCount != 0 {
|
|
t.Fatalf("serverCount (%d) must be divisible by rackCount (%d)", serverCount, rackCount)
|
|
}
|
|
|
|
nodesPerRack := serverCount / rackCount
|
|
// Eligible volumes must exceed the default fullness (0.95) and quiet (1h)
|
|
// thresholds; ineligible ones fall below the fullness threshold.
|
|
eligibleSize := uint64(96) * 1024 * 1024
|
|
ineligibleSize := uint64(10) * 1024 * 1024
|
|
modifiedAt := time.Now().Add(-2 * time.Hour).Unix()
|
|
|
|
volumeID := uint32(1)
|
|
dataCenters := make([]*master_pb.DataCenterInfo, 0, 1)
|
|
|
|
racks := make([]*master_pb.RackInfo, 0, rackCount)
|
|
for rack := 0; rack < rackCount; rack++ {
|
|
nodes := make([]*master_pb.DataNodeInfo, 0, nodesPerRack)
|
|
for node := 0; node < nodesPerRack; node++ {
|
|
address := fmt.Sprintf("10.0.%d.%d:8080", rack, node+1)
|
|
volumes := make([]*master_pb.VolumeInformationMessage, 0, volumesPerNode)
|
|
for v := 0; v < volumesPerNode; v++ {
|
|
size := ineligibleSize
|
|
if volumeID%2 == 0 {
|
|
size = eligibleSize
|
|
}
|
|
volumes = append(volumes, &master_pb.VolumeInformationMessage{
|
|
Id: volumeID,
|
|
Collection: "ec-bulk",
|
|
DiskId: 0,
|
|
Size: size,
|
|
DeletedByteCount: 0,
|
|
ModifiedAtSecond: modifiedAt,
|
|
ReplicaPlacement: 1,
|
|
ReadOnly: false,
|
|
})
|
|
volumeID++
|
|
}
|
|
|
|
diskInfo := &master_pb.DiskInfo{
|
|
DiskId: 0,
|
|
MaxVolumeCount: int64(volumesPerNode + 10),
|
|
VolumeCount: int64(volumesPerNode),
|
|
VolumeInfos: volumes,
|
|
}
|
|
|
|
nodes = append(nodes, &master_pb.DataNodeInfo{
|
|
Id: address,
|
|
Address: address,
|
|
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
|
|
})
|
|
}
|
|
|
|
racks = append(racks, &master_pb.RackInfo{
|
|
Id: fmt.Sprintf("rack-%d", rack+1),
|
|
DataNodeInfos: nodes,
|
|
})
|
|
}
|
|
|
|
dataCenters = append(dataCenters, &master_pb.DataCenterInfo{
|
|
Id: "dc-1",
|
|
RackInfos: racks,
|
|
})
|
|
|
|
response := &master_pb.VolumeListResponse{
|
|
VolumeSizeLimitMb: volumeSizeLimit,
|
|
TopologyInfo: &master_pb.TopologyInfo{
|
|
DataCenterInfos: dataCenters,
|
|
},
|
|
}
|
|
|
|
master := pluginworkers.NewMasterServer(t, response)
|
|
|
|
dialOption := pb.DialOption{}
|
|
handler := erasure_coding.NewErasureCodingHandler(dialOption, t.TempDir())
|
|
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
|
|
WorkerOptions: pluginworker.WorkerOptions{
|
|
GrpcDialOption: dialOption,
|
|
},
|
|
Handlers: []pluginworker.JobHandler{handler},
|
|
})
|
|
harness.WaitForJobType("erasure_coding")
|
|
|
|
totalVolumes := serverCount * volumesPerNode
|
|
expectedEligible := totalVolumes / 2
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
|
|
start := time.Now()
|
|
proposals, err := harness.Plugin().RunDetection(ctx, "erasure_coding", &plugin_pb.ClusterContext{
|
|
MasterGrpcAddresses: []string{master.Address()},
|
|
}, 0)
|
|
duration := time.Since(start)
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(proposals), 10, "should detect at least some proposals")
|
|
t.Logf("large topology detection completed in %s (proposals=%d, eligible=%d)", duration, len(proposals), expectedEligible)
|
|
if len(proposals) < expectedEligible {
|
|
t.Logf("large topology detection stopped early: %d proposals vs %d eligible", len(proposals), expectedEligible)
|
|
}
|
|
}
|