mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
163 lines
5.0 KiB
Go
163 lines
5.0 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
)
|
|
|
|
func (vs *VolumeServer) ScrubVolume(ctx context.Context, req *volume_server_pb.ScrubVolumeRequest) (*volume_server_pb.ScrubVolumeResponse, error) {
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return nil, err
|
|
}
|
|
vids := []needle.VolumeId{}
|
|
if len(req.GetVolumeIds()) == 0 {
|
|
for _, l := range vs.store.Locations {
|
|
vids = append(vids, l.VolumeIds()...)
|
|
}
|
|
} else {
|
|
for _, vid := range req.GetVolumeIds() {
|
|
vids = append(vids, needle.VolumeId(vid))
|
|
}
|
|
}
|
|
|
|
var details []string
|
|
var totalVolumes, totalFiles uint64
|
|
var brokenVolumes []*storage.Volume
|
|
var brokenVolumeIds []uint32
|
|
for _, vid := range vids {
|
|
v := vs.store.GetVolume(vid)
|
|
if v == nil {
|
|
return nil, fmt.Errorf("volume id %d not found", vid)
|
|
}
|
|
|
|
var files int64
|
|
var serrs []error
|
|
switch m := req.GetMode(); m {
|
|
case volume_server_pb.VolumeScrubMode_INDEX:
|
|
files, serrs = v.ScrubIndex()
|
|
case volume_server_pb.VolumeScrubMode_LOCAL:
|
|
// LOCAL is equivalent to FULL for regular volumes
|
|
fallthrough
|
|
case volume_server_pb.VolumeScrubMode_FULL:
|
|
files, serrs = v.Scrub()
|
|
default:
|
|
return nil, fmt.Errorf("unsupported volume scrub mode %d", m)
|
|
}
|
|
|
|
totalVolumes += 1
|
|
totalFiles += uint64(files)
|
|
if len(serrs) != 0 {
|
|
brokenVolumes = append(brokenVolumes, v)
|
|
brokenVolumeIds = append(brokenVolumeIds, uint32(v.Id))
|
|
for _, err := range serrs {
|
|
details = append(details, err.Error())
|
|
}
|
|
}
|
|
}
|
|
|
|
errs := []error{}
|
|
if req.GetMarkBrokenVolumesReadonly() {
|
|
for _, v := range brokenVolumes {
|
|
if err := vs.makeVolumeReadonly(ctx, v, true); err != nil {
|
|
errs = append(errs, err)
|
|
details = append(details, err.Error())
|
|
} else {
|
|
details = append(details, fmt.Sprintf("volume %d is now read-only", v.Id))
|
|
}
|
|
}
|
|
}
|
|
|
|
scrubLabels := prometheus.Labels{"mode": req.GetMode().String()}
|
|
stats.VolumeServerScrubLastTimeSeconds.With(scrubLabels).Set(float64(time.Now().Unix()))
|
|
stats.VolumeServerScrubVolumeFailures.With(scrubLabels).Add(float64(len(brokenVolumes)))
|
|
|
|
if len(errs) != 0 {
|
|
return nil, errors.Join(errs...)
|
|
}
|
|
|
|
res := &volume_server_pb.ScrubVolumeResponse{
|
|
TotalVolumes: totalVolumes,
|
|
TotalFiles: totalFiles,
|
|
BrokenVolumeIds: brokenVolumeIds,
|
|
Details: details,
|
|
}
|
|
return res, nil
|
|
}
|
|
|
|
func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb.ScrubEcVolumeRequest) (*volume_server_pb.ScrubEcVolumeResponse, error) {
|
|
vids := []needle.VolumeId{}
|
|
if len(req.GetVolumeIds()) == 0 {
|
|
for _, l := range vs.store.Locations {
|
|
vids = append(vids, l.EcVolumeIds()...)
|
|
}
|
|
} else {
|
|
for _, vid := range req.GetVolumeIds() {
|
|
vids = append(vids, needle.VolumeId(vid))
|
|
}
|
|
}
|
|
|
|
var details []string
|
|
var totalVolumes, totalFiles uint64
|
|
var brokenVolumeIds []uint32
|
|
var brokenShardInfos []*volume_server_pb.EcShardInfo
|
|
for _, vid := range vids {
|
|
v, found := vs.store.FindEcVolume(vid)
|
|
if !found {
|
|
return nil, fmt.Errorf("EC volume id %d not found", vid)
|
|
}
|
|
|
|
var files int64
|
|
var shardInfos []*volume_server_pb.EcShardInfo
|
|
var serrs []error
|
|
switch m := req.GetMode(); m {
|
|
case volume_server_pb.VolumeScrubMode_INDEX:
|
|
// index scrubs do not verify individual EC shards
|
|
files, serrs = v.ScrubIndex()
|
|
case volume_server_pb.VolumeScrubMode_LOCAL:
|
|
files, shardInfos, serrs = v.ScrubLocal()
|
|
case volume_server_pb.VolumeScrubMode_FULL:
|
|
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId)
|
|
case volume_server_pb.VolumeScrubMode_CHECKSUM:
|
|
// Verify each local shard's raw bytes against the bitrot sidecar,
|
|
// exercising cold parity shards. Read-only. ChecksumScrub's first
|
|
// return is blocks scanned, not files — discard it so TotalFiles
|
|
// (a needle/file count) isn't inflated by the block count.
|
|
_, shardInfos, serrs = v.ChecksumScrub()
|
|
default:
|
|
return nil, fmt.Errorf("unsupported EC volume scrub mode %d", m)
|
|
}
|
|
|
|
totalVolumes += 1
|
|
totalFiles += uint64(files)
|
|
if len(serrs) != 0 || len(shardInfos) != 0 {
|
|
brokenVolumeIds = append(brokenVolumeIds, uint32(v.VolumeId))
|
|
brokenShardInfos = append(brokenShardInfos, shardInfos...)
|
|
for _, err := range serrs {
|
|
details = append(details, err.Error())
|
|
}
|
|
}
|
|
}
|
|
|
|
scrubLabels := prometheus.Labels{"mode": req.GetMode().String()}
|
|
stats.VolumeServerScrubLastTimeSeconds.With(scrubLabels).Set(float64(time.Now().Unix()))
|
|
stats.VolumeServerScrubVolumeFailures.With(scrubLabels).Add(float64(len(brokenVolumeIds)))
|
|
stats.VolumeServerScrubShardFailures.With(scrubLabels).Add(float64(len(brokenShardInfos)))
|
|
|
|
res := &volume_server_pb.ScrubEcVolumeResponse{
|
|
TotalVolumes: totalVolumes,
|
|
TotalFiles: totalFiles,
|
|
BrokenVolumeIds: brokenVolumeIds,
|
|
BrokenShardInfos: brokenShardInfos,
|
|
Details: details,
|
|
}
|
|
return res, nil
|
|
}
|