mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
139 lines
4.0 KiB
Go
139 lines
4.0 KiB
Go
package storage
|
|
|
|
import (
|
|
"os"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
|
. "github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
"github.com/syndtr/goleveldb/leveldb/opt"
|
|
)
|
|
|
|
type NeedleMap struct {
|
|
baseNeedleMapper
|
|
m needle_map.NeedleValueMap
|
|
}
|
|
|
|
func NewCompactNeedleMap(file *os.File) *NeedleMap {
|
|
nm := &NeedleMap{
|
|
m: needle_map.NewCompactMap(),
|
|
}
|
|
nm.indexFile = file
|
|
stat, err := file.Stat()
|
|
if err != nil {
|
|
glog.Fatalf("stat file %s: %v", file.Name(), err)
|
|
}
|
|
nm.indexFileOffset = stat.Size()
|
|
return nm
|
|
}
|
|
|
|
func LoadCompactNeedleMap(file *os.File, version needle.Version) (*NeedleMap, error) {
|
|
nm := NewCompactNeedleMap(file)
|
|
return doLoading(file, nm, version)
|
|
}
|
|
|
|
func doLoading(file *os.File, nm *NeedleMap, version needle.Version) (*NeedleMap, error) {
|
|
e := idx.WalkIndexFile(file, 0, func(key NeedleId, offset Offset, size Size) error {
|
|
nm.MaybeSetMaxFileKey(key)
|
|
nm.MaybeSetMaxNeedleEnd(offset, size, version)
|
|
if !offset.IsZero() && !size.IsDeleted() {
|
|
nm.FileCounter++
|
|
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
|
|
oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
|
if !oldOffset.IsZero() && !oldSize.IsDeleted() {
|
|
nm.DeletionCounter++
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
|
}
|
|
} else {
|
|
oldSize := nm.m.Delete(NeedleId(key))
|
|
nm.DeletionCounter++
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
|
}
|
|
return nil
|
|
})
|
|
glog.V(1).Infof("max file key: %v count: %d deleted: %d for file: %s", nm.MaxFileKey(), nm.FileCount(), nm.DeletedCount(), file.Name())
|
|
return nm, e
|
|
}
|
|
|
|
func (nm *NeedleMap) Put(key NeedleId, offset Offset, size Size) error {
|
|
_, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
|
nm.logPut(key, oldSize, size)
|
|
return nm.appendToIndexFile(key, offset, size)
|
|
}
|
|
func (nm *NeedleMap) Get(key NeedleId) (element *needle_map.NeedleValue, ok bool) {
|
|
element, ok = nm.m.Get(NeedleId(key))
|
|
return
|
|
}
|
|
func (nm *NeedleMap) Delete(key NeedleId, offset Offset) error {
|
|
deletedBytes := nm.m.Delete(NeedleId(key))
|
|
nm.logDelete(deletedBytes)
|
|
return nm.appendToIndexFile(key, offset, TombstoneFileSize)
|
|
}
|
|
func (nm *NeedleMap) Close() {
|
|
if nm.indexFile == nil {
|
|
return
|
|
}
|
|
indexFileName := nm.indexFile.Name()
|
|
if err := nm.indexFile.Sync(); err != nil {
|
|
glog.Warningf("sync file %s failed, %v", indexFileName, err)
|
|
}
|
|
_ = nm.indexFile.Close()
|
|
}
|
|
func (nm *NeedleMap) Destroy() error {
|
|
nm.Close()
|
|
return os.Remove(nm.indexFile.Name())
|
|
}
|
|
|
|
func (nm *NeedleMap) UpdateNeedleMap(v *Volume, indexFile *os.File, opts *opt.Options, ldbTimeout int64) error {
|
|
if v.nm != nil {
|
|
v.nm.Close()
|
|
v.nm = nil
|
|
}
|
|
defer func() {
|
|
if v.tmpNm != nil {
|
|
v.tmpNm.Close()
|
|
v.tmpNm = nil
|
|
}
|
|
}()
|
|
nm.indexFile = indexFile
|
|
stat, err := indexFile.Stat()
|
|
if err != nil {
|
|
glog.Fatalf("stat file %s: %v", indexFile.Name(), err)
|
|
return err
|
|
}
|
|
nm.indexFileOffset = stat.Size()
|
|
v.nm = nm
|
|
v.tmpNm = nil
|
|
return nil
|
|
}
|
|
|
|
func (nm *NeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startFrom uint64) error {
|
|
glog.V(0).Infof("loading idx from offset %d for file: %s", startFrom, indexFile.Name())
|
|
version := needle.GetCurrentVersion()
|
|
if v != nil {
|
|
version = v.Version()
|
|
}
|
|
e := idx.WalkIndexFile(indexFile, startFrom, func(key NeedleId, offset Offset, size Size) error {
|
|
nm.MaybeSetMaxFileKey(key)
|
|
nm.MaybeSetMaxNeedleEnd(offset, size, version)
|
|
nm.FileCounter++
|
|
if !offset.IsZero() && !size.IsDeleted() {
|
|
nm.FileByteCounter = nm.FileByteCounter + uint64(size)
|
|
oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size)
|
|
if !oldOffset.IsZero() && !oldSize.IsDeleted() {
|
|
nm.DeletionCounter++
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
|
}
|
|
} else {
|
|
oldSize := nm.m.Delete(NeedleId(key))
|
|
nm.DeletionCounter++
|
|
nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize)
|
|
}
|
|
return nil
|
|
})
|
|
|
|
return e
|
|
}
|