mirror of
https://github.com/hanzoai/s3.git
synced 2026-11-03 03:07:41 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
261 lines
8.4 KiB
Go
261 lines
8.4 KiB
Go
package page_writer
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
)
|
|
|
|
func TestUploadPipeline(t *testing.T) {
|
|
|
|
uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16, "", nil)
|
|
|
|
writeRange(uploadPipeline, 0, 131072)
|
|
writeRange(uploadPipeline, 131072, 262144)
|
|
writeRange(uploadPipeline, 262144, 1025536)
|
|
|
|
confirmRange(t, uploadPipeline, 0, 1025536)
|
|
|
|
writeRange(uploadPipeline, 1025536, 1296896)
|
|
|
|
confirmRange(t, uploadPipeline, 1025536, 1296896)
|
|
|
|
writeRange(uploadPipeline, 1296896, 2162688)
|
|
|
|
confirmRange(t, uploadPipeline, 1296896, 2162688)
|
|
|
|
confirmRange(t, uploadPipeline, 1296896, 2162688)
|
|
}
|
|
|
|
// startOff and stopOff must be divided by 4
|
|
func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
|
|
p := make([]byte, 4)
|
|
for i := startOff / 4; i < stopOff/4; i += 4 {
|
|
util.Uint32toBytes(p, uint32(i))
|
|
uploadPipeline.SaveDataAt(p, i, false, 0)
|
|
}
|
|
}
|
|
|
|
func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
|
|
p := make([]byte, 4)
|
|
for i := startOff; i < stopOff/4; i += 4 {
|
|
uploadPipeline.MaybeReadDataAt(p, i, 0)
|
|
x := util.BytesToUint32(p)
|
|
if x != uint32(i) {
|
|
t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Pressure-driven eviction must not seal a chunk with a leading or
|
|
// internal gap (issue #9330).
|
|
func TestEvictOneWritableChunk_SkipsGappyChunks(t *testing.T) {
|
|
const cs int64 = 2 * 1024 * 1024
|
|
// nil saveToStorage so the async upload SaveContent is a no-op.
|
|
up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
|
|
|
|
block := make([]byte, cs/4)
|
|
|
|
// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
|
|
if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !up.EvictOneWritableChunk() {
|
|
t.Fatalf("EvictOneWritableChunk returned false; expected contiguous chunk 2 to be evictable")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
|
|
t.Errorf("chunk 2 should have moved to sealed")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
|
|
t.Errorf("chunk 0 (internal gap) must remain writable")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
|
|
t.Errorf("chunk 1 (leading gap) must remain writable")
|
|
}
|
|
|
|
// Filling holes makes IsComplete true; maybeMoveToSealed auto-seals.
|
|
if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
|
|
t.Errorf("chunk 0 should have auto-sealed after gap was filled")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
|
|
t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
|
|
}
|
|
}
|
|
|
|
// When every chunk is gappy, the fallback must still seal one so
|
|
// accountant.Reserve can wake; oldest-LastWriteTsNs wins.
|
|
func TestEvictOneWritableChunk_FallbackPicksOldestGappy(t *testing.T) {
|
|
const cs int64 = 2 * 1024 * 1024
|
|
up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
|
|
|
|
block := make([]byte, cs/4)
|
|
|
|
// Strictly increasing tsNs so chunk 0 is oldest; equal WrittenSize.
|
|
if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/4, true, 200); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/2, true, 201); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs, true, 300); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs+3*cs/4, true, 301); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !up.EvictOneWritableChunk() {
|
|
t.Fatalf("fallback must seal a gappy chunk to free a Reserve slot")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
|
|
t.Errorf("oldest gappy chunk (0) should have been picked by the fallback")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
|
|
t.Errorf("chunk 1 should remain writable")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; !stillWritable {
|
|
t.Errorf("chunk 2 should remain writable")
|
|
}
|
|
}
|
|
|
|
// Strict pass preempts the fallback even when a gappy chunk is older.
|
|
func TestEvictOneWritableChunk_PrefersStrictOverFallback(t *testing.T) {
|
|
const cs int64 = 2 * 1024 * 1024
|
|
up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
|
|
|
|
block := make([]byte, cs/4)
|
|
|
|
// chunk 0 (older, gappy), chunk 1 (newer, contiguous from 0).
|
|
if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs, true, 200); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/4, true, 201); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !up.EvictOneWritableChunk() {
|
|
t.Fatalf("EvictOneWritableChunk returned false")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
|
|
t.Errorf("contiguous chunk 1 must be picked over older gappy chunk 0")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
|
|
t.Errorf("gappy chunk 0 must remain writable")
|
|
}
|
|
}
|
|
|
|
// ProactiveFlush also skips gappy chunks (issue #9330). No liveness
|
|
// fallback here — unlike EvictOneWritableChunk, returning false is just
|
|
// a missed optimization.
|
|
func TestProactiveFlush_SkipsGappyChunks(t *testing.T) {
|
|
const cs int64 = 2 * 1024 * 1024
|
|
up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
|
|
|
|
block := make([]byte, cs/4)
|
|
|
|
// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
|
|
if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// nowNs >> chunk timestamps so age clears the idle/maxHold gates;
|
|
// fillRatio < WrittenSize so nearlyFull is true. Leaves
|
|
// IsContiguouslyWritten as the only differentiator.
|
|
const (
|
|
nowNs int64 = 1_000_000_000
|
|
idleThresholdNs int64 = 100
|
|
maxHoldNs int64 = 200
|
|
fillRatio int64 = cs / 8
|
|
)
|
|
if !up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
|
|
t.Fatalf("ProactiveFlush returned false; expected contiguous chunk 2 to be sealed")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
|
|
t.Errorf("chunk 2 should have moved to sealed")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
|
|
t.Errorf("chunk 0 (internal gap) must remain writable")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
|
|
t.Errorf("chunk 1 (leading gap) must remain writable")
|
|
}
|
|
|
|
if up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
|
|
t.Errorf("ProactiveFlush returned true with only gappy chunks left")
|
|
}
|
|
|
|
if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
|
|
t.Errorf("chunk 0 should have auto-sealed after gap was filled")
|
|
}
|
|
if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
|
|
t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
|
|
}
|
|
}
|