mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
63 lines
2.6 KiB
Python
63 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
# Deterministic ~1GB SQLite load on the FUSE mount.
|
|
# Each row: id, payload (4096 deterministic bytes derived from id), chk=crc32(payload).
|
|
# Commits in batches; after each commit, writes the committed row count to a
|
|
# progress file on LOCAL disk and fsyncs it -> a durable record of what was
|
|
# committed, used to verify "no committed data lost" after a crash.
|
|
import sys, os, sqlite3, zlib, random
|
|
|
|
db_path = sys.argv[1]
|
|
total_rows = int(sys.argv[2])
|
|
batch = int(sys.argv[3])
|
|
progress_path= sys.argv[4] # LOCAL disk
|
|
ref_path = sys.argv[5] if len(sys.argv) > 5 else None
|
|
sync_mode = sys.argv[6] if len(sys.argv) > 6 else "FULL"
|
|
journal_mode = sys.argv[7] if len(sys.argv) > 7 else "DELETE"
|
|
|
|
# Validate before interpolating into PRAGMAs: a typo must not silently weaken
|
|
# durability and invalidate the comparison / crash-survival claims.
|
|
if sync_mode.upper() not in {"OFF", "NORMAL", "FULL", "EXTRA"}:
|
|
raise SystemExit(f"invalid synchronous mode: {sync_mode!r}")
|
|
if journal_mode.upper() not in {"DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"}:
|
|
raise SystemExit(f"invalid journal mode: {journal_mode!r}")
|
|
|
|
def payload(i: int) -> bytes:
|
|
# deterministic but incompressible (PRNG seeded by id) -> 4096 bytes that
|
|
# actually consume ~1GB on the volume servers (defeats gzip storage).
|
|
return random.Random(i).randbytes(4096)
|
|
|
|
con = sqlite3.connect(db_path, isolation_level=None)
|
|
con.execute(f"PRAGMA journal_mode={journal_mode};")
|
|
con.execute(f"PRAGMA synchronous={sync_mode};")
|
|
con.execute("CREATE TABLE IF NOT EXISTS t(id INTEGER PRIMARY KEY, payload BLOB, chk INTEGER);")
|
|
|
|
def write_progress(n):
|
|
# atomic: write tmp + fsync + rename, so a kill -9 mid-write can't leave an
|
|
# empty/torn progress file (the crash-test lower bound must stay trustworthy).
|
|
tmp = progress_path + ".tmp"
|
|
with open(tmp, "w") as f:
|
|
f.write(str(n)); f.flush(); os.fsync(f.fileno())
|
|
os.replace(tmp, progress_path)
|
|
|
|
done = 0
|
|
sum_chk = 0
|
|
sum_id = 0
|
|
while done < total_rows:
|
|
end = min(done + batch, total_rows)
|
|
rows = []
|
|
for i in range(done, end):
|
|
p = payload(i); c = zlib.crc32(p)
|
|
rows.append((i, p, c)); sum_chk += c; sum_id += i
|
|
con.execute("BEGIN;")
|
|
con.executemany("INSERT INTO t(id,payload,chk) VALUES(?,?,?);", rows)
|
|
con.execute("COMMIT;")
|
|
done = end
|
|
write_progress(done)
|
|
print(f"committed {done}/{total_rows}", flush=True)
|
|
|
|
if ref_path:
|
|
with open(ref_path, "w") as f:
|
|
f.write(f"{total_rows} {sum_chk} {sum_id}\n"); f.flush(); os.fsync(f.fileno())
|
|
con.close()
|
|
print("DONE", flush=True)
|