mirror of
https://github.com/hanzoai/s3.git
synced 2026-11-02 13:11:24 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
88 lines
2.9 KiB
Python
88 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Append rows to an existing Iceberg table via the SeaweedFS REST catalog.
|
|
|
|
Used by the Dremio integration test to materialize data files so a downstream
|
|
SELECT from Dremio verifies the read path against non-empty results.
|
|
|
|
Usage:
|
|
python3 append_rows.py \\
|
|
--catalog-url http://localhost:8181 \\
|
|
--warehouse s3://my-bucket \\
|
|
--prefix my-bucket \\
|
|
--s3-endpoint http://localhost:8333 \\
|
|
--access-key AKIA... --secret-key wJalr... \\
|
|
--namespace foo --namespace bar \\
|
|
--table events
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
import pyarrow as pa
|
|
from pyiceberg.catalog import load_catalog
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--catalog-url", required=True)
|
|
p.add_argument("--warehouse", required=True, help="s3://<bucket-name>")
|
|
p.add_argument("--prefix", required=True, help="REST catalog prefix (table bucket name)")
|
|
p.add_argument("--s3-endpoint", required=True, help="http://host:port")
|
|
p.add_argument("--access-key", required=True)
|
|
p.add_argument("--secret-key", required=True)
|
|
p.add_argument("--region", default="us-east-1")
|
|
p.add_argument(
|
|
"--namespace",
|
|
action="append",
|
|
required=True,
|
|
help="One per level (e.g. --namespace foo --namespace bar for foo.bar).",
|
|
)
|
|
p.add_argument("--table", required=True)
|
|
args = p.parse_args()
|
|
|
|
# `credential` triggers OAuth2 client_credentials against
|
|
# <catalog_uri>/v1/oauth/tokens, matching the helper Go test uses for
|
|
# REST-API table creation. The s3.* keys are needed for parquet writes.
|
|
catalog = load_catalog(
|
|
"rest",
|
|
**{
|
|
"type": "rest",
|
|
"uri": args.catalog_url,
|
|
"warehouse": args.warehouse,
|
|
"prefix": args.prefix,
|
|
"credential": f"{args.access_key}:{args.secret_key}",
|
|
"s3.access-key-id": args.access_key,
|
|
"s3.secret-access-key": args.secret_key,
|
|
"s3.endpoint": args.s3_endpoint,
|
|
"s3.region": args.region,
|
|
"s3.path-style-access": "true",
|
|
},
|
|
)
|
|
|
|
table_id = tuple(args.namespace) + (args.table,)
|
|
table = catalog.load_table(table_id)
|
|
|
|
# Match the Iceberg table schema: id is `required long`, label is
|
|
# `optional string`. Default pyarrow columns are nullable, which fails
|
|
# PyIceberg's required-field compatibility check.
|
|
arrow_schema = pa.schema(
|
|
[
|
|
pa.field("id", pa.int64(), nullable=False),
|
|
pa.field("label", pa.string(), nullable=True),
|
|
]
|
|
)
|
|
arrow_table = pa.Table.from_pydict(
|
|
{
|
|
"id": [1, 2, 3],
|
|
"label": ["one", "two", "three"],
|
|
},
|
|
schema=arrow_schema,
|
|
)
|
|
table.append(arrow_table)
|
|
print(f"appended {arrow_table.num_rows} rows to {'.'.join(table_id)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|