mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zeekayandHanzo Dev
af0f387bbe
The binary is s3 and the product is Hanzo S3; the rest of the tree now says so too. - Environment: WEED_* and SEAWEEDFS_* are S3_*, no aliases. fla9's flag prefix is S3, matching viper's existing "s3" prefix, so the S3_S3_SSE_* names mini.go exports are the ones viper reads (WEED_S3_SSE_KEK was never seen once viper moved to "s3"). - Java and Hadoop: packages, fs.hanzofs.* keys and the hanzofs:// scheme under the hanzofs namespace the protos already declare; classes take a neutral Filer prefix and file names match them again (the earlier pass left Seas3*.java holding Seaweed* classes). Client, hdfs3, examples, s3copier, random_access and the Spark tests compile. - Helm chart: helpers are hanzo.*, values live under global.s3; the chart renders and lints, which it did not before. - Terraform: the s3.service.tftpl reference matches the file; 12/12 render tests pass. Config paths point at /etc/hanzo, which the binary reads. - loadtest.pb.go regenerated from loadtest.proto. - Upstream logo, sponsor images and unreferenced diagrams removed from note/. Kept: NOTICE attribution, LICENSE, the erasure-coding fixture s3/storage/erasure_coding/1.dat (test bytes), and unmaintained/ and third_party/ (vendored). No on-disk format, wire protocol or metadata key changed. Co-authored-by: Hanzo Dev <[email protected]>
240 lines
5.6 KiB
YAML
240 lines
5.6 KiB
YAML
name: admin_integration
|
|
|
|
networks:
|
|
hanzo_net:
|
|
driver: bridge
|
|
|
|
services:
|
|
master:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "9333:9333"
|
|
- "19333:19333"
|
|
command: "master -ip=master -mdir=/data -volumeSizeLimitMB=50"
|
|
environment:
|
|
- S3_MASTER_VOLUME_GROWTH_COPY_1=1
|
|
- S3_MASTER_VOLUME_GROWTH_COPY_2=2
|
|
- S3_MASTER_VOLUME_GROWTH_COPY_OTHER=1
|
|
volumes:
|
|
- ./data/master:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume1:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8080:8080"
|
|
- "18080:18080"
|
|
command: "volume -master=master:9333 -ip=volume1 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume1:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume2:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8081:8080"
|
|
- "18081:18080"
|
|
command: "volume -master=master:9333 -ip=volume2 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume2:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume3:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8082:8080"
|
|
- "18082:18080"
|
|
command: "volume -master=master:9333 -ip=volume3 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume3:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume4:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8083:8080"
|
|
- "18083:18080"
|
|
command: "volume -master=master:9333 -ip=volume4 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume4:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume5:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8084:8080"
|
|
- "18084:18080"
|
|
command: "volume -master=master:9333 -ip=volume5 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume5:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
volume6:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8085:8080"
|
|
- "18085:18080"
|
|
command: "volume -master=master:9333 -ip=volume6 -dir=/data -max=10"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/volume6:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
filer:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "8888:8888"
|
|
- "18888:18888"
|
|
command: "filer -master=master:9333 -ip=filer"
|
|
depends_on:
|
|
- master
|
|
volumes:
|
|
- ./data/filer:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
admin:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
ports:
|
|
- "23646:23646" # HTTP admin interface (default port)
|
|
- "33646:33646" # gRPC worker communication (23646 + 10000)
|
|
command: "-v=2 admin -port=23646 -masters=master:9333 -dataDir=/data"
|
|
depends_on:
|
|
- master
|
|
- filer
|
|
volumes:
|
|
- ./data/admin:/data
|
|
networks:
|
|
- hanzo_net
|
|
|
|
worker1:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
|
depends_on:
|
|
- admin
|
|
volumes:
|
|
- ./data/worker1:/data
|
|
networks:
|
|
- hanzo_net
|
|
environment:
|
|
- WORKER_ID=worker-1
|
|
|
|
worker2:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
|
depends_on:
|
|
- admin
|
|
volumes:
|
|
- ./data/worker2:/data
|
|
networks:
|
|
- hanzo_net
|
|
environment:
|
|
- WORKER_ID=worker-2
|
|
|
|
worker3:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
command: "-v=2 worker -admin=admin:23646 -capabilities=erasure_coding,vacuum -maxConcurrent=2"
|
|
depends_on:
|
|
- admin
|
|
volumes:
|
|
- ./data/worker3:/data
|
|
networks:
|
|
- hanzo_net
|
|
environment:
|
|
- WORKER_ID=worker-3
|
|
|
|
load_generator:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
entrypoint: ["/bin/sh"]
|
|
command: >
|
|
-c "
|
|
echo 'Starting load generator...';
|
|
sleep 30;
|
|
echo 'Generating continuous load with 50MB volume limit...';
|
|
while true; do
|
|
echo 'Writing test files...';
|
|
echo 'Test file content at $(date)' | /usr/bin/s3 upload -server=master:9333;
|
|
sleep 5;
|
|
echo 'Deleting some files...';
|
|
/usr/bin/s3 shell -master=master:9333 <<< 'fs.rm /test_file_*' || true;
|
|
sleep 10;
|
|
done
|
|
"
|
|
depends_on:
|
|
- master
|
|
- filer
|
|
- admin
|
|
networks:
|
|
- hanzo_net
|
|
|
|
monitor:
|
|
image: alpine:latest
|
|
entrypoint: ["/bin/sh"]
|
|
command: >
|
|
-c "
|
|
apk add --no-cache curl jq;
|
|
echo 'Starting cluster monitor...';
|
|
sleep 30;
|
|
while true; do
|
|
echo '=== Cluster Status $(date) ===';
|
|
echo 'Master status:';
|
|
curl -s http://master:9333/cluster/status | jq '.IsLeader, .Peers' || echo 'Master not ready';
|
|
echo;
|
|
echo 'Admin status:';
|
|
curl -s http://admin:23646/ | grep -o 'Admin.*Interface' || echo 'Admin not ready';
|
|
echo;
|
|
echo 'Volume count by server:';
|
|
curl -s http://master:9333/vol/status | jq '.Volumes | length' || echo 'Volumes not ready';
|
|
echo;
|
|
sleep 60;
|
|
done
|
|
"
|
|
depends_on:
|
|
- master
|
|
- admin
|
|
- filer
|
|
networks:
|
|
- hanzo_net
|
|
|
|
vacuum-tester:
|
|
image: ghcr.io/hanzoai/s3:local
|
|
entrypoint: ["/bin/sh"]
|
|
command: >
|
|
-c "
|
|
echo 'Installing dependencies for vacuum testing...';
|
|
apk add --no-cache jq curl go bash;
|
|
echo 'Vacuum tester ready...';
|
|
echo 'Use: docker-compose exec vacuum-tester sh';
|
|
echo 'Available commands: go, s3, curl, jq, bash, sh';
|
|
sleep infinity
|
|
"
|
|
depends_on:
|
|
- master
|
|
- admin
|
|
- filer
|
|
volumes:
|
|
- .:/testing
|
|
working_dir: /testing
|
|
networks:
|
|
- hanzo_net
|
|
environment:
|
|
- MASTER_HOST=master:9333
|
|
- ADMIN_HOST=admin:23646 |