s3/test/kafka/docker-compose.yml at forge/feat/seaweedfs

mirror of https://github.com/hanzoai/s3.git synced 2026-10-30 14:59:09 +00:00

Files

T

zooqueen dd74d3e6d6 Replace AGPL MinIO fork with Apache-2.0 SeaweedFS fork; rebrand CLI weed->s3

hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.

- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
  "Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
  SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
  (replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
  keep their upstream import paths with zero churn. Internal identifiers and
  import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
  help/usage/flag strings, shell autocomplete program name, and the product name
  in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
  (e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
  `cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
  updated to ignore the `s3` build output.

2026-06-18 23:41:25 -07:00

327 lines

8.7 KiB

YAML

Raw Permalink Blame History

 x-seaweedfs-build: &seaweedfs-build
   build:
     context: ../..
     dockerfile: test/kafka/Dockerfile.seaweedfs
   image: kafka-seaweedfs-dev
 services:
   # Zookeeper for Kafka
   zookeeper:
     image: confluentinc/cp-zookeeper:7.4.0
     container_name: kafka-zookeeper
     ports:
       - "2181:2181"
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "2181"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 10s
     networks:
       - kafka-test-net
   # Kafka Broker
   kafka:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-broker
     ports:
       - "9092:9092"
       - "29092:29092"
     depends_on:
       zookeeper:
         condition: service_healthy
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
       KAFKA_NUM_PARTITIONS: 3
       KAFKA_DEFAULT_REPLICATION_FACTOR: 1
     healthcheck:
       test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 30s
     networks:
       - kafka-test-net
   # Schema Registry
   schema-registry:
     image: confluentinc/cp-schema-registry:7.4.0
     container_name: kafka-schema-registry
     ports:
       - "8081:8081"
     depends_on:
       kafka:
         condition: service_healthy
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
       SCHEMA_REGISTRY_DEBUG: "true"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 20s
     networks:
       - kafka-test-net
   # SeaweedFS Master
   seaweedfs-master:
     <<: *seaweedfs-build
     container_name: seaweedfs-master
     ports:
       - "9333:9333"
       - "19333:19333"  # gRPC port
     command:
       - master
       - -ip=seaweedfs-master
       - -port=9333
       - -port.grpc=19333
       - -volumeSizeLimitMB=1024
       - -defaultReplication=000
     volumes:
       - seaweedfs-master-data:/data
     healthcheck:
       test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://seaweedfs-master:9333/cluster/status || curl -sf http://seaweedfs-master:9333/cluster/status"]
       interval: 10s
       timeout: 5s
       retries: 10
       start_period: 20s
     networks:
       - kafka-test-net
   # SeaweedFS Volume Server
   seaweedfs-volume:
     <<: *seaweedfs-build
     container_name: seaweedfs-volume
     ports:
       - "8080:8080"
       - "18080:18080"  # gRPC port
     command:
       - volume
       - -master=seaweedfs-master:9333
       - -ip=seaweedfs-volume
       - -port=8080
       - -port.grpc=18080
       - -publicUrl=seaweedfs-volume:8080
       - -preStopSeconds=1
     depends_on:
       seaweedfs-master:
         condition: service_healthy
     volumes:
       - seaweedfs-volume-data:/data
     healthcheck:
       test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-volume:8080/status"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 10s
     networks:
       - kafka-test-net
   # SeaweedFS Filer
   seaweedfs-filer:
     <<: *seaweedfs-build
     container_name: seaweedfs-filer
     ports:
       - "8888:8888"
       - "18888:18888"  # gRPC port
     command:
       - filer
       - -master=seaweedfs-master:9333
       - -ip=seaweedfs-filer
       - -port=8888
       - -port.grpc=18888
     depends_on:
       seaweedfs-master:
         condition: service_healthy
       seaweedfs-volume:
         condition: service_healthy
     volumes:
       - seaweedfs-filer-data:/data
     healthcheck:
       test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://seaweedfs-filer:8888/"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
     networks:
       - kafka-test-net
   # SeaweedFS MQ Broker
   seaweedfs-mq-broker:
     <<: *seaweedfs-build
     container_name: seaweedfs-mq-broker
     ports:
       - "17777:17777"  # MQ Broker port
       - "18777:18777"  # pprof profiling port
     command:
       - mq.broker
       - -master=seaweedfs-master:9333
       - -ip=seaweedfs-mq-broker
       - -port=17777
       - -debug
       - -debug.port=18777
     depends_on:
       seaweedfs-filer:
         condition: service_healthy
     volumes:
       - seaweedfs-mq-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "17777"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 20s
     networks:
       - kafka-test-net
   # SeaweedFS MQ Agent
   seaweedfs-mq-agent:
     <<: *seaweedfs-build
     container_name: seaweedfs-mq-agent
     ports:
       - "16777:16777"  # MQ Agent port
     command:
       - mq.agent
       - -broker=seaweedfs-mq-broker:17777
       - -ip=0.0.0.0
       - -port=16777
     depends_on:
       seaweedfs-mq-broker:
         condition: service_healthy
     volumes:
       - seaweedfs-mq-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "16777"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 25s
     networks:
       - kafka-test-net
   # Kafka Gateway (SeaweedFS with Kafka protocol)
   kafka-gateway:
     build:
       context: ../..  # Build from project root
       dockerfile: test/kafka/Dockerfile.kafka-gateway
     container_name: kafka-gateway
     ports:
       - "9093:9093"  # Kafka protocol port
       - "10093:10093"  # pprof profiling port
     depends_on:
       seaweedfs-mq-agent:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
     environment:
       - SEAWEEDFS_MASTERS=seaweedfs-master:9333
       - SEAWEEDFS_FILER_GROUP=
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
       - KAFKA_PORT=9093
       - PPROF_PORT=10093
     volumes:
       - kafka-gateway-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "9093"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 30s
     networks:
       - kafka-test-net
   # Test Data Setup Service
   test-setup:
     build:
       context: ../..
       dockerfile: test/kafka/Dockerfile.test-setup
     container_name: kafka-test-setup
     depends_on:
       kafka:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
       kafka-gateway:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
       - KAFKA_GATEWAY_URL=kafka-gateway:9093
     networks:
       - kafka-test-net
     restart: "no"  # Run once to set up test data
     profiles:
       - setup  # Only start when explicitly requested
   # Kafka Producer for Testing
   kafka-producer:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-producer
     depends_on:
       kafka:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
     networks:
       - kafka-test-net
     profiles:
       - producer  # Only start when explicitly requested
     command: >
       sh -c "
         echo 'Creating test topics...';
         kafka-topics --create --topic test-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
         kafka-topics --create --topic avro-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
         kafka-topics --create --topic schema-test --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --if-not-exists;
         echo 'Topics created successfully';
         kafka-topics --list --bootstrap-server kafka:29092;
       "
   # Kafka Consumer for Testing
   kafka-consumer:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-consumer
     depends_on:
       kafka:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
     networks:
       - kafka-test-net
     profiles:
       - consumer  # Only start when explicitly requested
     command: >
       kafka-console-consumer
       --bootstrap-server kafka:29092
       --topic test-topic
       --from-beginning
       --max-messages 10
 volumes:
   seaweedfs-master-data:
   seaweedfs-volume-data:
   seaweedfs-filer-data:
   seaweedfs-mq-data:
   kafka-gateway-data:
 networks:
   kafka-test-net:
     driver: bridge
     name: kafka-integration-test

327 lines 8.7 KiB YAML Raw Permalink Blame History

327 lines

8.7 KiB

YAML

Raw Permalink Blame History