s3/test/kafka/docker-compose.yml at main

mirror of https://github.com/hanzoai/s3.git synced 2026-10-30 14:59:09 +00:00

Files

T

zeekayandHanzo Dev af0f387bbe Remove weed and SeaweedFS naming from the tree

The binary is s3 and the product is Hanzo S3; the rest of the tree now
says so too.

- Environment: WEED_* and SEAWEEDFS_* are S3_*, no aliases. fla9's flag
  prefix is S3, matching viper's existing "s3" prefix, so the S3_S3_SSE_*
  names mini.go exports are the ones viper reads (WEED_S3_SSE_KEK was never
  seen once viper moved to "s3").
- Java and Hadoop: packages, fs.hanzofs.* keys and the hanzofs:// scheme
  under the hanzofs namespace the protos already declare; classes take a
  neutral Filer prefix and file names match them again (the earlier pass left
  Seas3*.java holding Seaweed* classes). Client, hdfs3, examples, s3copier,
  random_access and the Spark tests compile.
- Helm chart: helpers are hanzo.*, values live under global.s3; the chart
  renders and lints, which it did not before.
- Terraform: the s3.service.tftpl reference matches the file; 12/12 render
  tests pass. Config paths point at /etc/hanzo, which the binary reads.
- loadtest.pb.go regenerated from loadtest.proto.
- Upstream logo, sponsor images and unreferenced diagrams removed from note/.

Kept: NOTICE attribution, LICENSE, the erasure-coding fixture
s3/storage/erasure_coding/1.dat (test bytes), and unmaintained/ and
third_party/ (vendored). No on-disk format, wire protocol or metadata key
changed.

Co-authored-by: Hanzo Dev <[email protected]>

2026-10-02 08:06:48 -07:00

327 lines

8.5 KiB

YAML

Raw Permalink Blame History

 x-hanzo-build: &hanzo-build
   build:
     context: ../..
     dockerfile: test/kafka/Dockerfile.s3
   image: kafka-hanzo-dev
 services:
   # Zookeeper for Kafka
   zookeeper:
     image: confluentinc/cp-zookeeper:7.4.0
     container_name: kafka-zookeeper
     ports:
       - "2181:2181"
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "2181"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 10s
     networks:
       - kafka-test-net
   # Kafka Broker
   kafka:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-broker
     ports:
       - "9092:9092"
       - "29092:29092"
     depends_on:
       zookeeper:
         condition: service_healthy
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
       KAFKA_NUM_PARTITIONS: 3
       KAFKA_DEFAULT_REPLICATION_FACTOR: 1
     healthcheck:
       test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 30s
     networks:
       - kafka-test-net
   # Schema Registry
   schema-registry:
     image: confluentinc/cp-schema-registry:7.4.0
     container_name: kafka-schema-registry
     ports:
       - "8081:8081"
     depends_on:
       kafka:
         condition: service_healthy
     environment:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
       SCHEMA_REGISTRY_KAFKASTORE_TOPIC: _schemas
       SCHEMA_REGISTRY_DEBUG: "true"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 20s
     networks:
       - kafka-test-net
   # Hanzo Master
   hanzo-master:
     <<: *hanzo-build
     container_name: hanzo-master
     ports:
       - "9333:9333"
       - "19333:19333"  # gRPC port
     command:
       - master
       - -ip=hanzo-master
       - -port=9333
       - -port.grpc=19333
       - -volumeSizeLimitMB=1024
       - -defaultReplication=000
     volumes:
       - hanzo-master-data:/data
     healthcheck:
       test: ["CMD-SHELL", "wget --quiet --tries=1 --spider http://hanzo-master:9333/cluster/status || curl -sf http://hanzo-master:9333/cluster/status"]
       interval: 10s
       timeout: 5s
       retries: 10
       start_period: 20s
     networks:
       - kafka-test-net
   # Hanzo Volume Server
   hanzo-volume:
     <<: *hanzo-build
     container_name: hanzo-volume
     ports:
       - "8080:8080"
       - "18080:18080"  # gRPC port
     command:
       - volume
       - -master=hanzo-master:9333
       - -ip=hanzo-volume
       - -port=8080
       - -port.grpc=18080
       - -publicUrl=hanzo-volume:8080
       - -preStopSeconds=1
     depends_on:
       hanzo-master:
         condition: service_healthy
     volumes:
       - hanzo-volume-data:/data
     healthcheck:
       test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://hanzo-volume:8080/status"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 10s
     networks:
       - kafka-test-net
   # Hanzo Filer
   hanzo-filer:
     <<: *hanzo-build
     container_name: hanzo-filer
     ports:
       - "8888:8888"
       - "18888:18888"  # gRPC port
     command:
       - filer
       - -master=hanzo-master:9333
       - -ip=hanzo-filer
       - -port=8888
       - -port.grpc=18888
     depends_on:
       hanzo-master:
         condition: service_healthy
       hanzo-volume:
         condition: service_healthy
     volumes:
       - hanzo-filer-data:/data
     healthcheck:
       test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://hanzo-filer:8888/"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 15s
     networks:
       - kafka-test-net
   # Hanzo MQ Broker
   hanzo-mq-broker:
     <<: *hanzo-build
     container_name: hanzo-mq-broker
     ports:
       - "17777:17777"  # MQ Broker port
       - "18777:18777"  # pprof profiling port
     command:
       - mq.broker
       - -master=hanzo-master:9333
       - -ip=hanzo-mq-broker
       - -port=17777
       - -debug
       - -debug.port=18777
     depends_on:
       hanzo-filer:
         condition: service_healthy
     volumes:
       - hanzo-mq-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "17777"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 20s
     networks:
       - kafka-test-net
   # Hanzo MQ Agent
   hanzo-mq-agent:
     <<: *hanzo-build
     container_name: hanzo-mq-agent
     ports:
       - "16777:16777"  # MQ Agent port
     command:
       - mq.agent
       - -broker=hanzo-mq-broker:17777
       - -ip=0.0.0.0
       - -port=16777
     depends_on:
       hanzo-mq-broker:
         condition: service_healthy
     volumes:
       - hanzo-mq-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "16777"]
       interval: 10s
       timeout: 5s
       retries: 3
       start_period: 25s
     networks:
       - kafka-test-net
   # Kafka Gateway (Hanzo with Kafka protocol)
   kafka-gateway:
     build:
       context: ../..  # Build from project root
       dockerfile: test/kafka/Dockerfile.kafka-gateway
     container_name: kafka-gateway
     ports:
       - "9093:9093"  # Kafka protocol port
       - "10093:10093"  # pprof profiling port
     depends_on:
       hanzo-mq-agent:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
     environment:
       - S3_MASTERS=hanzo-master:9333
       - S3_FILER_GROUP=
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
       - KAFKA_PORT=9093
       - PPROF_PORT=10093
     volumes:
       - kafka-gateway-data:/data
     healthcheck:
       test: ["CMD", "nc", "-z", "localhost", "9093"]
       interval: 10s
       timeout: 5s
       retries: 5
       start_period: 30s
     networks:
       - kafka-test-net
   # Test Data Setup Service
   test-setup:
     build:
       context: ../..
       dockerfile: test/kafka/Dockerfile.test-setup
     container_name: kafka-test-setup
     depends_on:
       kafka:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
       kafka-gateway:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
       - KAFKA_GATEWAY_URL=kafka-gateway:9093
     networks:
       - kafka-test-net
     restart: "no"  # Run once to set up test data
     profiles:
       - setup  # Only start when explicitly requested
   # Kafka Producer for Testing
   kafka-producer:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-producer
     depends_on:
       kafka:
         condition: service_healthy
       schema-registry:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
       - SCHEMA_REGISTRY_URL=http://schema-registry:8081
     networks:
       - kafka-test-net
     profiles:
       - producer  # Only start when explicitly requested
     command: >
       sh -c "
         echo 'Creating test topics...';
         kafka-topics --create --topic test-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
         kafka-topics --create --topic avro-topic --bootstrap-server kafka:29092 --partitions 3 --replication-factor 1 --if-not-exists;
         kafka-topics --create --topic schema-test --bootstrap-server kafka:29092 --partitions 1 --replication-factor 1 --if-not-exists;
         echo 'Topics created successfully';
         kafka-topics --list --bootstrap-server kafka:29092;
       "
   # Kafka Consumer for Testing
   kafka-consumer:
     image: confluentinc/cp-kafka:7.4.0
     container_name: kafka-consumer
     depends_on:
       kafka:
         condition: service_healthy
     environment:
       - KAFKA_BOOTSTRAP_SERVERS=kafka:29092
     networks:
       - kafka-test-net
     profiles:
       - consumer  # Only start when explicitly requested
     command: >
       kafka-console-consumer
       --bootstrap-server kafka:29092
       --topic test-topic
       --from-beginning
       --max-messages 10
 volumes:
   hanzo-master-data:
   hanzo-volume-data:
   hanzo-filer-data:
   hanzo-mq-data:
   kafka-gateway-data:
 networks:
   kafka-test-net:
     driver: bridge
     name: kafka-integration-test

327 lines 8.5 KiB YAML Raw Permalink Blame History

327 lines

8.5 KiB

YAML

Raw Permalink Blame History