mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zeekay
8f5d0b9333
Full internal rename — build-verified, every module green (root + s3-rdma-sidecar
+ telemetry/server + all test modules):
- Dirs: weed/→s3/ (main tree); cmd/weed-{db,sql}→cmd/s3-{db,sql};
seaweed-volume→s3-volume; seaweedfs-rdma-sidecar→s3-rdma-sidecar;
s3/mount/weedfs_*.go→s3fs_*.go (~113 path renames).
- Imports: github.com/hanzoai/s3/weed/ → /s3/ across 1700+ files.
- gRPC proto services Seaweed/SeaweedFiler/SeaweedMessaging/... → Hanzo*
(8 services) via full protoc regeneration (correct descriptors, no sed-corruption).
- Brand SeaweedFS/Seaweed/seaweed → Hanzo/hanzo; internal X-Seaweedfs-* headers
→ X-Hanzo-*; .seaweedfs config dir → .s3.
- CLI is `s3`: go install github.com/hanzoai/s3/s3@latest | make install | Docker.
install.sh rewritten as a Hanzo S3 installer.
- External forks github.com/seaweedfs/{raft,go-fuse,goexif,cockroachdb-parser}
kept verbatim — immutable third-party module paths; NOTICE retained.
Residual (cosmetic, non-build): kafka loadtest test-proto package com.seaweedfs.loadtest.
207 lines
7.8 KiB
Makefile
207 lines
7.8 KiB
Makefile
# Kafka Integration Testing Makefile - Refactored
|
|
# This replaces the existing Makefile with better organization
|
|
|
|
# Configuration
|
|
ifndef DOCKER_COMPOSE
|
|
DOCKER_COMPOSE := $(if $(shell command -v docker-compose 2>/dev/null),docker-compose,docker compose)
|
|
endif
|
|
TEST_TIMEOUT ?= 10m
|
|
KAFKA_BOOTSTRAP_SERVERS ?= localhost:9092
|
|
KAFKA_GATEWAY_URL ?= localhost:9093
|
|
SCHEMA_REGISTRY_URL ?= http://localhost:8081
|
|
|
|
# Colors for output
|
|
BLUE := \033[36m
|
|
GREEN := \033[32m
|
|
YELLOW := \033[33m
|
|
RED := \033[31m
|
|
NC := \033[0m # No Color
|
|
|
|
.PHONY: help setup test clean logs status
|
|
|
|
help: ## Show this help message
|
|
@echo "$(BLUE)Hanzo Kafka Integration Testing - Refactored$(NC)"
|
|
@echo ""
|
|
@echo "Available targets:"
|
|
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " $(GREEN)%-20s$(NC) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
|
|
|
# Environment Setup
|
|
setup: ## Set up test environment (Kafka + Schema Registry + Hanzo)
|
|
@echo "$(YELLOW)Setting up Kafka integration test environment...$(NC)"
|
|
@$(DOCKER_COMPOSE) up -d
|
|
@echo "$(BLUE)Waiting for all services to be ready...$(NC)"
|
|
@./scripts/wait-for-services.sh
|
|
@echo "$(GREEN)Test environment ready!$(NC)"
|
|
|
|
setup-schemas: setup ## Set up test environment and register schemas
|
|
@echo "$(YELLOW)Registering test schemas...$(NC)"
|
|
@$(DOCKER_COMPOSE) --profile setup run --rm test-setup
|
|
@echo "$(GREEN)Schemas registered!$(NC)"
|
|
|
|
# Test Categories
|
|
test: test-unit test-integration test-e2e ## Run all tests
|
|
|
|
test-unit: ## Run unit tests
|
|
@echo "$(YELLOW)Running unit tests...$(NC)"
|
|
@go test -v -timeout=$(TEST_TIMEOUT) ./unit/...
|
|
|
|
test-integration: ## Run integration tests
|
|
@echo "$(YELLOW)Running integration tests...$(NC)"
|
|
@go test -v -timeout=$(TEST_TIMEOUT) ./integration/...
|
|
|
|
test-e2e: setup-schemas ## Run end-to-end tests
|
|
@echo "$(YELLOW)Running end-to-end tests...$(NC)"
|
|
@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
|
|
KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
|
|
SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) ./e2e/...
|
|
|
|
test-docker: setup-schemas ## Run Docker integration tests
|
|
@echo "$(YELLOW)Running Docker integration tests...$(NC)"
|
|
@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
|
|
KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
|
|
SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Docker
|
|
|
|
# Schema-specific tests
|
|
test-schema: setup-schemas ## Run schema registry integration tests
|
|
@echo "$(YELLOW)Running schema registry integration tests...$(NC)"
|
|
@SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Schema
|
|
|
|
# Client-specific tests
|
|
test-sarama: setup-schemas ## Run Sarama client tests
|
|
@echo "$(YELLOW)Running Sarama client tests...$(NC)"
|
|
@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
|
|
KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run Sarama
|
|
|
|
test-kafka-go: setup-schemas ## Run kafka-go client tests
|
|
@echo "$(YELLOW)Running kafka-go client tests...$(NC)"
|
|
@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
|
|
KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) ./integration/ -run KafkaGo
|
|
|
|
# Performance tests
|
|
test-performance: setup-schemas ## Run performance benchmarks
|
|
@echo "$(YELLOW)Running Kafka performance benchmarks...$(NC)"
|
|
@KAFKA_BOOTSTRAP_SERVERS=$(KAFKA_BOOTSTRAP_SERVERS) \
|
|
KAFKA_GATEWAY_URL=$(KAFKA_GATEWAY_URL) \
|
|
SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) \
|
|
go test -v -timeout=$(TEST_TIMEOUT) -bench=. ./...
|
|
|
|
# Development targets
|
|
dev-kafka: ## Start only Kafka ecosystem for development
|
|
@$(DOCKER_COMPOSE) up -d zookeeper kafka schema-registry
|
|
@sleep 20
|
|
@$(DOCKER_COMPOSE) --profile setup run --rm test-setup
|
|
|
|
dev-hanzo: ## Start only Hanzo for development
|
|
@$(DOCKER_COMPOSE) up -d hanzo-master hanzo-volume hanzo-filer hanzo-mq-broker hanzo-mq-agent
|
|
|
|
dev-gateway: dev-hanzo ## Start Kafka Gateway for development
|
|
@$(DOCKER_COMPOSE) up -d kafka-gateway
|
|
|
|
dev-test: dev-kafka ## Quick test with just Kafka ecosystem
|
|
@SCHEMA_REGISTRY_URL=$(SCHEMA_REGISTRY_URL) go test -v -timeout=30s ./unit/...
|
|
|
|
# Cleanup
|
|
clean: ## Clean up test environment
|
|
@echo "$(YELLOW)Cleaning up test environment...$(NC)"
|
|
@$(DOCKER_COMPOSE) down -v --remove-orphans
|
|
@docker system prune -f
|
|
@echo "$(GREEN)Environment cleaned up!$(NC)"
|
|
|
|
# Monitoring and debugging
|
|
logs: ## Show logs from all services
|
|
@$(DOCKER_COMPOSE) logs --tail=50 -f
|
|
|
|
logs-kafka: ## Show Kafka logs
|
|
@$(DOCKER_COMPOSE) logs --tail=100 -f kafka
|
|
|
|
logs-schema-registry: ## Show Schema Registry logs
|
|
@$(DOCKER_COMPOSE) logs --tail=100 -f schema-registry
|
|
|
|
logs-hanzo: ## Show Hanzo logs
|
|
@$(DOCKER_COMPOSE) logs --tail=100 -f hanzo-master hanzo-volume hanzo-filer hanzo-mq-broker hanzo-mq-agent
|
|
|
|
logs-gateway: ## Show Kafka Gateway logs
|
|
@$(DOCKER_COMPOSE) logs --tail=100 -f kafka-gateway
|
|
|
|
status: ## Show status of all services
|
|
@echo "$(BLUE)Service Status:$(NC)"
|
|
@$(DOCKER_COMPOSE) ps
|
|
@echo ""
|
|
@echo "$(BLUE)Kafka Status:$(NC)"
|
|
@curl -s http://localhost:9092 > /dev/null && echo "Kafka accessible" || echo "Kafka not accessible"
|
|
@echo ""
|
|
@echo "$(BLUE)Schema Registry Status:$(NC)"
|
|
@curl -s $(SCHEMA_REGISTRY_URL)/subjects > /dev/null && echo "Schema Registry accessible" || echo "Schema Registry not accessible"
|
|
@echo ""
|
|
@echo "$(BLUE)Kafka Gateway Status:$(NC)"
|
|
@nc -z localhost 9093 && echo "Kafka Gateway accessible" || echo "Kafka Gateway not accessible"
|
|
|
|
debug: ## Debug test environment
|
|
@echo "$(BLUE)Debug Information:$(NC)"
|
|
@echo "Kafka Bootstrap Servers: $(KAFKA_BOOTSTRAP_SERVERS)"
|
|
@echo "Schema Registry URL: $(SCHEMA_REGISTRY_URL)"
|
|
@echo "Kafka Gateway URL: $(KAFKA_GATEWAY_URL)"
|
|
@echo ""
|
|
@echo "Docker Compose Status:"
|
|
@$(DOCKER_COMPOSE) ps
|
|
@echo ""
|
|
@echo "Network connectivity:"
|
|
@docker network ls | grep kafka-integration-test || echo "No Kafka test network found"
|
|
@echo ""
|
|
@echo "Schema Registry subjects:"
|
|
@curl -s $(SCHEMA_REGISTRY_URL)/subjects 2>/dev/null || echo "Schema Registry not accessible"
|
|
|
|
# Utility targets
|
|
install-deps: ## Install required dependencies
|
|
@echo "$(YELLOW)Installing test dependencies...$(NC)"
|
|
@which docker > /dev/null || (echo "$(RED)Docker not found$(NC)" && exit 1)
|
|
@which docker-compose > /dev/null || (echo "$(RED)Docker Compose not found$(NC)" && exit 1)
|
|
@which curl > /dev/null || (echo "$(RED)curl not found$(NC)" && exit 1)
|
|
@which nc > /dev/null || (echo "$(RED)netcat not found$(NC)" && exit 1)
|
|
@echo "$(GREEN)All dependencies available$(NC)"
|
|
|
|
check-env: ## Check test environment setup
|
|
@echo "$(BLUE)Environment Check:$(NC)"
|
|
@echo "KAFKA_BOOTSTRAP_SERVERS: $(KAFKA_BOOTSTRAP_SERVERS)"
|
|
@echo "SCHEMA_REGISTRY_URL: $(SCHEMA_REGISTRY_URL)"
|
|
@echo "KAFKA_GATEWAY_URL: $(KAFKA_GATEWAY_URL)"
|
|
@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
|
|
@make install-deps
|
|
|
|
# CI targets
|
|
ci-test: ## Run tests in CI environment
|
|
@echo "$(YELLOW)Running CI tests...$(NC)"
|
|
@make setup-schemas
|
|
@make test-unit
|
|
@make test-integration
|
|
@make clean
|
|
|
|
ci-e2e: ## Run end-to-end tests in CI
|
|
@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
|
|
@make test-e2e
|
|
@make clean
|
|
|
|
# Interactive targets
|
|
shell-kafka: ## Open shell in Kafka container
|
|
@$(DOCKER_COMPOSE) exec kafka bash
|
|
|
|
shell-gateway: ## Open shell in Kafka Gateway container
|
|
@$(DOCKER_COMPOSE) exec kafka-gateway sh
|
|
|
|
topics: ## List Kafka topics
|
|
@$(DOCKER_COMPOSE) exec kafka kafka-topics --list --bootstrap-server localhost:29092
|
|
|
|
create-topic: ## Create a test topic (usage: make create-topic TOPIC=my-topic)
|
|
@$(DOCKER_COMPOSE) exec kafka kafka-topics --create --topic $(TOPIC) --bootstrap-server localhost:29092 --partitions 3 --replication-factor 1
|
|
|
|
produce: ## Produce test messages (usage: make produce TOPIC=my-topic)
|
|
@$(DOCKER_COMPOSE) exec kafka kafka-console-producer --bootstrap-server localhost:29092 --topic $(TOPIC)
|
|
|
|
consume: ## Consume messages (usage: make consume TOPIC=my-topic)
|
|
@$(DOCKER_COMPOSE) exec kafka kafka-console-consumer --bootstrap-server localhost:29092 --topic $(TOPIC) --from-beginning
|