s3/test/kafka/kafka-client-loadtest/scripts/setup-monitoring.sh

#!/bin/bash

# Setup monitoring for Kafka Client Load Test
# This script sets up Prometheus and Grafana configurations

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
MONITORING_DIR="$PROJECT_DIR/monitoring"

# Colors
GREEN='\033[0;32m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

# Create monitoring directory structure
setup_directories() {
    log_info "Setting up monitoring directories..."
    
    mkdir -p "$MONITORING_DIR/prometheus"
    mkdir -p "$MONITORING_DIR/grafana/dashboards"
    mkdir -p "$MONITORING_DIR/grafana/provisioning/dashboards"
    mkdir -p "$MONITORING_DIR/grafana/provisioning/datasources"
    
    log_success "Directories created"
}

# Create Prometheus configuration
create_prometheus_config() {
    log_info "Creating Prometheus configuration..."
    
    cat > "$MONITORING_DIR/prometheus/prometheus.yml" << 'EOF'
# Prometheus configuration for Kafka Load Test monitoring

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

scrape_configs:
  # Scrape Prometheus itself
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Scrape load test metrics
  - job_name: 'kafka-loadtest'
    static_configs:
      - targets: ['kafka-client-loadtest-runner:8080']
    scrape_interval: 5s
    metrics_path: '/metrics'

  # Scrape Hanzo Master metrics
  - job_name: 'hanzo-master'
    static_configs:
      - targets: ['hanzo-master:9333']
    metrics_path: '/metrics'

  # Scrape Hanzo Volume metrics  
  - job_name: 'hanzo-volume'
    static_configs:
      - targets: ['hanzo-volume:8080']
    metrics_path: '/metrics'

  # Scrape Hanzo Filer metrics
  - job_name: 'hanzo-filer'
    static_configs:
      - targets: ['hanzo-filer:8888']
    metrics_path: '/metrics'

  # Scrape Hanzo MQ Broker metrics (if available)
  - job_name: 'hanzo-mq-broker'
    static_configs:
      - targets: ['hanzo-mq-broker:17777']
    metrics_path: '/metrics'
    scrape_interval: 10s

  # Scrape Kafka Gateway metrics (if available)
  - job_name: 'kafka-gateway'
    static_configs:
      - targets: ['kafka-gateway:9093']
    metrics_path: '/metrics'
    scrape_interval: 10s
EOF

    log_success "Prometheus configuration created"
}

# Create Grafana datasource configuration
create_grafana_datasource() {
    log_info "Creating Grafana datasource configuration..."
    
    cat > "$MONITORING_DIR/grafana/provisioning/datasources/datasource.yml" << 'EOF'
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    orgId: 1
    url: http://prometheus:9090
    basicAuth: false
    isDefault: true
    editable: true
    version: 1
EOF

    log_success "Grafana datasource configuration created"
}

# Create Grafana dashboard provisioning
create_grafana_dashboard_provisioning() {
    log_info "Creating Grafana dashboard provisioning..."
    
    cat > "$MONITORING_DIR/grafana/provisioning/dashboards/dashboard.yml" << 'EOF'
apiVersion: 1

providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    editable: true
    options:
      path: /var/lib/grafana/dashboards
EOF

    log_success "Grafana dashboard provisioning created"
}

# Create Kafka Load Test dashboard
create_loadtest_dashboard() {
    log_info "Creating Kafka Load Test Grafana dashboard..."
    
    cat > "$MONITORING_DIR/grafana/dashboards/kafka-loadtest.json" << 'EOF'
{
  "dashboard": {
    "id": null,
    "title": "Kafka Client Load Test Dashboard",
    "tags": ["kafka", "loadtest", "hanzo"],
    "timezone": "browser",
    "panels": [
      {
        "id": 1,
        "title": "Messages Produced/Consumed",
        "type": "stat",
        "targets": [
          {
            "expr": "rate(kafka_loadtest_messages_produced_total[5m])",
            "legendFormat": "Produced/sec"
          },
          {
            "expr": "rate(kafka_loadtest_messages_consumed_total[5m])",
            "legendFormat": "Consumed/sec"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
      },
      {
        "id": 2,
        "title": "Message Latency",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, kafka_loadtest_message_latency_seconds)",
            "legendFormat": "95th percentile"
          },
          {
            "expr": "histogram_quantile(0.99, kafka_loadtest_message_latency_seconds)",
            "legendFormat": "99th percentile"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
      },
      {
        "id": 3,
        "title": "Error Rates",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(kafka_loadtest_producer_errors_total[5m])",
            "legendFormat": "Producer Errors/sec"
          },
          {
            "expr": "rate(kafka_loadtest_consumer_errors_total[5m])",
            "legendFormat": "Consumer Errors/sec"
          }
        ],
        "gridPos": {"h": 8, "w": 24, "x": 0, "y": 8}
      },
      {
        "id": 4,
        "title": "Throughput (MB/s)",
        "type": "graph", 
        "targets": [
          {
            "expr": "rate(kafka_loadtest_bytes_produced_total[5m]) / 1024 / 1024",
            "legendFormat": "Produced MB/s"
          },
          {
            "expr": "rate(kafka_loadtest_bytes_consumed_total[5m]) / 1024 / 1024", 
            "legendFormat": "Consumed MB/s"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}
      },
      {
        "id": 5,
        "title": "Active Clients",
        "type": "stat",
        "targets": [
          {
            "expr": "kafka_loadtest_active_producers",
            "legendFormat": "Producers"
          },
          {
            "expr": "kafka_loadtest_active_consumers", 
            "legendFormat": "Consumers"
          }
        ],
        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}
      },
      {
        "id": 6,
        "title": "Consumer Lag",
        "type": "graph",
        "targets": [
          {
            "expr": "kafka_loadtest_consumer_lag_messages",
            "legendFormat": "{{consumer_group}}-{{topic}}-{{partition}}"
          }
        ],
        "gridPos": {"h": 8, "w": 24, "x": 0, "y": 24}
      }
    ],
    "time": {"from": "now-30m", "to": "now"},
    "refresh": "5s",
    "schemaVersion": 16,
    "version": 0
  }
}
EOF

    log_success "Kafka Load Test dashboard created"
}

# Create Hanzo dashboard
create_hanzo_dashboard() {
    log_info "Creating Hanzo Grafana dashboard..."
    
    cat > "$MONITORING_DIR/grafana/dashboards/hanzo.json" << 'EOF'
{
  "dashboard": {
    "id": null,
    "title": "Hanzo Cluster Dashboard",
    "tags": ["hanzo", "storage"],
    "timezone": "browser", 
    "panels": [
      {
        "id": 1,
        "title": "Master Status",
        "type": "stat",
        "targets": [
          {
            "expr": "up{job=\"hanzo-master\"}",
            "legendFormat": "Master Up"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0}
      },
      {
        "id": 2, 
        "title": "Volume Status",
        "type": "stat",
        "targets": [
          {
            "expr": "up{job=\"hanzo-volume\"}",
            "legendFormat": "Volume Up"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0}
      },
      {
        "id": 3,
        "title": "Filer Status", 
        "type": "stat",
        "targets": [
          {
            "expr": "up{job=\"hanzo-filer\"}",
            "legendFormat": "Filer Up"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0}
      },
      {
        "id": 4,
        "title": "MQ Broker Status",
        "type": "stat", 
        "targets": [
          {
            "expr": "up{job=\"hanzo-mq-broker\"}",
            "legendFormat": "MQ Broker Up"
          }
        ],
        "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0}
      }
    ],
    "time": {"from": "now-30m", "to": "now"},
    "refresh": "10s",
    "schemaVersion": 16,
    "version": 0
  }
}
EOF

    log_success "Hanzo dashboard created"
}

# Main setup function
main() {
    log_info "Setting up monitoring for Kafka Client Load Test..."
    
    setup_directories
    create_prometheus_config
    create_grafana_datasource 
    create_grafana_dashboard_provisioning
    create_loadtest_dashboard
    create_hanzo_dashboard
    
    log_success "Monitoring setup completed!"
    log_info "You can now start the monitoring stack with:"
    log_info "  ./scripts/run-loadtest.sh monitor"
    log_info ""
    log_info "After starting, access:"
    log_info "  Prometheus: http://localhost:9090"
    log_info "  Grafana:    http://localhost:3000 (admin/admin)"
}

main "$@"