mirror of
https://github.com/hanzoai/insights.git
synced 2026-10-30 16:15:31 +00:00
zeekay
168c1136b4
The funnel queries call the unversioned UDFs (aggregate_funnel_array and kin), declared only in docker/datastore/user_defined_function.xml, which no image carries. The binaries already ship in insights/user_scripts, so the config moves beside them: a datastore takes both from one image.
423 lines
14 KiB
YAML
423 lines
14 KiB
YAML
#
|
|
# `docker-compose` file used ONLY for hobby deployments.
|
|
#
|
|
# Please take a look at https://insights.hanzo.ai/docs/self-host/deploy/hobby
|
|
# for more info.
|
|
#
|
|
# Insights has sunset support for self-hosted K8s deployments.
|
|
# See: https://insights.hanzo.ai/blog/sunsetting-helm-support-insights
|
|
#
|
|
|
|
services:
|
|
db:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: db
|
|
image: ${DOCKER_REGISTRY_PREFIX:-}ghcr.io/hanzoai/sql:latest
|
|
volumes:
|
|
- postgres-data:/var/lib/postgresql/data
|
|
|
|
kv:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: kv
|
|
volumes:
|
|
- kv-data:/data
|
|
|
|
datastore:
|
|
#
|
|
# Note: please keep the default version in sync across
|
|
# `insights` and the `charts-datastore` repos
|
|
#
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: datastore
|
|
restart: on-failure
|
|
depends_on:
|
|
- kafka
|
|
volumes:
|
|
- ./insights/insights/idl:/idl
|
|
- ./insights/docker/datastore/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
|
|
- ./insights/docker/datastore/config.xml:/etc/datastore-server/config.xml
|
|
- ./insights/docker/datastore/config.d/default.xml:/etc/datastore-server/config.d/default.xml
|
|
- ./insights/docker/datastore/users.xml:/etc/datastore-server/users.xml
|
|
- ./insights/insights/user_scripts/user_defined_function.xml:/etc/datastore-server/user_defined_function.xml
|
|
- ./insights/insights/user_scripts:/var/lib/datastore/user_scripts
|
|
- datastore-data:/var/lib/datastore
|
|
|
|
kafka:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: kafka
|
|
environment:
|
|
KAFKA_LOG_RETENTION_MS: 3600000
|
|
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
|
|
KAFKA_LOG_RETENTION_HOURS: 1
|
|
volumes:
|
|
- kafka-data:/bitnami/kafka
|
|
|
|
worker:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: worker
|
|
environment:
|
|
SITE_URL: https://$DOMAIN
|
|
SECRET_KEY: $INSIGHTS_SECRET
|
|
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
|
|
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
|
|
OBJECT_STORAGE_ENDPOINT: http://objectstorage:19000
|
|
SESSION_RECORDING_V2_S3_ENDPOINT: http://seaweedfs:8333
|
|
SESSION_RECORDING_V2_S3_ACCESS_KEY_ID: 'any'
|
|
SESSION_RECORDING_V2_S3_SECRET_ACCESS_KEY: 'any'
|
|
OBJECT_STORAGE_ENABLED: true
|
|
RECORDING_API_URL: http://plugins:6738
|
|
ENCRYPTION_SALT_KEYS: $ENCRYPTION_SALT_KEYS
|
|
INSIGHTS_SKIP_MIGRATION_CHECKS: '1'
|
|
image: $REGISTRY_URL:$INSIGHTS_APP_TAG
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
kv:
|
|
condition: service_healthy
|
|
datastore:
|
|
condition: service_started
|
|
kafka:
|
|
condition: service_healthy
|
|
web:
|
|
condition: service_started
|
|
web:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: web
|
|
command: /compose/start
|
|
volumes:
|
|
- ./compose:/compose
|
|
image: $REGISTRY_URL:$INSIGHTS_APP_TAG
|
|
environment:
|
|
SITE_URL: https://$DOMAIN
|
|
LIVESTREAM_HOST: 'https://${DOMAIN}/livestream'
|
|
SECRET_KEY: $INSIGHTS_SECRET
|
|
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
|
|
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
|
|
SESSION_RECORDING_V2_S3_ACCESS_KEY_ID: 'any'
|
|
SESSION_RECORDING_V2_S3_SECRET_ACCESS_KEY: 'any'
|
|
OBJECT_STORAGE_ENDPOINT: http://objectstorage:19000
|
|
SESSION_RECORDING_V2_S3_ENDPOINT: http://seaweedfs:8333
|
|
OBJECT_STORAGE_ENABLED: true
|
|
RECORDING_API_URL: http://plugins:6738
|
|
ENCRYPTION_SALT_KEYS: $ENCRYPTION_SALT_KEYS
|
|
OTEL_SERVICE_NAME: 'insights'
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: ''
|
|
OTEL_SDK_DISABLED: 'true'
|
|
USE_GRANIAN: 'true'
|
|
GRANIAN_WORKERS: '2'
|
|
OPT_OUT_CAPTURE: ${OPT_OUT_CAPTURE:-false}
|
|
DEPLOYMENT: 'hobby'
|
|
depends_on:
|
|
- db
|
|
- kv
|
|
- datastore
|
|
- kafka
|
|
- objectstorage
|
|
- seaweedfs
|
|
|
|
plugins:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: plugins
|
|
image: ${REGISTRY_URL}-node:${INSIGHTS_NODE_TAG:-latest}
|
|
environment:
|
|
SITE_URL: https://$DOMAIN
|
|
SECRET_KEY: $INSIGHTS_SECRET
|
|
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
|
|
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
|
|
SESSION_RECORDING_V2_S3_ACCESS_KEY_ID: 'any'
|
|
SESSION_RECORDING_V2_S3_SECRET_ACCESS_KEY: 'any'
|
|
SESSION_RECORDING_V2_S3_TIMEOUT_MS: 120000
|
|
SESSION_RECORDING_API_REDIS_HOST: kv
|
|
SESSION_RECORDING_API_REDIS_PORT: 6379
|
|
OBJECT_STORAGE_ENDPOINT: http://objectstorage:19000
|
|
SESSION_RECORDING_V2_S3_ENDPOINT: http://seaweedfs:8333
|
|
OBJECT_STORAGE_ENABLED: true
|
|
CDP_REDIS_HOST: kv
|
|
CDP_REDIS_PORT: 6379
|
|
LOGS_REDIS_HOST: kv
|
|
LOGS_REDIS_PORT: 6379
|
|
LOGS_REDIS_TLS: 'false'
|
|
ENCRYPTION_SALT_KEYS: $ENCRYPTION_SALT_KEYS
|
|
CYCLOTRON_DATABASE_URL: 'postgres://insights:insights@db:5432/insights'
|
|
PERSONS_DATABASE_URL: 'postgres://insights:insights@db:5432/insights'
|
|
BEHAVIORAL_COHORTS_DATABASE_URL: 'postgres://insights:insights@db:5432/insights'
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: ''
|
|
volumes:
|
|
- ./share:/share
|
|
depends_on:
|
|
- db
|
|
- kv
|
|
- datastore
|
|
- kafka
|
|
- objectstorage
|
|
- seaweedfs
|
|
|
|
proxy:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: proxy
|
|
ports:
|
|
- '80:80'
|
|
- '443:443'
|
|
volumes:
|
|
- caddy-data:/data
|
|
- caddy-config:/config
|
|
environment:
|
|
CADDY_TLS_BLOCK: $TLS_BLOCK
|
|
CADDY_HOST: '$DOMAIN, http://, https://'
|
|
depends_on:
|
|
- web
|
|
- livestream
|
|
objectstorage:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: objectstorage
|
|
|
|
restart: on-failure
|
|
volumes:
|
|
- objectstorage:/data
|
|
ports:
|
|
- '19000:19000'
|
|
- '19001:19001'
|
|
|
|
seaweedfs:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: seaweedfs
|
|
restart: on-failure
|
|
volumes:
|
|
- seaweedfs:/data
|
|
healthcheck:
|
|
test:
|
|
[
|
|
'CMD',
|
|
'sh',
|
|
'-c',
|
|
"echo 's3.bucket.list' | /usr/bin/weed shell -main=localhost:9333 2>&1 | grep -q insights",
|
|
]
|
|
interval: 5s
|
|
timeout: 10s
|
|
retries: 30
|
|
start_period: 10s
|
|
|
|
asyncmigrationscheck:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: asyncmigrationscheck
|
|
image: $REGISTRY_URL:$INSIGHTS_APP_TAG
|
|
environment:
|
|
SITE_URL: https://$DOMAIN
|
|
SECRET_KEY: $INSIGHTS_SECRET
|
|
SKIP_ASYNC_MIGRATIONS_SETUP: 0
|
|
|
|
# Temporal containers
|
|
temporal:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: temporal
|
|
environment:
|
|
- ENABLE_ES=false
|
|
ports:
|
|
- 7233:7233
|
|
volumes:
|
|
- ./insights/docker/temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
|
|
elasticsearch:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: elasticsearch
|
|
temporal-admin-tools:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: temporal-admin-tools
|
|
depends_on:
|
|
- temporal
|
|
temporal-ui:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: temporal-ui
|
|
ports:
|
|
- 8081:8080
|
|
depends_on:
|
|
temporal:
|
|
condition: service_started
|
|
db:
|
|
condition: service_healthy
|
|
temporal-django-worker:
|
|
command: /compose/temporal-django-worker
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: temporal-django-worker
|
|
volumes:
|
|
- ./compose:/compose
|
|
image: $REGISTRY_URL:$INSIGHTS_APP_TAG
|
|
environment:
|
|
SITE_URL: https://$DOMAIN
|
|
SECRET_KEY: $INSIGHTS_SECRET
|
|
depends_on:
|
|
- db
|
|
- kv
|
|
- datastore
|
|
- kafka
|
|
- objectstorage
|
|
- seaweedfs
|
|
- temporal
|
|
|
|
cyclotron-janitor:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: cyclotron-janitor
|
|
build:
|
|
context: ./insights/rust
|
|
environment:
|
|
DATABASE_URL: 'postgres://insights:insights@db:5432/insights'
|
|
KAFKA_HOSTS: 'kafka:9092'
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
kafka:
|
|
condition: service_started
|
|
|
|
capture:
|
|
build:
|
|
context: ./insights/rust
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: capture
|
|
|
|
replay-capture:
|
|
build:
|
|
context: ./insights/rust
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: replay-capture
|
|
|
|
property-defs-rs:
|
|
build:
|
|
context: ./insights/rust
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: property-defs-rs
|
|
depends_on:
|
|
kafka-init:
|
|
condition: service_completed_successfully
|
|
db:
|
|
condition: service_healthy
|
|
|
|
livestream:
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: livestream
|
|
environment:
|
|
- LIVESTREAM_JWT_SECRET=${INSIGHTS_SECRET}
|
|
volumes:
|
|
- ./insights/docker/livestream/configs-hobby.yml:/configs/configs.yml
|
|
|
|
feature-flags:
|
|
build:
|
|
context: ./insights/rust
|
|
extends:
|
|
file: docker-compose.base.yml
|
|
service: feature-flags
|
|
depends_on:
|
|
- db
|
|
- kv
|
|
|
|
kafka-init:
|
|
image: docker.redpanda.com/redpandadata/redpanda:latest
|
|
entrypoint: /bin/sh
|
|
command:
|
|
- -c
|
|
- |
|
|
set -x
|
|
echo "Waiting for Kafka broker to accept connections..."
|
|
TIMEOUT=60
|
|
ELAPSED=0
|
|
# Use rpk topic list as the readiness check - it's a simple operation
|
|
# that confirms the broker is accepting Kafka protocol requests
|
|
until rpk topic list --brokers kafka:9092 2>/dev/null; do
|
|
echo "Kafka broker not ready yet (elapsed: ${ELAPSED}s)..."
|
|
sleep 2
|
|
ELAPSED=$((ELAPSED + 2))
|
|
if [ $ELAPSED -ge $TIMEOUT ]; then
|
|
echo "Timeout waiting for Kafka broker after ${TIMEOUT}s"
|
|
echo "Final attempt to list topics:"
|
|
rpk topic list --brokers kafka:9092 || true
|
|
exit 1
|
|
fi
|
|
done
|
|
echo "Kafka broker is accepting requests, creating topics..."
|
|
# Create topics needed by rust services
|
|
# Note: $$ escapes $ for docker-compose variable substitution
|
|
for topic in exceptions_ingestion datastore_events_json; do
|
|
if rpk topic create "$$topic" --brokers kafka:9092 -p 1 -r 1 2>&1; then
|
|
echo "Topic $$topic created successfully"
|
|
else
|
|
if rpk topic list --brokers kafka:9092 | grep -q "$$topic"; then
|
|
echo "Topic $$topic already exists, continuing"
|
|
else
|
|
echo "Failed to create topic $$topic"
|
|
exit 1
|
|
fi
|
|
fi
|
|
done
|
|
echo "Final topic list:"
|
|
rpk topic list --brokers kafka:9092
|
|
echo "Topics ready"
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
|
|
cymbal:
|
|
image: ghcr.i./insights/insights/cymbal:main
|
|
build:
|
|
context: ./insights/rust
|
|
args:
|
|
BIN: cymbal
|
|
restart: on-failure
|
|
volumes:
|
|
- ./share:/share
|
|
environment:
|
|
KAFKA_HOSTS: 'kafka:9092'
|
|
KAFKA_CONSUMER_GROUP: cymbal
|
|
KAFKA_CONSUMER_TOPIC: exceptions_ingestion
|
|
OBJECT_STORAGE_BUCKET: insights
|
|
OBJECT_STORAGE_ACCESS_KEY_ID: 'any'
|
|
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'any'
|
|
OBJECT_STORAGE_ENDPOINT: 'http://seaweedfs:8333'
|
|
OBJECT_STORAGE_FORCE_PATH_STYLE: 'true'
|
|
BIND_HOST: '0.0.0.0'
|
|
BIND_PORT: '3302'
|
|
DATABASE_URL: 'postgres://insights:insights@db:5432/insights'
|
|
PERSONS_URL: 'postgres://insights:insights@db:5432/insights'
|
|
MAXMIND_DB_PATH: '/share/GeoLite2-City.mmdb'
|
|
REDIS_URL: 'redis://kv:6379/'
|
|
ISSUE_BUCKETS_REDIS_URL: 'redis://kv:6379/'
|
|
RUST_LOG: 'info'
|
|
depends_on:
|
|
kafka-init:
|
|
condition: service_completed_successfully
|
|
seaweedfs:
|
|
condition: service_healthy
|
|
db:
|
|
condition: service_healthy
|
|
kv:
|
|
condition: service_healthy
|
|
|
|
volumes:
|
|
objectstorage:
|
|
seaweedfs:
|
|
postgres-data:
|
|
datastore-data:
|
|
caddy-data:
|
|
caddy-config:
|
|
kv-data:
|
|
kafka-data:
|
|
redpanda-data:
|