Files
zeekay 19fdf55ceb feat(auth): one key scheme, minted and read in one place
A key now opens with what it is allowed to touch. `pk-` is publishable and
safe in a browser; `sk-` is secret and never leaves a server. The five marks
it replaces all began `ph` -- the upstream's initials -- and they were the
last of its brand a customer read back to us, in every SDK snippet. No word
rule could reach them, because `ph` is not a word.

The marks were also not telling the truth. The constant said `phc_` while
every live project token said `hi_`, so the middleware that routes on the
constant matched no real token at all. Two spellings existed and one was
fiction. Minting now happens in exactly one function and reading in exactly
one, so a second spelling has nowhere to live.

`phx_` (personal) and `phs_` (project secret) both become `sk-`. The mark
now says only that a value is secret; which store holds it is settled by
asking. That was already how personal keys worked -- keys minted before the
marks carry none, so the store always decided -- and both secret resolvers
already returned None on a miss and fell through, so one shared extractor
replaces the two that could drift apart.

OAuth's access and refresh tokens stay distinguishable as `at-` and `rt-`,
because the parser tells them apart before either is looked up, and the
domain-verification token becomes `vt-` rather than keeping a sixth mint of
its own.

`bin/debrand --check` now refuses the marks as a shape, so a seventh cannot
be coined. Two rules that were about to rename things that are not ours to
spell are bounded: `Lemon` no longer eats `LemonSqueezy`, and the `hog_flow`
rules no longer reach a Postgres table, a Datastore table or a route.

Keys are stored hashed, so a mark cannot be rewritten in place: the three
live project tokens have to be reminted.
2026-08-08 19:42:44 -07:00

747 lines
26 KiB
YAML
Executable File

# Process registry for the intent-based dev environment.
#
# This file defines all available processes and their capabilities.
# Normally, `insightscli dev:generate` creates a tailored config at
# .insights/.generated/mprocs.yaml based on your selected intents.
# This file is used as fallback when insightscli isn't available.
#
# To customize which processes run, use `insightscli dev:setup`.
# See devenv/intent-map.yaml for intent-to-capability mappings.
procs:
info:
shell: |
echo ''
printf '\033[38;2;245;78;0m\033[1m Insights Dev Environment\033[0m\n'
printf '\033[38;5;245m ─────────────────────────────────────\033[0m\n'
echo ''
if [ -f devenv/news.txt ]; then
printf ' \033[38;2;245;78;0m\033[1mNews:\033[0m\n'
while IFS= read -r line || [ -n "$line" ]; do
[ -z "$line" ] && continue
printf ' \033[38;5;245m·\033[0m %s\n' "$line"
done < devenv/news.txt
echo ''
fi
printf ' \033[1mCommands:\033[0m\n'
printf ' \033[38;2;29;74;255minsightscli dev:setup\033[0m Configure which services run\n'
printf ' \033[38;2;29;74;255minsightscli dev:explain\033[0m Show why each service is running\n'
echo ''
printf '\033[38;5;245m ─────────────────────────────────────\033[0m\n'
printf ' \033[38;5;245mRun \033[0m\033[38;2;29;74;255minsightscli dev:setup\033[0m\033[38;5;245m to tailor this to your workflow.\033[0m\n'
groups:
layer: pinned
tech: pinned
capability: pinned
backend:
sandbox: true
shell: 'bin/wait-for-docker && ./bin/start-backend'
autorestart: true
ready_pattern: 'Listening at'
groups:
layer: Application
tech: Python
# no capability: always_required via intent-map.yaml
celery-worker:
sandbox: true
shell: 'bin/wait-for-docker && ./bin/start-celery worker'
autorestart: true
capability: celery_workers
ready_pattern: 'celery@'
groups:
layer: Processing
tech: Python
celery-beat:
sandbox: true
shell: 'bin/wait-for-docker && ./bin/start-celery beat'
autorestart: true
capability: celery_scheduler
ready_pattern: 'celery beat'
groups:
layer: Processing
tech: Python
nodejs:
sandbox: true
shell: 'bin/wait-for-docker && ./bin/insights-node'
capability: event_ingestion
ready_pattern: 'All systems go'
groups:
layer: Processing
tech: Node
# Node.js capabilities are controlled via NODEJS_CAPABILITY_GROUPS env var
# Set by insightscli dev:generate based on selected nodejs_* capabilities
# Valid groups: cdp_workflows, realtime_cohorts, session_replay, logs, traces, metrics, feature_flags
# Example: NODEJS_CAPABILITY_GROUPS=cdp_workflows,session_replay
ingestion-errortracking:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-errortracking HTTP_SERVER_PORT=6742 ./bin/insights-node'
capability: nodejs_error_tracking
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
ingestion-logs:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-logs HTTP_SERVER_PORT=6743 ./bin/insights-node'
capability: nodejs_logs
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
ingestion-metrics:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-metrics HTTP_SERVER_PORT=6744 ./bin/insights-node'
capability: nodejs_metrics
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
ingestion-traces:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-traces HTTP_SERVER_PORT=6745 ./bin/insights-node'
capability: nodejs_traces
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
ingestion-ai:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-v2 INGESTION_PIPELINE=ai HTTP_SERVER_PORT=6746 ./bin/insights-node'
capability: ai_capture
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
env:
INGESTION_CONSUMER_CONSUME_TOPIC: ai_events_ingestion
INGESTION_CONSUMER_GROUP_ID: ingestion-ai
PERSONFN_ENABLED: 'true'
PERSONFN_ADDR: '127.0.0.1:50052'
AI_BLOB_S3_BUCKET: ai-blobs
AI_BLOB_S3_PREFIX: 'aio/'
AI_BLOB_S3_ENDPOINT: 'http://localhost:19000'
AI_BLOB_S3_REGION: us-east-1
AI_BLOB_S3_ACCESS_KEY_ID: object_storage_root_user
AI_BLOB_S3_SECRET_ACCESS_KEY: object_storage_root_password
AI_BLOB_OFFLOAD_TEAMS: '1'
ingestion:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=ingestion-v2-combined HTTP_SERVER_PORT=6739 ./bin/insights-node'
capability: event_ingestion
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
ingestion-sessionreplay:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=recordings-blob-ingestion-v2 HTTP_SERVER_PORT=6740 ./bin/insights-node'
capability: nodejs_session_replay
ready_pattern: 'All systems go'
groups:
layer: Ingestion pipeline
tech: Node
recording-api:
sandbox: true
shell: 'bin/wait-for-docker && PLUGIN_SERVER_MODE=recording-api HTTP_SERVER_PORT=6741 ./bin/insights-node'
capability: nodejs_session_replay
ready_pattern: 'All systems go'
groups:
layer: Product services
tech: Node
frontend:
sandbox: true
shell: './bin/start-frontend'
autorestart: true
ready_pattern: 'VITE.*ready in'
groups:
layer: Application
tech: Frontend
# no capability: always_required via intent-map.yaml
typegen:
sandbox: true
shell: 'cd frontend && pnpm run typegen:watch'
autostart: false
ask_skip: true # wizard asks if user wants to skip this
ready_pattern: 'Starting TypeScript watch mode'
groups:
layer: Tools
tech: Frontend
temporal-worker:
sandbox: true
shell: |-
bin/wait-for-docker temporal && \
bin/check_ducklake_up && \
python manage.py register_temporal_search_attributes && \
if [ -n "$TEMPORAL_DISABLE_HOT_RELOAD" ] || ! command -v nodemon >/dev/null 2>&1; then \
python manage.py start_temporal_worker --task-queue development-task-queue --max-concurrent-workflow-tasks=1000 --max-concurrent-activities=1000; \
else \
nodemon -w common -w dags -w ee -w insights -w products -w pyproject.toml -e py --signal SIGTERM --exec "python manage.py start_temporal_worker --task-queue development-task-queue --max-concurrent-workflow-tasks=1000 --max-concurrent-activities=1000"; \
fi
capability: temporal_workflows
groups:
layer: Processing
tech: Python
webhook-s3-sink:
sandbox: true
shell: |-
bin/wait-for-docker && \
python manage.py run_webhook_s3_sink --consumer-group dwh-webhook-sink-dev --health-port 8091
capability: warehouse_webhook_sink
ready_pattern: 'webhook_s3_sink_starting'
groups:
layer: Processing
tech: Python
stripe-mock:
sandbox: true
shell: 'bin/start-stripe-mock'
capability: stripe_mock
ready_pattern: 'Application startup complete'
groups:
layer: Product services
tech: Python
recording-rasterizer:
shell: |-
bin/wait-for-docker temporal && \
bin/temporal-recording-rasterizer-worker
capability: recording_rasterizer
ready_pattern: 'worker started'
groups:
layer: Product services
tech: Other
env:
VIDEO_EXPORT_OBJECT_STORAGE_ENDPOINT: 'http://127.0.0.1:19000'
VIDEO_EXPORT_OBJECT_STORAGE_REGION: 'us-east-1'
AWS_ACCESS_KEY_ID: 'object_storage_root_user'
AWS_SECRET_ACCESS_KEY: 'object_storage_root_password'
SECRET_KEY: '<randomly generated secret key>'
dagster:
sandbox: true
shell: |-
bin/wait-for-docker && \
bin/check_ducklake_up && \
dagster dev --workspace $DAGSTER_HOME/workspace.yaml -p $DAGSTER_UI_PORT
capability: dagster_pipelines
ready_pattern: 'Launching Dagster services'
groups:
layer: Processing
tech: Python
# The shell below is generated by build_static_docker_compose_shell in
# tools/insightscli-commands/insightscli_commands/devenv/generator.py. Edit it there and run
# `insightscli dev:regenerate-mprocs`; CI pre-flight fails if the two drift apart.
docker-compose:
shell: 'if [ "${INSIGHTS_SANDBOX:-}" = "1" ]; then echo "Sandbox: infra managed by docker compose externally" && echo "docker-compose ready" && sleep infinity; else docker compose -p insights -f docker-compose.dev.yml -f docker-compose.profiles.yml --profile dev_tools --profile etcd --profile localstack --profile observability --profile replay --profile temporal up --pull always -d || { echo "docker compose up failed (see the error above). A common cause: another stack or process holding a required port (see the pre-flight output from bin/start). Inspect with: docker ps --all"; exit 1; } && echo "docker-compose ready" && docker compose -p insights -f docker-compose.dev.yml -f docker-compose.profiles.yml --profile dev_tools --profile etcd --profile localstack --profile observability --profile replay --profile temporal logs --tail=100 -f; fi'
capability: core_infra
ready_pattern: 'docker-compose ready'
groups:
layer: Infrastructure
tech: Docker
cyclotron-janitor:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cyclotron-janitor
capability: cyclotron
ready_pattern: 'Running janitor loop'
groups:
layer: Processing
tech: Rust
ai-gateway:
shell: 'bin/wait-for-docker && bin/start-ai-gateway'
capability: ai_gateway
ready_pattern: 'listening on'
groups:
layer: Product services
tech: Go
cymbal:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cymbal
capability: error_symbolication
ready_pattern: 'Server started and listening on *'
groups:
layer: Product services
tech: Rust
cymbal-resolution:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cymbal-resolution
capability: error_symbolication
ready_pattern: 'gRPC server listening on *'
groups:
layer: Product services
tech: Rust
cymbal-notifications:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cymbal-notifications
capability: error_notifications
ready_pattern: 'Starting cymbal-notifications consumer'
groups:
layer: Product services
tech: Rust
embedding-worker:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service embedding-worker
capability: embedding_service
ready_pattern: 'Generated embedding'
groups:
layer: Product services
tech: Rust
feature-flags:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service feature-flags
capability: flag_evaluation
ready_pattern: 'listening on *'
groups:
layer: Product services
tech: Rust
flags-consumer:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service flags-consumer
# POC skeleton — opt-in until the CDC consumer logic lands
autostart: false
ready_pattern: 'listening on *'
groups:
layer: Product services
tech: Rust
hypercache-server:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service hypercache-server
# no capability: always_required via intent-map.yaml
ready_pattern: 'listening on *'
groups:
layer: Product services
tech: Rust
livestream:
shell: |-
bin/wait-for-docker redis-cluster && \
bin/start-go-service livestream
capability: livestream
ready_pattern: 'Livestream service ready'
groups:
layer: Product services
tech: Other
property-defs-rs:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service property-defs-rs
capability: property_definitions
ready_pattern: 'Subscribed to topic: *'
groups:
layer: Ingestion pipeline
tech: Rust
property-vals-rs:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service property-vals-rs
capability: property_values
ready_pattern: 'Subscribed to topic: *'
groups:
layer: Ingestion pipeline
tech: Rust
capture:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service capture
capability: event_ingestion
ready_pattern: 'listening on *'
groups:
layer: Ingestion pipeline
tech: Rust
capture-replay:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service capture-replay
capability: replay_storage
ready_pattern: 'listening on *'
groups:
layer: Ingestion pipeline
tech: Rust
capture-ai:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service capture-ai
capability: ai_capture
ready_pattern: 'listening on *'
groups:
layer: Ingestion pipeline
tech: Rust
capture-logs:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service capture-logs
capability: logs_ingestion
groups:
layer: Ingestion pipeline
tech: Rust
batch-import-worker:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service batch-import-worker
capability: batch_imports
groups:
layer: Ingestion pipeline
tech: Rust
personinsights-replica:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-replica
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
personinsights-router:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-router
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
personinsights-identity:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-identity
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
personinsights-etcd-init:
shell: |-
bin/wait-for-docker && \
docker exec insights-etcd-1 etcdctl put /personinsights/config/total_partitions 4 && \
echo 'etcd init complete'
capability: personinsights
ready_pattern: 'etcd init complete'
autostart: true
groups:
layer: Product services
tech: Rust
personinsights-router-leader:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-router-leader
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
personinsights-leader:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-leader
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
personinsights-writer:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service personinsights-writer
capability: personinsights
ready_pattern: 'Metrics server listening on *'
groups:
layer: Product services
tech: Rust
# Behavioral Cohorts stream rebuild (Rust). The shuffler keys datastore_events_json into
# cohort_stream_events; the processor consumes that and emits membership changes. Opt-in
# (autostart: false).
cohort-event-shuffler:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cohort-event-shuffler
autostart: false
ready_pattern: 'observability server starting'
groups:
layer: Product services
tech: Rust
cohort-stream-processor:
shell: |-
bin/wait-for-docker && \
bin/start-rust-service cohort-stream-processor
autostart: false
ready_pattern: 'observability server starting'
groups:
layer: Product services
tech: Rust
llm-gateway:
sandbox: true
shell: 'bin/wait-for-docker && bin/start-llm-gateway'
capability: llm_gateway
ready_pattern: 'Application startup complete'
groups:
layer: Product services
tech: Go
env:
LLM_GATEWAY_DEBUG: 'true'
LLM_GATEWAY_TEAM_RATE_LIMIT_MULTIPLIERS: '{"1": 10}'
LLM_GATEWAY_INSIGHTS_PROJECT_TOKEN: 'pk-localinsightsprojecttoken'
LLM_GATEWAY_INSIGHTS_HOST: 'http://localhost:8010'
agent-proxy:
shell: 'bin/wait-for-docker --only redis7 && pnpm --filter @hanzo/agent-proxy dev'
capability: agent_proxy
ready_pattern: 'server:started'
groups:
layer: Product services
tech: Node
mcp:
sandbox: true
shell: 'bin/start-mcp-server'
capability: mcp_server
ready_pattern: 'Server started on'
groups:
layer: Product services
tech: Node
mcp-ui-apps:
sandbox: true
shell: 'cd services/mcp && pnpm run build:ui-apps -- --watch'
capability: mcp_server
ready_pattern: 'Initial build complete'
groups:
layer: Product services
tech: Frontend
mcp-inspector:
sandbox: true
shell: 'cd services/mcp && pnpm run inspector'
capability: mcp_server
autostart: false
ready_pattern: 'MCP Inspector'
groups:
layer: Tools
tech: Frontend
stripe-app:
sandbox: true
shell: 'bin/start-stripe-app'
capability: stripe_app
groups:
layer: Product services
tech: Python
desktop:
shell: 'bin/start-desktop'
capability: desktop_app
groups:
layer: Application
tech: Node
migrate-postgres:
shell: 'bin/wait-for-docker && bin/insightscli db:restore-schema-if-fresh && bin/migrate --scope=postgres'
capability: core_infra
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
ensure-local-setup:
# Idempotent local-dev bootstrap (feature flags, dev API key, wizard OAuth app). phrocs has no
# depends_on, so wait for the tables it touches before running (same approach as
# migrate-datastore waiting on insights_instancesetting).
shell: 'bin/wait-for-docker && bin/wait-for-postgres-tables insights_featureflag insights_personalapikey insights_oauthapplication && bin/ensure-local-setup'
capability: core_infra
ready_pattern: 'Local setup complete'
# Unlike the other Tools-layer entries (manual `autostart: false` utilities), this one
# must run automatically every startup to keep local dev bootstrapped.
autostart: true
groups:
layer: Tools
tech: Setup
migrate-datastore:
# Wait on insights_instancesetting because Django queries that table at
# import time -- it's created by migrate-postgres. Without the guard,
# migrate-datastore races migrate-postgres on a fresh DB and crashes
# with ProgrammingError. phrocs has no depends_on field, so the
# dependency lives here in the shell command.
shell: 'bin/wait-for-docker && bin/wait-for-postgres-tables insights_instancesetting && bin/migrate --scope=datastore'
capability: core_infra
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
migrate-cyclotron:
shell: 'bin/wait-for-docker && bin/migrate --scope=cyclotron'
capability: cyclotron
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
migrate-persons-db:
shell: 'bin/wait-for-docker && bin/migrate --scope=persons'
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
# no capability - optional migration
migrate-behavioral-cohorts:
shell: 'bin/wait-for-docker && bin/migrate --scope=behavioral-cohorts'
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
# no capability - optional migration
migrate-flags-read-store:
shell: 'bin/wait-for-docker && bin/migrate --scope=flags-read-store'
autostart: false
ready_pattern: 'All migrations completed successfully'
groups:
layer: Infrastructure
tech: Migrations
# no capability - optional migration
generate-demo-data:
sandbox: true
shell: |-
bin/wait-for-docker && \
./manage.py generate_demo_data
autostart: false
ready_pattern: 'Demo data ready'
groups:
layer: Tools
tech: Python
# no capability - manual start
sync-feature-flags:
sandbox: true
shell: |-
bin/wait-for-docker && \
./manage.py sync_feature_flags
autostart: false
ready_pattern: 'Feature flag sync complete'
groups:
layer: Tools
tech: Python
# no capability - manual start
sync-script-function-templates:
sandbox: true
shell: |-
bin/wait-for-docker && \
./manage.py sync_insights_function_templates
autostart: false
ready_pattern: 'Script function template sync complete'
groups:
layer: Tools
tech: Python
# no capability - manual start
storybook:
sandbox: true
# dev-open-when-ready is peeled out to run unsandboxed (it opens the browser
# once :6006 is up — a path the sandbox denies); the rest stays sandboxed.
shell: 'bin/dev-open-when-ready http://localhost:6006 && pnpm --filter=@hanzo/storybook install && pnpm run storybook'
autostart: false
ready_pattern: 'Storybook *'
groups:
layer: Tools
tech: Frontend
# no capability - manual start
hedgebox-dummy:
sandbox: true
shell: 'bin/wait-for-docker && cd tools/hedgebox-dummy && pnpm install && pnpm run dev'
autostart: false
ready_pattern: 'Ready in *'
groups:
layer: Tools
tech: Frontend
# no capability - manual start
warehouse-sources-load:
sandbox: true
shell: |-
bin/wait-for-docker && \
python manage.py run_warehouse_sources_load
autostart: false
ready_pattern: 'batch_consumer_started'
groups:
layer: Tools
tech: Python
# no capability - manual start
warehouse-sources-duckgres-load:
shell: |-
bin/wait-for-docker && \
python manage.py run_warehouse_sources_duckgres_load --health-port 8092
autostart: false
ready_pattern: 'duckgres_batch_consumer_started'
groups:
layer: Tools
tech: Python
# no capability - manual start
# Grouping dimension the sidebar starts grouped by (press `g` to cycle/disable).
default_group: layer
group_order:
layer: [Application, Processing, Ingestion pipeline, Infrastructure, Product services, Tools]
tech: [Python, Frontend, Node, Rust, Docker, Migrations, Other]
# `capability` is inferred: each proc's `capability:` is copied into
# Groups["capability"] at load time. Display order falls back to the order
# capabilities first appear across procs.
mouse_scroll_speed: 1
scrollback: 10000