chore: release v3.22.0

fix

.env.dev-azure.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -35,17 +35,18 @@ EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
 # DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
-# S3 storage
-S3_ENDPOINT=http://localhost:10000/devstoreaccount1
-S3_ACCESS_KEY_ID=devstoreaccount1
-S3_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
-S3_BUCKET_NAME=langfuse
-S3_REGION=auto
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_BATCH_EXPORT_REGION=auto
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:10000/devstoreaccount1
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
@@ -57,7 +58,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
@@ -82,11 +82,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=false
-LANGFUSE_RETURN_FROM_CLICKHOUSE=true
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
-
-LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -34,17 +34,18 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -56,7 +57,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -79,11 +79,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=false
-LANGFUSE_RETURN_FROM_CLICKHOUSE=true
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
-
-LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.legacy.example

@@ -1,81 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-
-# Clickhouse
-CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
-CLICKHOUSE_URL="http://localhost:8123"
-CLICKHOUSE_USER="clickhouse"
-CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_CLUSTER_ENABLED="false"
-
-# Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-# NEXTAUTH_SECRET=""
-NEXTAUTH_URL="http://localhost:3000"
-NEXTAUTH_SECRET="secret"
-
-# Langfuse Cloud Environment
-NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
-
-# Langfuse experimental features
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
-
-# Salt for API key hashing
-SALT="salt"
-
-# Email
-EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
-SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
-
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
-## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
-
-# # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
-LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
-
-# S3 Event Bucket Upload
-## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
-LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
-
-# Set during docker build of application
-# Used to disable environment verification at build time
-# DOCKER_BUILD=1
-
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
-
-# speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.local.example

@@ -1,22 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
-# Next Auth
-NEXTAUTH_SECRET="secret"
-NEXTAUTH_URL="http://localhost:3000"
-
-# feature flag to enable experimental features locally
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
-
-# Redis
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,12 +92,18 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_CLIENT_AUTH_METHOD="client_secret_basic" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -100,16 +111,16 @@ OTEL_SERVICE_NAME="langfuse"
 # Defines the connection url for smtp server. 
 # SMTP_CONNECTION_URL=
 
-# S3 storage, optional, used for exports from the UI
-# S3_ENDPOINT=
-# S3_ACCESS_KEY_ID=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION=
-# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
+# S3 Batch Exports
+# LANGFUSE_S3_BATCH_EXPORT_ENABLED=
+# LANGFUSE_S3_BATCH_EXPORT_BUCKET=
+# LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=
+# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=
+# LANGFUSE_S3_BATCH_EXPORT_REGION=
+# LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=
+# LANGFUSE_S3_BATCH_EXPORT_PREFIX=
 
 # S3 storage for events, optional, used to persist all incoming events
-# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
 # LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
 # Optional prefix to be used within the bucket. Must end with `/` if set
 # LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
@@ -235,14 +246,12 @@ OTEL_SERVICE_NAME="langfuse"
 # CLICKHOUSE_URL=
 # CLICKHOUSE_USER=
 # CLICKHOUSE_PASSWORD=
+# CLICKHOUSE_DB=
 
 # Ingestion
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
-# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-# QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion

.github/workflows/_deploy_ecs_service.yml

@@ -58,7 +58,6 @@ jobs:
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-
       - name: Render AWS ECS Task Definition
         id: render-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1

.github/workflows/codespell.yml

@@ -3,9 +3,14 @@ name: Codespell
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
   pull_request:
-    branches: [main]
+    branches:
+      - "**"
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -10,11 +10,29 @@ on:
   merge_group:
   pull_request:
     branches:
-      - "*"
+      - "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  pre-job:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    timeout-minutes: 15
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          do_not_skip: '["workflow_dispatch"]'
+
   lint:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
@@ -37,10 +55,14 @@ jobs:
   test-docker-build:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -69,6 +91,9 @@ jobs:
   tests-web-sync:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
@@ -82,13 +107,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -105,7 +131,9 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -133,10 +161,15 @@ jobs:
           LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run test-sync
         run: pnpm --filter=web run test-sync
+      - name: run test-client
+        run: pnpm --filter=web run test-client
 
   tests-web-async:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
@@ -151,13 +184,14 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -174,7 +208,9 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev${{ matrix.blob-provider }}.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -206,6 +242,9 @@ jobs:
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
@@ -222,6 +261,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -237,7 +277,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -270,6 +310,9 @@ jobs:
 
   e2e-tests:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
@@ -281,6 +324,7 @@ jobs:
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -310,9 +354,13 @@ jobs:
 
   e2e-server-tests:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -330,14 +378,12 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=true" >> .env
-          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
@@ -448,9 +494,9 @@ jobs:
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
@@ -469,13 +515,16 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse-worker # GitHub
             langfuse/langfuse-worker # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/snyk.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   snyk:
@@ -15,13 +20,13 @@ jobs:
       - name: Build a Docker image
         run: docker compose -f docker-compose.build.yml up -d
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
         continue-on-error: true
         uses: snyk/actions/docker@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-server
+          image: langfuse-langfuse-web
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
@@ -36,7 +41,7 @@ jobs:
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-worker
+          image: langfuse-langfuse-worker
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning

.gitignore

@@ -38,9 +38,7 @@ yarn-error.log*
 .env*
 !.env.dev.example
 !.env.dev-azure.example
-!.env.local.example
 !.env.prod.example
-!.env.dev.legacy.example
 
 # vercel
 .vercel

CONTRIBUTING.md

@@ -121,36 +121,40 @@ flowchart TB
         end
     end
 
-    DB[Postgres Database]
-    Redis[Redis Cache/Queue]
-    Clickhouse[Clickhouse Database]
+    subgraph s9 ["VPC (US and EU separated)"]
+        DB[Postgres Database]
+        Redis[Redis Cache/Queue]
+        Clickhouse[Clickhouse Database]
+
+        subgraph s1["Application (langfuse/langfuse/web)"]
+            API[Public HTTP API]
+            G[TRPC API]
+            I[NextAuth]
+            H[React Frontend]
+            ORM
+            H --> G
+            H --> I
+            G --> I
+            G --- ORM
+            API --- ORM
+            I --- ORM
+        end
+
+        subgraph s5["Application (langfuse/langfuse/worker)"]
+            Worker
+        end
+
+        Worker --- DB
+        Worker --- Redis
+        Worker --- Clickhouse
+
+        ORM --- DB
+        ORM --- Redis
+        ORM --- Clickhouse
+
 
-    subgraph s1["Application (langfuse/langfuse/web)"]
-        API[Public HTTP API]
-        G[TRPC API]
-        I[NextAuth]
-        H[React Frontend]
-        ORM
-        H --> G
-        H --> I
-        G --> I
-        G --- ORM
-        API --- ORM
-        I --- ORM
     end
 
-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker
-    end
-
-    Worker --- DB
-    Worker --- Redis
-    Worker --- Clickhouse
-
-    ORM --- DB
-    ORM --- Redis
-    ORM --- Clickhouse
-
     JS --- API
     Python --- API
 ```
@@ -397,7 +401,7 @@ The background color of the following component will be `hsl(var(--primary))` an
 | --primary-accent         | Primary accent color used for branding                             | Layout                           |
 | --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
 | --muted-green            | Muted green for Event label                                        | ObservationTree                  |
-| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-magenta          | Muted magenta for Generation label                                 | ObservationTree                  |
 | --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
 | --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
 | --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
@@ -439,7 +443,7 @@ You can update the default AI models and prices by adding or updating an entry i
 Please note that
 
 - prices are in USD
-- the list is ordered by ID, so make sure to keep this order
+- the list is ordered by ID, so make sure to keep this order and insert new models at the end of the list
 - the `updated_at` field must be updated with the current date in ISO 8601 format. Otherwise, the change will be ignored.
 
 ### Transition period until V3 release

LICENSE

@@ -2,8 +2,8 @@ Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
-- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- All content that resides under the "ee/", "web/src/ee/", and/or "worker/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Langfuse Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -42,9 +42,7 @@
 
 ## Langfuse Overview
 
-_Unmute video for voice-over_
-
-https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+[![Langfuse Overview Video](https://github.com/user-attachments/assets/3926b288-ff61-4b95-8aa1-45d041c70866)](https://langfuse.com/watch-demo)
 
 ### Develop
 
@@ -191,3 +189,54 @@ You can opt-out by setting `TELEMETRY_ENABLED=false`.
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
  </picture>
 </a>
+
+### Open Source Projects Using Langfuse
+
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents)):
+
+| Repository                                                                                                                                                                                                                                                          | Stars |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 54865 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 51531 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/131470832?s=40&v=4" width="20" height="20" alt=""> &nbsp; [lobehub](https://github.com/lobehub) / [lobe-chat](https://github.com/lobehub/lobe-chat)                                           | 49003 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 39093 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37368 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/139558948?s=40&v=4" width="20" height="20" alt=""> &nbsp; [chatchat-space](https://github.com/chatchat-space) / [Langchain-Chatchat](https://github.com/chatchat-space/Langchain-Chatchat)    | 32486 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/128289781?s=40&v=4" width="20" height="20" alt=""> &nbsp; [FlowiseAI](https://github.com/FlowiseAI) / [Flowise](https://github.com/FlowiseAI/Flowise)                                         | 32448 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26931 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/119600397?s=40&v=4" width="20" height="20" alt=""> &nbsp; [twentyhq](https://github.com/twentyhq) / [twenty](https://github.com/twentyhq/twenty)                                              | 24195 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22618 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 15151 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/179202840?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mediar-ai](https://github.com/mediar-ai) / [screenpipe](https://github.com/mediar-ai/screenpipe)                                   | 11037 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/105877416?s=40&v=4" width="20" height="20" alt=""> &nbsp; [formbricks](https://github.com/formbricks) / [formbricks](https://github.com/formbricks/formbricks)                                |  9386 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/76263028?s=40&v=4" width="20" height="20" alt=""> &nbsp; [anthropics](https://github.com/anthropics) / [courses](https://github.com/anthropics/courses)                                       |  8385 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7374 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5391 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/137907881?s=40&v=4" width="20" height="20" alt=""> &nbsp; [promptfoo](https://github.com/promptfoo) / [promptfoo](https://github.com/promptfoo/promptfoo)                                     |  4976 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/157326433?s=40&v=4" width="20" height="20" alt=""> &nbsp; [onlook-dev](https://github.com/onlook-dev) / [onlook](https://github.com/onlook-dev/onlook)                                        |  4141 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2526 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  2061 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1161 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1100 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/18422723?s=40&v=4" width="20" height="20" alt=""> &nbsp; [alishobeiri](https://github.com/alishobeiri) / [thread](https://github.com/alishobeiri/thread)                                      |  1074 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   971 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   823 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   677 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   639 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   514 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   394 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   384 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   370 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   305 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/154247157?s=40&v=4" width="20" height="20" alt=""> &nbsp; [vespperhq](https://github.com/vespperhq) / [vespper](https://github.com/vespperhq/vespper)                                         |   304 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   295 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   291 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/2357342?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dmayboroda](https://github.com/dmayboroda) / [minima](https://github.com/dmayboroda/minima)                                          |   221 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   172 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/148684274?s=40&v=4" width="20" height="20" alt=""> &nbsp; [i-am-alice](https://github.com/i-am-alice) / [3rd-devs](https://github.com/i-am-alice/3rd-devs)                                    |   148 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   138 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/104478511?s=40&v=4" width="20" height="20" alt=""> &nbsp; [babelcloud](https://github.com/babelcloud) / [LLM-RGB](https://github.com/babelcloud/LLM-RGB)                                      |   135 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/15125613?s=40&v=4" width="20" height="20" alt=""> &nbsp; [souzatharsis](https://github.com/souzatharsis) / [tamingLLMs](https://github.com/souzatharsis/tamingLLMs)                           |   129 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169401942?s=40&v=4" width="20" height="20" alt=""> &nbsp; [LibreChat-AI](https://github.com/LibreChat-AI) / [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)                      |   128 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   126 |

careers

@@ -1,13 +1,19 @@
 _We are Hiring_
-Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+Join us in building out Langfuse in Berlin, Germany. Langfuse is the open source LLM engineering platform: we build tooling to help developers [build & improve LLM applications](https://langfuse.com/docs). 
+
+We are an open source company, we hire in person (4+ days a week), we only hire excellent technical talent. Find more information on our [careers page](https://langfuse.com/careers)
 
 _Open Roles_
 
-- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
-- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Product Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Backend Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/1bO16H6-backend-engineer
+- Design Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mDquP95-design-engineer
+- Developer Advocate, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
 
 _More Info_
 
 - https://langfuse.com/careers
 - https://langfuse.com/docs
+- https://langfuse.com/why
 - https://langfuse.com/changelog

docker-compose.build.yml

@@ -1,34 +1,53 @@
-version: "3.5"
-
 services:
-  server:
+  langfuse-web:
     build:
       dockerfile: ./web/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: &langfuse-depends-on
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
     ports:
       - "3000:3000"
-    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+    environment: &langfuse-web-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      NEXTAUTH_SECRET: mysecret
+      SALT: mysalt
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: http://localhost:3000
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -36,26 +55,17 @@ services:
       timeout: 10s
       retries: 3
 
-  worker:
+  langfuse-worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: *langfuse-depends-on
     ports:
       - "3030:3030"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - PORT=${PORT:-3030}
-      - REDIS_HOST=${REDIS_HOST:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      <<: *langfuse-web-env
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
@@ -63,26 +73,85 @@ services:
       timeout: 10s
       retries: 3
 
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
-    image: redis:7.2.4
+    image: redis:7
+    restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
-    restart: always
     ports:
       - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

docker-compose.v3preview.yml

@@ -1,145 +0,0 @@
-services:
-  langfuse-worker:
-    image: langfuse/langfuse-worker:latest
-    depends_on: &langfuse-depends-on
-      postgres:
-        condition: service_healthy
-      minio:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "3030:3030"
-    environment: &langfuse-worker-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      SALT: "mysalt"
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
-      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
-      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
-      LANGFUSE_ASYNC_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_INGESTION_PROCESSING:-true}
-      LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:-true}
-      LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: ${LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE:-true}
-      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
-      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
-      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
-      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
-      LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENABLED: ${LANGFUSE_S3_MEDIA_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
-
-  langfuse-web:
-    image: langfuse/langfuse:latest
-    depends_on: *langfuse-depends-on
-    ports:
-      - "3000:3000"
-    environment:
-      <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
-      NEXTAUTH_SECRET: mysecret
-      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
-      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
-      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
-      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
-      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
-      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
-      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-
-  clickhouse:
-    image: clickhouse/clickhouse-server
-    user: "101:101"
-    container_name: clickhouse
-    hostname: clickhouse
-    environment:
-      CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
-    volumes:
-      - langfuse_clickhouse_data:/var/lib/clickhouse
-      - langfuse_clickhouse_logs:/var/log/clickhouse-server
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    depends_on:
-      - postgres
-
-  minio:
-    image: minio/minio
-    container_name: minio
-    entrypoint: sh
-    # create the 'langfuse' bucket before starting the service
-    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
-    environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
-    ports:
-      - "9090:9000"
-      - "9091:9001"
-    volumes:
-      - langfuse_minio_data:/data
-    healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
-      interval: 1s
-      timeout: 5s
-      retries: 5
-      start_period: 1s
-
-  redis:
-    image: redis:7
-    restart: always
-    command: >
-      --requirepass ${REDIS_AUTH:-myredissecret}
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 3s
-      timeout: 10s
-      retries: 10
-
-  postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
-    restart: always
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 3s
-      timeout: 3s
-      retries: 10
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
-    ports:
-      - 5432:5432
-    volumes:
-      - langfuse_postgres_data:/var/lib/postgresql/data
-
-volumes:
-  langfuse_postgres_data:
-    driver: local
-  langfuse_clickhouse_data:
-    driver: local
-  langfuse_clickhouse_logs:
-    driver: local
-  langfuse_minio_data:
-    driver: local

docker-compose.yml

@@ -1,31 +1,129 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:3
+    restart: always
+    depends_on: &langfuse-depends-on
+      postgres:
         condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
+      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:3
+    restart: always
+    depends_on: *langfuse-depends-on
     ports:
       - "3000:3000"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
 
-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    restart: always
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -33,14 +131,20 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

ee/package.json

@@ -27,8 +27,9 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "https-proxy-agent": "^7.0.6",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +48,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8"
     }
   }
 }

ee/src/env.ts

@@ -1,12 +1,9 @@
 import { z } from "zod";
-import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
+import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = {
-  ...sharedEnv,
-  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
-};
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

fern/apis/client/generators.yml

@@ -12,7 +12,7 @@ groups:
       #     namespaceExport: Langfuse
       #     allowCustomFetcher: true
       - name: fernapi/fern-openapi
-        version: 0.0.26
+        version: 0.1.7
         output:
           location: local-file-system
           path: ../../../web/public/generated/api-client

fern/apis/server/definition/comments.yml

@@ -58,7 +58,7 @@ types:
         docs: The id of the object to attach the comment to. If this does not reference a valid existing object, an error will be thrown.
       content:
         type: string
-        docs: The content of the comment. May include markdown. Currently limited to 500 characters.
+        docs: The content of the comment. May include markdown. Currently limited to 3000 characters.
       authorUserId:
         type: optional<string>
         docs: The id of the user who created the comment.

fern/apis/server/definition/commons.yml

@@ -126,7 +126,7 @@ types:
         docs: The output data of the observation
       usage:
         type: optional<Usage>
-        docs: The usage data of the observation
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The usage data of the observation
       level:
         type: ObservationLevel
         docs: The level of the observation
@@ -139,6 +139,12 @@ types:
       promptId:
         type: optional<string>
         docs: The prompt ID associated with the observation
+      usageDetails:
+        type: optional<map<string, integer>>
+        docs: The usage details of the observation. Key is the name of the usage metric, value is the number of units consumed. The total key is the sum of all (non-total) usage metrics or the total value ingested.
+      costDetails:
+        type: optional<map<string, double>>
+        docs: The cost details of the observation. Key is the name of the cost metric, value is the cost in USD. The total key is the sum of all (non-total) cost metrics or the total value ingested.
 
   ObservationsView:
     extends: Observation
@@ -163,13 +169,13 @@ types:
         docs: The total price in USD.
       calculatedInputCost:
         type: optional<double>
-        docs: The calculated cost of the input in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the input in USD
       calculatedOutputCost:
         type: optional<double>
-        docs: The calculated cost of the output in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the output in USD
       calculatedTotalCost:
         type: optional<double>
-        docs: The calculated total cost in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated total cost in USD
       latency:
         type: optional<double>
         docs: The latency in seconds.
@@ -178,7 +184,7 @@ types:
         docs: The time to the first token in seconds
 
   Usage:
-    docs: Standard interface for usage and cost
+    docs: (Deprecated. Use usageDetails and costDetails instead.) Standard interface for usage and cost
     properties:
       input:
         docs: Number of input units (e.g. tokens)
@@ -372,10 +378,10 @@ types:
         type: string
       startDate:
         docs: Apply only to generations which are newer than this ISO date.
-        type: optional<date>
+        type: optional<datetime>
       unit:
         docs: Unit used by this model.
-        type: ModelUsageUnit
+        type: optional<ModelUsageUnit>
       inputPrice:
         docs: Price (USD) per input unit
         type: optional<double>

fern/apis/server/definition/ingestion.yml

@@ -11,6 +11,7 @@ service:
         Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
 
         Notes:
+        - Introduction to data model: https://langfuse.com/docs/tracing-data-model
         - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
         - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
@@ -126,6 +127,8 @@ types:
       model: optional<string>
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, double>>
       promptName: optional<string>
       promptVersion: optional<integer>
 
@@ -137,6 +140,8 @@ types:
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
       promptName: optional<string>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, double>>
       promptVersion: optional<integer>
 
   ObservationBody:
@@ -312,3 +317,17 @@ types:
     properties:
       successes: list<IngestionSuccess>
       errors: list<IngestionError>
+
+  OpenAIUsageSchema:
+    properties:
+      prompt_tokens: integer
+      completion_tokens: integer
+      total_tokens: integer
+      prompt_tokens_details: optional<map<string, integer>>
+      completion_tokens_details: optional<map<string, integer>>
+
+  UsageDetails:
+    discriminated: false
+    union:
+      - map<string, integer>
+      - OpenAIUsageSchema

fern/apis/server/definition/media.yml

@@ -99,5 +99,63 @@ types:
         docs: The unique langfuse identifier of a media record
 
   MediaContentType:
-    type: literal<"image/png","image/jpeg","image/jpg","image/webp","audio/mpeg","audio/mp3","audio/wav","text/plain","application/pdf">
+    enum:
+      - value: image/png
+        name: IMAGE_PNG
+      - value: image/jpeg
+        name: IMAGE_JPEG
+      - value: image/jpg
+        name: IMAGE_JPG
+      - value: image/webp
+        name: IMAGE_WEBP
+      - value: image/gif
+        name: IMAGE_GIF
+      - value: image/svg+xml
+        name: IMAGE_SVG_XML
+      - value: image/tiff
+        name: IMAGE_TIFF
+      - value: image/bmp
+        name: IMAGE_BMP
+      - value: audio/mpeg
+        name: AUDIO_MPEG
+      - value: audio/mp3
+        name: AUDIO_MP3
+      - value: audio/wav
+        name: AUDIO_WAV
+      - value: audio/ogg
+        name: AUDIO_OGG
+      - value: audio/oga
+        name: AUDIO_OGA
+      - value: audio/aac
+        name: AUDIO_AAC
+      - value: audio/mp4
+        name: AUDIO_MP4
+      - value: audio/flac
+        name: AUDIO_FLAC
+      - value: video/mp4
+        name: VIDEO_MP4
+      - value: video/webm
+        name: VIDEO_WEBM
+      - value: text/plain
+        name: TEXT_PLAIN
+      - value: text/html
+        name: TEXT_HTML
+      - value: text/css
+        name: TEXT_CSS
+      - value: text/csv
+        name: TEXT_CSV
+      - value: application/pdf
+        name: APPLICATION_PDF
+      - value: application/msword
+        name: APPLICATION_MSWORD
+      - value: application/vnd.ms-excel
+        name: APPLICATION_MS_EXCEL
+      - value: application/zip
+        name: APPLICATION_ZIP
+      - value: application/json
+        name: APPLICATION_JSON
+      - value: application/xml
+        name: APPLICATION_XML
+      - value: application/octet-stream
+        name: APPLICATION_OCTET_STREAM
     docs: The MIME type of the media record

fern/apis/server/definition/models.yml

@@ -58,7 +58,7 @@ types:
         type: optional<datetime>
       unit:
         docs: Unit used by this model.
-        type: commons.ModelUsageUnit
+        type: optional<commons.ModelUsageUnit>
       inputPrice:
         docs: Price (USD) per input unit
         type: optional<double>

fern/apis/server/definition/prompt-version.yml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  prompts: ./prompts.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public/v2
+  endpoints:
+    update:
+      docs: Update labels for a specific prompt version
+      method: PATCH
+      path: /prompts/{name}/versions/{version}
+      path-parameters:
+        name:
+          type: string
+          docs: The name of the prompt
+        version:
+          type: integer
+          docs: Version of the prompt to update
+      request:
+        name: UpdatePromptRequest
+        body:
+          properties:
+            newLabels:
+              type: list<string>
+              docs: New labels for the prompt version. Labels are unique across versions. The "latest" label is reserved and managed by Langfuse.
+      response: prompts.Prompt

fern/apis/server/definition/prompts.yml

@@ -90,6 +90,9 @@ types:
       tags:
         type: optional<list<string>>
         docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   CreateTextPromptRequest:
     properties:
@@ -102,6 +105,9 @@ types:
       tags:
         type: optional<list<string>>
         docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   Prompt:
     union:
@@ -119,6 +125,9 @@ types:
       tags:
         type: list<string>
         docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -59,7 +59,7 @@ service:
             type: optional<commons.ScoreDataType>
             docs: Retrieve only scores with a specific dataType.
           traceTags:
-            type: optional<list<string>>
+            type: optional<string>
             allow-multiple: true
             docs: Only scores linked to traces that include all of these tags will be returned.
       response: GetScoresResponse

fern/apis/server/generators.yml

@@ -3,7 +3,7 @@ groups:
   local:
     generators:
       - name: fernapi/fern-openapi
-        version: 0.0.31
+        version: 0.1.7
         output:
           location: local-file-system
           path: ../../../web/public/generated/api

fern/fern.config.json

@@ -1,4 +1,5 @@
 {
-  "organization": "finto",
+  "organization": "langfuse",
   "version": "0.43.7"
-}
+}
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.92.0",
+  "version": "3.22.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -25,7 +25,6 @@
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate",
     "release": "dotenv -e ../.env -- release-it",
     "prepare": "husky"
   },
@@ -81,10 +80,15 @@
       }
     }
   },
+  "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
     }
-  },
-  "packageManager": "pnpm@9.5.0"
+  }
 }

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/down.sh

@@ -18,23 +18,33 @@ then
     exit 1
 fi
 
-# Construct the database URL
-if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
-else
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+else
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
 fi

packages/shared/clickhouse/scripts/drop.sh

@@ -12,11 +12,16 @@ then
     exit 1
 fi
 
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
 fi
 # Execute the drop command
 migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/seed.ts

@@ -2,7 +2,7 @@ import {
   clickhouseClient,
   ObservationRecordReadType,
 } from "@langfuse/shared/src/server";
-import { Prisma, prisma } from "../../src/db";
+import { prisma } from "../../src/db";
 import { redis } from "@langfuse/shared/src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
 import { createDatasets } from "../../prisma/seed";

packages/shared/clickhouse/scripts/up.sh

@@ -18,23 +18,33 @@ then
     exit 1
 fi
 
-# Construct the database URL
-if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
-else
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+else
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
 fi

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 110",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -64,6 +64,7 @@
     "@langchain/anthropic": "^0.3.8",
     "@langchain/aws": "^0.1.2",
     "@langchain/core": "^0.3.18",
+    "@langchain/google-vertexai": "^0.1.3",
     "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "@prisma/client": "^5.22.0",
@@ -72,16 +73,17 @@
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
-    "bullmq": "^5.12.10",
+    "bullmq": "^5.34.10",
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
     "langchain": "^0.3.6",
     "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
@@ -120,7 +122,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8"
     }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -170,6 +170,17 @@ export type BatchExport = {
     url: string | null;
     log: string | null;
 };
+export type BillingMeterBackup = {
+    stripe_customer_id: string;
+    meter_id: string;
+    start_time: Timestamp;
+    end_time: Timestamp;
+    aggregated_value: number;
+    event_name: string;
+    org_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Comment = {
     id: string;
     project_id: string;
@@ -293,6 +304,8 @@ export type LlmApiKeys = {
     base_url: string | null;
     custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
+    extra_headers: string | null;
+    extra_header_keys: Generated<string[]>;
     config: unknown | null;
     project_id: string;
 };
@@ -453,7 +466,9 @@ export type Project = {
     org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    deleted_at: Timestamp | null;
     name: string;
+    retention_days: number | null;
 };
 export type ProjectMembership = {
     org_membership_id: string;
@@ -477,6 +492,14 @@ export type Prompt = {
     config: Generated<unknown>;
     tags: Generated<string[]>;
     labels: Generated<string[]>;
+    commit_message: string | null;
+};
+export type QueueBackUp = {
+    id: string;
+    project_id: string | null;
+    queue_name: string;
+    content: unknown;
+    created_at: Generated<Timestamp>;
 };
 export type Score = {
     id: string;
@@ -603,6 +626,7 @@ export type DB = {
     audit_logs: AuditLog;
     background_migrations: BackgroundMigration;
     batch_exports: BatchExport;
+    billing_meter_backups: BillingMeterBackup;
     comments: Comment;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
@@ -627,6 +651,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    queue_backups: QueueBackUp;
     score_configs: ScoreConfig;
     scores: Score;
     Session: Session;

packages/shared/prisma/migrations/20241124115100_add_projects_deleted_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN "deleted_at" TIMESTAMP(3);

packages/shared/prisma/migrations/20241206115829_remove_trace_score_observation_constraints/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_output_score_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/migrations/20250108220721_add_queue_backup_table/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "queue_backups" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT,
+    "queue_name" TEXT NOT NULL,
+    "content" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "queue_backups_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_project_id_fkey" FOREIGN KEY ("session_id", "project_id") REFERENCES "trace_sessions"("id", "project_id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250109083346_drop_trace_tracesession_fk/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/migrations/20250116154613_add_billing_meter_backups/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "billing_meter_backups" (
+    "stripe_customer_id" TEXT NOT NULL,
+    "meter_id" TEXT NOT NULL,
+    "start_time" TIMESTAMP(3) NOT NULL,
+    "end_time" TIMESTAMP(3) NOT NULL,
+    
+    "aggregated_value" INTEGER NOT NULL,
+    
+    "event_name" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- CreateIndex
+CREATE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_idx" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_key" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");

packages/shared/prisma/migrations/20250122152102_add_llm_api_keys_extra_headers/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "llm_api_keys"
+    ADD COLUMN "extra_headers" TEXT,
+    ADD COLUMN "extra_header_keys" TEXT[] NOT NULL DEFAULT '{}'::TEXT[];

packages/shared/prisma/migrations/20250123103200_add_retention_days_to_projects/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "projects"
+    ADD COLUMN "retention_days" INTEGER;

packages/shared/prisma/migrations/20250128144418_llm_adapter_rename_google_vertex_ai/migration.sql

@@ -0,0 +1 @@
+UPDATE "llm_api_keys" SET adapter = 'google-vertex-ai' WHERE adapter = 'vertex-ai';

packages/shared/prisma/migrations/20250128163035_add_nullable_commit_message_prompts/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "commit_message" TEXT;

packages/shared/prisma/schema.prisma

@@ -113,7 +113,9 @@ model Project {
   orgId               String                 @map("org_id")
   createdAt           DateTime               @default(now()) @map("created_at")
   updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+  deletedAt           DateTime?              @map("deleted_at")
   name                String
+  retentionDays       Int?                   @map("retention_days")
   projectMembers      ProjectMembership[]
   organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
   traces              Trace[]
@@ -191,6 +193,8 @@ model LlmApiKeys {
   baseURL           String?  @map("base_url")
   customModels      String[] @default([]) @map("custom_models")
   withDefaultModels Boolean  @default(true) @map("with_default_models")
+  extraHeaders      String?  @map("extra_headers")
+  extraHeaderKeys   String[] @default([]) @map("extra_header_keys")
   config            Json?
 
   projectId String  @map("project_id")
@@ -271,7 +275,6 @@ model TraceSession {
   project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
   bookmarked Boolean  @default(false)
   public     Boolean  @default(false)
-  traces     Trace[]
 
   @@id([id, projectId])
   @@index([projectId])
@@ -283,25 +286,24 @@ model TraceSession {
 // Update TraceView below when making changes to this model!
 
 model Trace {
-  id         String        @id @default(cuid())
-  externalId String?       @map("external_id")
-  timestamp  DateTime      @default(now())
+  id         String   @id @default(cuid())
+  externalId String?  @map("external_id")
+  timestamp  DateTime @default(now())
   name       String?
-  userId     String?       @map("user_id")
+  userId     String?  @map("user_id")
   metadata   Json?
   release    String?
   version    String?
-  projectId  String        @map("project_id")
-  project    Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  public     Boolean       @default(false)
-  bookmarked Boolean       @default(false)
-  tags       String[]      @default([])
+  projectId  String   @map("project_id")
+  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  public     Boolean  @default(false)
+  bookmarked Boolean  @default(false)
+  tags       String[] @default([])
   input      Json?
   output     Json?
-  sessionId  String?       @map("session_id")
-  session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
-  createdAt  DateTime      @default(now()) @map("created_at")
-  updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
+  sessionId  String?  @map("session_id")
+  createdAt  DateTime @default(now()) @map("created_at")
+  updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")
 
   @@index([projectId, timestamp])
   @@index([sessionId])
@@ -468,25 +470,24 @@ enum ObservationLevel {
 }
 
 model Score {
-  id            String         @id @default(cuid())
-  timestamp     DateTime       @default(now())
-  projectId     String         @map("project_id")
-  project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  id            String        @id @default(cuid())
+  timestamp     DateTime      @default(now())
+  projectId     String        @map("project_id")
+  project       Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        ScoreSource
-  authorUserId  String?        @map("author_user_id")
+  authorUserId  String?       @map("author_user_id")
   comment       String?
-  traceId       String         @map("trace_id")
-  observationId String?        @map("observation_id")
-  configId      String?        @map("config_id")
-  stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
-  queueId       String?        @map("queue_id")
-  createdAt     DateTime       @default(now()) @map("created_at")
-  updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
-  dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
-  JobExecution  JobExecution[]
-  scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
+  traceId       String        @map("trace_id")
+  observationId String?       @map("observation_id")
+  configId      String?       @map("config_id")
+  stringValue   String?       @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+  queueId       String?       @map("queue_id")
+  createdAt     DateTime      @default(now()) @map("created_at")
+  updatedAt     DateTime      @default(now()) @updatedAt @map("updated_at")
+  dataType      ScoreDataType @default(NUMERIC) @map("data_type")
+  scoreConfig   ScoreConfig?  @relation(fields: [configId], references: [id], onDelete: SetNull)
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -737,14 +738,15 @@ model Prompt {
 
   createdBy String @map("created_by")
 
-  prompt   Json
-  name     String
-  version  Int
-  type     String   @default("text")
-  isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-  config   Json     @default("{}") @db.Json
-  tags     String[] @default([])
-  labels   String[] @default([])
+  prompt        Json
+  name          String
+  version       Int
+  type          String   @default("text")
+  isActive      Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+  config        Json     @default("{}") @db.Json
+  tags          String[] @default([])
+  labels        String[] @default([])
+  commitMessage String?  @map("commit_message")
 
   @@unique([projectId, name, version])
   @@index([projectId, id])
@@ -900,10 +902,9 @@ model JobExecution {
 
   jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
 
-  jobInputDatasetItemId String?      @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
+  jobInputDatasetItemId String? @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
-  score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
   @@index([projectId, status])
   @@index([projectId, id])
@@ -1018,3 +1019,37 @@ model ObservationMedia {
   @@index([projectId, observationId])
   @@map("observation_media")
 }
+
+model QueueBackUp {
+  id        String  @id @default(cuid())
+  projectId String? @map("project_id")
+  queueName String  @map("queue_name")
+  content   Json
+
+  createdAt DateTime @default(now()) @map("created_at")
+
+  @@map("queue_backups")
+}
+
+model BillingMeterBackup {
+  // unique
+  stripeCustomerId String   @map("stripe_customer_id")
+  meterId          String   @map("meter_id")
+  startTime        DateTime @map("start_time")
+  endTime          DateTime @map("end_time")
+
+  // value
+  aggregatedValue Int @map("aggregated_value")
+
+  // labels
+  eventName String @map("event_name")
+  orgId     String @map("org_id")
+
+  // ts
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([stripeCustomerId, meterId, startTime, endTime])
+  @@index([stripeCustomerId, meterId, startTime, endTime])
+  @@map("billing_meter_backups")
+}

packages/shared/prisma/seed.ts

@@ -450,6 +450,9 @@ export async function createDatasets(
           Math.random() > 0.3
             ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
+        if (!sourceObservation) {
+          continue;
+        }
         const datasetItem = await prisma.datasetItem.create({
           data: {
             projectId,
@@ -510,6 +513,9 @@ export async function createDatasets(
               Math.floor(Math.random() * relevantObservations.length)
             ];
 
+          if (!observation) {
+            continue;
+          }
           await prisma.datasetRunItems.create({
             data: {
               projectId,

packages/shared/scripts/prepareClickhouse.ts

@@ -96,21 +96,37 @@ export const prepareClickhouse = async (
       'version' AS version,
       repeat('input', toInt64(randExponential(1 / 100))) AS input,
       repeat('output', toInt64(randExponential(1 / 100))) AS output,
-      case
-        when number % 2 = 0 then 'claude-3-haiku-20240307'
-        else 'gpt-4'
-      end as provided_model_name,
-      case
-        when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
-        else 'clrntkjgy000f08jx79v9g1xj'
-      end as internal_model_id,
-      '{"temperature": 0.7, "max_tokens": 150}' AS model_parameters,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS provided_usage_details,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS usage_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
-      toDecimal64(randUniform(0, 2000), 12) AS total_cost,
-      start_time AS completion_start_time,
+      if("type" = 'GENERATION',
+        case
+          when number % 2 = 0 then 'claude-3-haiku-20240307'
+          else 'gpt-4'
+        end,
+        NULL) as provided_model_name,
+      if("type" = 'GENERATION',
+        case
+          when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
+          else 'clrntkjgy000f08jx79v9g1xj'
+        end,
+        NULL) as internal_model_id,
+      if("type" = 'GENERATION',
+        '{"temperature": 0.7, "max_tokens": 150}',
+        '{}') AS model_parameters,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS provided_usage_details,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS usage_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS provided_cost_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS cost_details,
+      if("type" = 'GENERATION',
+        toDecimal64(randUniform(0, 2000), 12),
+        NULL) AS total_cost,
+      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
       array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
         ",",
       )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,

packages/shared/src/env.ts

@@ -5,6 +5,7 @@ const EnvSchema = z.object({
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
+  NEXTAUTH_URL: z.string().url().optional(),
   REDIS_HOST: z.string().nullish(),
   REDIS_PORT: z.coerce
     .number({
@@ -27,15 +28,12 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
-  CLICKHOUSE_URL: z.string().url().optional(),
-  CLICKHOUSE_USER: z.string().optional(),
-  CLICKHOUSE_PASSWORD: z.string().optional(),
-  LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_ASYNC_INGESTION_PROCESSING: z
-    .enum(["true", "false"])
-    .default("false"),
+  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
+  CLICKHOUSE_DB: z.string().default("default"),
+  CLICKHOUSE_USER: z.string(),
+  CLICKHOUSE_PASSWORD: z.string(),
+
   LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
     .number()
     .nonnegative()
@@ -48,8 +46,9 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
+    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
+  }),
   LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
   LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
   LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
@@ -60,6 +59,16 @@ const EnvSchema = z.object({
     .default("false"),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
   STRIPE_SECRET_KEY: z.string().optional(),
+
+  LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
 });
 
-export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));
+export const env: z.infer<typeof EnvSchema> =
+  process.env.DOCKER_BUILD === "1"
+    ? (process.env as any)
+    : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/errors/ApiError.ts

@@ -1,7 +1,7 @@
 import { BaseError } from "./BaseError";
 
 export class ApiError extends BaseError {
-  constructor(description = "Api call failed") {
-    super("ApiError", 500, description, true);
+  constructor(description = "Api call failed", status = 500) {
+    super("ApiError", status, description, true);
   }
 }

packages/shared/src/features/comments/types.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+const MAX_COMMENT_LENGTH = 3000;
+
 const COMMENT_OBJECT_TYPES = [
   "TRACE",
   "OBSERVATION",
@@ -9,7 +11,7 @@ const COMMENT_OBJECT_TYPES = [
 
 export const CreateCommentData = z.object({
   projectId: z.string(),
-  content: z.string().trim().min(1).max(500),
+  content: z.string().trim().min(1).max(MAX_COMMENT_LENGTH),
   objectId: z.string(),
   objectType: z.enum(COMMENT_OBJECT_TYPES),
 });

packages/shared/src/features/entitlements/plans.ts

@@ -3,15 +3,17 @@ export const planLabels = {
   "cloud:hobby": "Hobby",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
-  "self-hosted:enterprise": "Enterprise",
+  "self-hosted:pro": "Pro (self-hosted)",
+  "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
 
 export type Plan = keyof typeof planLabels;
 
 export const plans = Object.keys(planLabels) as Plan[];
 
-// This function is kept here to ensure consistency when updating plan names in the future.
+// These functions are kept here to ensure consistency when updating plan names in the future.
 export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/features/experiments/utils.ts

@@ -0,0 +1,15 @@
+import { Prisma } from "../../db";
+
+export const datasetItemMatchesVariable = (
+  input: Prisma.JsonValue,
+  variable: string,
+) => {
+  if (
+    input === null ||
+    input === undefined ||
+    typeof input !== "object" ||
+    Array.isArray(input)
+  )
+    return false;
+  return Object.keys(input).includes(variable);
+};

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -6,7 +6,7 @@ import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 
 /**
@@ -17,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx
+  ctx: z.RefinementCtx,
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -94,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value
+          category.value === expectedCategories[index].value,
       );
     }),
 });
@@ -123,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      })
+      }),
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
@@ -170,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType
+  scoreConfig: ScoreConfigDbType,
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -205,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      })
+      }),
     ),
   ])
   .superRefine((data, ctx) => {
@@ -227,7 +227,7 @@ export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
 
 // GET /score-configs
 export const GetScoreConfigsQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
 });
 
 export const GetScoreConfigsResponse = z.object({

packages/shared/src/features/scores/scoreTypes.ts

@@ -6,7 +6,7 @@ import { isPresent, stringDateTime } from "../../utils/typeChecks";
 import {
   NonEmptyString,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 import { Category as ConfigCategory } from "./scoreConfigTypes";
 
@@ -84,13 +84,13 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
     z.object({
       value: z.number(),
       dataType: z.literal("NUMERIC"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -98,7 +98,7 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
         message: "Value must be either 0 or 1",
       }),
       dataType: z.literal("BOOLEAN"),
-    })
+    }),
   ),
 ]);
 
@@ -162,7 +162,7 @@ export const ScorePropsAgainstConfig = z.union([
  */
 export const filterAndValidateDbScoreList = (
   scores: Score[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): APIScore[] =>
   scores.reduce((acc, ts) => {
     const result = APIScoreSchema.safeParse(ts);
@@ -199,14 +199,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       value: z.number(),
       dataType: z.literal("NUMERIC"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -216,14 +216,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       }),
       dataType: z.literal("BOOLEAN"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.union([z.string(), z.number()]),
       dataType: z.undefined(),
       configId: z.string().nullish(),
-    })
+    }),
   ),
 ]);
 
@@ -231,7 +231,7 @@ export const PostScoresResponse = z.object({ id: z.string() });
 
 // GET /scores
 export const GetScoresQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
   userId: z.string().nullish(),
   dataType: z.enum(ScoreDataType).nullish(),
   configId: z.string().nullish(),
@@ -260,7 +260,7 @@ const LegacyGetScoreResponseDataV1 = z.intersection(
       userId: z.string().nullish(),
       tags: z.array(z.string()).nullish(),
     }),
-  })
+  }),
 );
 export const GetScoresResponse = z.object({
   data: z.array(LegacyGetScoreResponseDataV1),
@@ -269,7 +269,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
     (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
@@ -282,7 +282,7 @@ export const legacyFilterAndValidateV1GetScoreList = (
       }
       return acc;
     },
-    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[]
+    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[],
   );
 
 // GET /scores/{scoreId}

packages/shared/src/index.ts

@@ -7,7 +7,6 @@ export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
 export * from "./tableDefinitions/tracesTable";
-export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
@@ -16,7 +15,6 @@ export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
-export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";
@@ -34,6 +32,9 @@ export * from "./features/scores";
 // comments
 export * from "./features/comments/types";
 
+// experiments
+export * from "./features/experiments/utils";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -10,3 +10,19 @@ export const BedrockCredentialSchema = z
   })
   .optional();
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
+
+export const GCPServiceAccountKeySchema = z.object({
+  type: z.literal("service_account"),
+  project_id: z.string(),
+  private_key_id: z.string(),
+  private_key: z.string(),
+  client_email: z.string(),
+  client_id: z.string(),
+  auth_uri: z.string(),
+  token_uri: z.string(),
+  auth_provider_x509_cert_url: z.string(),
+  client_x509_cert_url: z.string(),
+});
+
+export type GCPServiceAccountKey = z.infer<typeof GCPServiceAccountKeySchema>;
+export default GCPServiceAccountKeySchema;

packages/shared/src/observationsTable.ts

@@ -20,6 +20,13 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "type",
+    id: "type",
+    type: "stringOptions",
+    options: [],
+    internal: 'o."type"',
+  },
   { name: "Trace ID", id: "traceId", type: "string", internal: 't."id"' },
   {
     name: "Trace Name",
@@ -112,6 +119,14 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "Model ID",
+    id: "modelId",
+    type: "stringOptions",
+    internal: 'o."internal_model_id"',
+    options: [], // to be added at runtime
+    nullable: true,
+  },
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -186,20 +201,25 @@ export const observationsTableCols: ColumnDefinition[] = [
 // allows for undefined options, to offer filters while options are still loading
 export type ObservationOptions = {
   model: Array<OptionsDefinition>;
+  modelId: Array<OptionsDefinition>;
   name: Array<OptionsDefinition>;
   traceName: Array<OptionsDefinition>;
   scores_avg: Array<string>;
   promptName: Array<OptionsDefinition>;
   tags: Array<OptionsDefinition>;
+  type: Array<OptionsDefinition>;
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions
+  options?: ObservationOptions,
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {
       return { ...col, options: options?.model ?? [] };
     }
+    if (col.id === "modelId") {
+      return { ...col, options: options?.modelId ?? [] };
+    }
     if (col.id === "name") {
       return { ...col, options: options?.name ?? [] };
     }
@@ -215,6 +235,9 @@ export function observationsTableColsWithOptions(
     if (col.id === "tags") {
       return { ...col, options: options?.tags ?? [] };
     }
+    if (col.id === "type") {
+      return { ...col, options: options?.type ?? [] };
+    }
     return col;
   });
 }

packages/shared/src/server/auth/customSsoProvider.ts

@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
     authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
-    idToken: true,
     profile(profile) {
       return {
         id: profile.sub,

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+  options: OAuthUserConfig<P> & {
+    enterprise?: {
+      baseUrl?: string;
+    };
+  },
+): OAuthConfig<P> {
+  const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com";
+  const apiBaseUrl = options?.enterprise?.baseUrl
+    ? `${options?.enterprise?.baseUrl}/api/v3`
+    : "https://api.github.com";
+
+  return {
+    id: "github-enterprise",
+    name: "GitHub Enterprise",
+    type: "oauth",
+    authorization: {
+      url: `${baseUrl}/login/oauth/authorize`,
+      params: { scope: "read:user user:email" },
+    },
+    token: `${baseUrl}/login/oauth/access_token`,
+    userinfo: {
+      url: `${apiBaseUrl}/user`,
+      async request({ client, tokens }) {
+        const profile = await client.userinfo(tokens.access_token!);
+
+        if (!profile.email) {
+          // If the user does not have a public email, get another via the GitHub API
+          // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+          const res = await fetch(`${apiBaseUrl}/user/emails`, {
+            headers: { Authorization: `token ${tokens.access_token}` },
+          });
+
+          if (res.ok) {
+            const emails = (await res.json()) as GithubEmail[];
+            profile.email = (emails.find((e) => e.primary) ?? emails[0]).email;
+          }
+        }
+
+        return profile;
+      },
+    },
+    profile(profile) {
+      return {
+        id: profile.id.toString(),
+        name: profile.name ?? profile.login,
+        email: profile.email,
+        image: profile.avatar_url,
+      };
+    },
+    style: {
+      logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+      logoDark:
+        "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+      bg: "#fff",
+      bgDark: "#000",
+      text: "#000",
+      textDark: "#fff",
+    },
+    options,
+  };
+}

packages/shared/src/server/clickhouse/client.ts

@@ -10,7 +10,7 @@ export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
     url: env.CLICKHOUSE_URL,
     username: env.CLICKHOUSE_USER,
     password: env.CLICKHOUSE_PASSWORD,
-    database: "default",
+    database: env.CLICKHOUSE_DB,
     clickhouse_settings: {
       async_insert: 1,
       wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse

packages/shared/src/server/index.ts

@@ -6,7 +6,9 @@ export * from "./services/PromptService";
 export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
+export * from "./llm/utils";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
@@ -21,17 +23,22 @@ export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
-export * from "./redis/CloudUsageMeteringQueue";
+export * from "./redis/cloudUsageMeteringQueue";
 export * from "./redis/getQueue";
+export * from "./redis/traceDelete";
+export * from "./redis/projectDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
-export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/postHogIntegrationQueue";
+export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/dataRetentionQueue";
+export * from "./redis/dataRetentionProcessingQueue";
+export * from "./redis/coreDataS3ExportQueue";
+export * from "./redis/meteringDataPostgresExportQueue";
 export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
-export * from "./ingestion/legacy/index";
 export * from "./queues";
-export * from "./ingestion/legacy/EventProcessor";
 export * from "./orderByToPrisma";
 export * from "./filterToPrisma";
 export * from "./instrumentation";
@@ -39,3 +46,7 @@ export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
+export * from "./services/sessions-ui-table-service";
+
+// test utils
+export * from "./test-utils";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -1,786 +0,0 @@
-import { v4 } from "uuid";
-import { type z } from "zod";
-import Decimal from "decimal.js";
-import { findModel } from "../modelMatch";
-import {
-  ObservationEvent,
-  eventTypes,
-  legacyObservationCreateEvent,
-  generationCreateEvent,
-  traceEvent,
-  scoreEvent,
-  sdkLogEvent,
-  ingestionEvent,
-} from "../types";
-import { validateAndInflateScore } from "../validateAndInflateScore";
-import { Trace, Observation, Score, Prisma, Model } from "@prisma/client";
-import { ForbiddenError, LangfuseNotFoundError } from "../../../errors";
-import { mergeJson } from "../../../utils/json";
-import { jsonSchema } from "../../../utils/zod";
-import { prisma } from "../../../db";
-import { LegacyIngestionAccessScope } from ".";
-import { logger } from "../../logger";
-import { env } from "../../../env";
-import { upsertTrace } from "../../repositories";
-import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
-
-export interface EventProcessor {
-  auth(apiScope: LegacyIngestionAccessScope): void;
-
-  process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> | undefined;
-}
-
-export const getProcessorForEvent = (
-  event: z.infer<typeof ingestionEvent>,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-): EventProcessor => {
-  switch (event.type) {
-    case eventTypes.TRACE_CREATE:
-      return new TraceProcessor(event);
-    case eventTypes.OBSERVATION_CREATE:
-    case eventTypes.OBSERVATION_UPDATE:
-    case eventTypes.EVENT_CREATE:
-    case eventTypes.SPAN_CREATE:
-    case eventTypes.SPAN_UPDATE:
-    case eventTypes.GENERATION_CREATE:
-    case eventTypes.GENERATION_UPDATE:
-      return new ObservationProcessor(event, calculateTokenDelegate);
-    case eventTypes.SCORE_CREATE: {
-      return new ScoreProcessor(event);
-    }
-    case eventTypes.SDK_LOG:
-      return new SdkLogProcessor(event);
-  }
-};
-
-export class ObservationProcessor implements EventProcessor {
-  event: ObservationEvent;
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined;
-
-  constructor(
-    event: ObservationEvent,
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-  ) {
-    this.event = event;
-    this.calculateTokenDelegate = calculateTokenDelegate;
-  }
-
-  async convertToObservation(
-    apiScope: LegacyIngestionAccessScope,
-    existingObservation: Omit<Observation, "input" | "output"> | null,
-  ): Promise<{
-    id: string;
-    create: Prisma.ObservationUncheckedCreateInput;
-    update: Prisma.ObservationUncheckedUpdateInput;
-  }> {
-    let type: "EVENT" | "SPAN" | "GENERATION";
-    switch (this.event.type) {
-      case eventTypes.OBSERVATION_CREATE:
-      case eventTypes.OBSERVATION_UPDATE:
-        type = this.event.body.type;
-        break;
-      case eventTypes.EVENT_CREATE:
-        type = "EVENT" as const;
-        break;
-      case eventTypes.SPAN_CREATE:
-      case eventTypes.SPAN_UPDATE:
-        type = "SPAN" as const;
-        break;
-      case eventTypes.GENERATION_CREATE:
-      case eventTypes.GENERATION_UPDATE:
-        type = "GENERATION" as const;
-        break;
-    }
-
-    if (
-      this.event.type === eventTypes.OBSERVATION_UPDATE &&
-      !existingObservation
-    ) {
-      throw new LangfuseNotFoundError(
-        `Observation with id ${this.event.id} not found`,
-      );
-    }
-
-    // find matching model definition based on event and existing observation in db
-    const internalModel: Model | undefined | null =
-      type === "GENERATION"
-        ? await findModel({
-            event: {
-              projectId: apiScope.projectId,
-              model:
-                "model" in this.event.body
-                  ? (this.event.body.model ?? undefined)
-                  : undefined,
-              unit:
-                "usage" in this.event.body
-                  ? (this.event.body.usage?.unit ?? undefined)
-                  : undefined,
-              startTime: this.event.body.startTime
-                ? new Date(this.event.body.startTime)
-                : undefined,
-            },
-            existingDbObservation: existingObservation ?? undefined,
-          })
-        : undefined;
-
-    // Token counts
-    const [newInputCount, newOutputCount] =
-      "usage" in this.event.body
-        ? await this.calculateTokenCounts(
-            apiScope.projectId,
-            this.event.body,
-            this.calculateTokenDelegate,
-            internalModel ?? undefined,
-            existingObservation ?? undefined,
-          )
-        : [undefined, undefined];
-
-    const newTotalCount =
-      "usage" in this.event.body
-        ? (this.event.body.usage?.total ??
-          (newInputCount != null || newOutputCount != null
-            ? (newInputCount ?? 0) + (newOutputCount ?? 0)
-            : undefined))
-        : undefined;
-
-    const userProvidedTokenCosts = {
-      inputCost:
-        "usage" in this.event.body && this.event.body.usage?.inputCost != null // inputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.inputCost)
-          : existingObservation?.inputCost,
-      outputCost:
-        "usage" in this.event.body && this.event.body.usage?.outputCost != null // outputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.outputCost)
-          : existingObservation?.outputCost,
-      totalCost:
-        "usage" in this.event.body && this.event.body.usage?.totalCost != null // totalCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.totalCost)
-          : existingObservation?.totalCost,
-    };
-
-    const tokenCounts = {
-      input: newInputCount ?? existingObservation?.promptTokens,
-      output: newOutputCount ?? existingObservation?.completionTokens,
-      total: newTotalCount || existingObservation?.totalTokens,
-    };
-
-    const calculatedCosts = ObservationProcessor.calculateTokenCosts(
-      internalModel,
-      userProvidedTokenCosts,
-      tokenCounts,
-    );
-
-    // merge metadata from existingObservation.metadata and metadata
-    const mergedMetadata = mergeJson(
-      existingObservation?.metadata
-        ? jsonSchema.parse(existingObservation.metadata)
-        : undefined,
-      this.event.body.metadata ?? undefined,
-    );
-
-    const prompt =
-      "promptName" in this.event.body &&
-      typeof this.event.body.promptName === "string" &&
-      "promptVersion" in this.event.body &&
-      typeof this.event.body.promptVersion === "number"
-        ? await prisma.prompt.findUnique({
-            where: {
-              projectId_name_version: {
-                projectId: apiScope.projectId,
-                name: this.event.body.promptName,
-                version: this.event.body.promptVersion,
-              },
-            },
-          })
-        : undefined;
-
-    // Only null if promptName and promptVersion are set but prompt is not found
-    if (prompt === null) {
-      logger.warn("Prompt not found for observation", this.event.body);
-    }
-
-    const observationId =
-      this.event.body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `observation.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    let traceId = this.event.body?.traceId;
-    if (!this.event.body.traceId && !existingObservation) {
-      // Create trace if no traceId
-      traceId = observationId;
-
-      // Insert trace into postgres
-      await prisma.trace.upsert({
-        where: {
-          id: observationId,
-        },
-        create: {
-          projectId: apiScope.projectId,
-          name: this.event.body.name,
-          id: observationId,
-          timestamp: this.event.body.startTime || new Date(),
-        },
-        update: {},
-      });
-
-      if (env.CLICKHOUSE_URL) {
-        // Insert trace into clickhouse if enabled
-        await upsertTrace({
-          id: observationId,
-          project_id: apiScope.projectId,
-          timestamp: convertDateToClickhouseDateTime(
-            this.event.body.startTime
-              ? new Date(this.event.body.startTime)
-              : new Date(),
-          ),
-          created_at: convertDateToClickhouseDateTime(new Date()),
-          updated_at: convertDateToClickhouseDateTime(new Date()),
-        });
-      }
-    }
-
-    return {
-      id: observationId,
-      create: {
-        id: observationId,
-        traceId,
-        type: type,
-        name: this.event.body.name,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        projectId: apiScope.projectId,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-      update: {
-        name: this.event.body.name ?? undefined,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-    };
-  }
-
-  async calculateTokenCounts(
-    projectId: string,
-    body:
-      | z.infer<typeof legacyObservationCreateEvent>["body"]
-      | z.infer<typeof generationCreateEvent>["body"],
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-    model?: Model,
-    existingObservation?: Omit<Observation, "input" | "output">,
-  ) {
-    let newPromptTokens = body.usage?.input;
-    if (newPromptTokens === undefined && model && model.tokenizerId) {
-      if (body.input) {
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: body.input,
-        });
-      } else {
-        logger.debug(
-          `No input provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationInput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            input: true,
-          },
-        });
-
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: observationInput?.input,
-        });
-      }
-    }
-
-    let newCompletionTokens = body.usage?.output;
-
-    if (newCompletionTokens === undefined && model && model.tokenizerId) {
-      if (body.output) {
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: body.output,
-        });
-      } else {
-        logger.debug(
-          `No output provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationOutput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            output: true,
-          },
-        });
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: observationOutput?.output,
-        });
-      }
-    }
-
-    return [newPromptTokens ?? undefined, newCompletionTokens ?? undefined];
-  }
-
-  static calculateTokenCosts(
-    model: Model | null | undefined,
-    userProvidedCosts: {
-      inputCost?: Decimal | null;
-      outputCost?: Decimal | null;
-      totalCost?: Decimal | null;
-    },
-    tokenCounts: { input?: number; output?: number; total?: number },
-  ): {
-    inputCost?: Decimal | null;
-    outputCost?: Decimal | null;
-    totalCost?: Decimal | null;
-  } {
-    // If user has provided any cost point, do not calculate anything else
-    if (
-      userProvidedCosts.inputCost ||
-      userProvidedCosts.outputCost ||
-      userProvidedCosts.totalCost
-    ) {
-      return {
-        ...userProvidedCosts,
-        totalCost:
-          userProvidedCosts.totalCost ??
-          (userProvidedCosts.inputCost ?? new Decimal(0)).add(
-            userProvidedCosts.outputCost ?? new Decimal(0),
-          ),
-      };
-    }
-
-    const finalInputCost =
-      tokenCounts.input !== undefined && model?.inputPrice
-        ? model.inputPrice.mul(tokenCounts.input)
-        : undefined;
-
-    const finalOutputCost =
-      tokenCounts.output !== undefined && model?.outputPrice
-        ? model.outputPrice.mul(tokenCounts.output)
-        : finalInputCost
-          ? new Decimal(0)
-          : undefined;
-
-    const finalTotalCost =
-      tokenCounts.total !== undefined && model?.totalPrice
-        ? model.totalPrice.mul(tokenCounts.total)
-        : (finalInputCost ?? finalOutputCost)
-          ? new Decimal(finalInputCost ?? 0).add(finalOutputCost ?? 0)
-          : undefined;
-
-    return {
-      inputCost: finalInputCost,
-      outputCost: finalOutputCost,
-      totalCost: finalTotalCost,
-    };
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for observation creation");
-  }
-
-  async process(apiScope: LegacyIngestionAccessScope): Promise<Observation> {
-    this.auth(apiScope);
-
-    const existingObservation = this.event.body.id
-      ? await prisma.observation.findFirst({
-          select: {
-            // do not select I/O to spare our db
-            input: false,
-            output: false,
-
-            id: true,
-            traceId: true,
-            projectId: true,
-            type: true,
-            startTime: true,
-            endTime: true,
-            name: true,
-            metadata: true,
-            parentObservationId: true,
-            level: true,
-            statusMessage: true,
-            version: true,
-            createdAt: true,
-            updatedAt: true,
-            model: true,
-            internalModelId: true,
-            modelParameters: true,
-            promptTokens: true,
-            completionTokens: true,
-            totalTokens: true,
-            unit: true,
-            inputCost: true,
-            outputCost: true,
-            totalCost: true,
-            calculatedInputCost: true,
-            calculatedOutputCost: true,
-            calculatedTotalCost: true,
-            completionStartTime: true,
-            promptId: true,
-            internalModel: true,
-          },
-          where: { id: this.event.body.id, projectId: apiScope.projectId },
-        })
-      : null;
-
-    if (
-      existingObservation &&
-      existingObservation.projectId !== apiScope.projectId
-    ) {
-      throw new ForbiddenError(
-        `Access denied for observation creation ${existingObservation.projectId} `,
-      );
-    }
-
-    const obs = await this.convertToObservation(apiScope, existingObservation);
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    return await prisma.observation.upsert({
-      where: {
-        id: obs.id,
-      },
-      create: obs.create,
-      update: obs.update,
-    });
-  }
-}
-
-export class TraceProcessor implements EventProcessor {
-  event: z.infer<typeof traceEvent>;
-
-  constructor(event: z.infer<typeof traceEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for trace creation");
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const internalId =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `trace.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    logger.debug(
-      `Trying to create trace, project ${apiScope.projectId}, id: ${internalId}`,
-    );
-
-    const existingTrace = await prisma.trace.findFirst({
-      where: {
-        id: internalId,
-      },
-    });
-
-    if (existingTrace && existingTrace.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for trace creation ${existingTrace.projectId}`,
-      );
-    }
-
-    const mergedMetadata = mergeJson(
-      existingTrace?.metadata
-        ? jsonSchema.parse(existingTrace.metadata)
-        : undefined,
-      body.metadata ?? undefined,
-    );
-
-    const mergedTags =
-      existingTrace?.tags && body.tags
-        ? Array.from(new Set(existingTrace.tags.concat(body.tags ?? []))).sort()
-        : body.tags
-          ? Array.from(new Set(body.tags)).sort()
-          : undefined;
-
-    if (body.sessionId) {
-      try {
-        await prisma.traceSession.upsert({
-          where: {
-            id_projectId: {
-              id: body.sessionId,
-              projectId: apiScope.projectId,
-            },
-          },
-          create: {
-            id: body.sessionId,
-            projectId: apiScope.projectId,
-          },
-          update: {},
-        });
-      } catch (e) {
-        if (
-          e instanceof Prisma.PrismaClientKnownRequestError &&
-          e.code === "P2002"
-        ) {
-          logger.warn(
-            `Failed to upsert session. Session ${body.sessionId} in project ${apiScope.projectId} already exists`,
-          );
-        } else {
-          throw e;
-        }
-      }
-    }
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    const upsertedTrace = await prisma.trace.upsert({
-      where: {
-        id: internalId,
-      },
-      create: {
-        id: internalId,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        name: body.name ?? undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        projectId: apiScope.projectId,
-        tags: mergedTags ?? undefined,
-      },
-      update: {
-        name: body.name ?? undefined,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        tags: mergedTags ?? undefined,
-      },
-    });
-    return upsertedTrace;
-  }
-}
-
-export class ScoreProcessor implements EventProcessor {
-  event: z.infer<typeof scoreEvent>;
-
-  constructor(event: z.infer<typeof scoreEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
-      throw new ForbiddenError(
-        `Access denied for score creation, ${apiScope.accessLevel}`,
-      );
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const id =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `score.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    const existingScore = await prisma.score.findFirst({
-      where: {
-        id: id,
-      },
-      select: {
-        projectId: true,
-      },
-    });
-    if (existingScore && existingScore.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for score creation ${existingScore.projectId}`,
-      );
-    }
-
-    const validatedScore = await validateAndInflateScore({
-      body,
-      scoreId: id,
-      projectId: apiScope.projectId,
-    });
-
-    return await prisma.score.upsert({
-      where: {
-        id_projectId: {
-          id,
-          projectId: apiScope.projectId,
-        },
-      },
-      create: {
-        ...validatedScore,
-      },
-      update: {
-        ...validatedScore,
-      },
-    });
-  }
-}
-
-export class SdkLogProcessor implements EventProcessor {
-  event: z.infer<typeof sdkLogEvent>;
-
-  constructor(event: z.infer<typeof sdkLogEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    return;
-  }
-
-  process() {
-    try {
-      logger.info("SDK Log", this.event);
-      return undefined;
-    } catch (error) {
-      return undefined;
-    }
-  }
-}

packages/shared/src/server/ingestion/legacy/index.ts

@@ -1,161 +0,0 @@
-import { env } from "node:process";
-import z from "zod";
-import { ForbiddenError, UnauthorizedError } from "../../../errors";
-import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
-import { getProcessorForEvent } from "./EventProcessor";
-import { ApiAccessScope } from "../../auth/types";
-import { backOff } from "exponential-backoff";
-import { Model } from "../../..";
-import { logger } from "../../logger";
-
-export type BatchResult = {
-  result: unknown;
-  id: string;
-  type: string;
-};
-
-type TokenCountInput = {
-  model: Model;
-  text: unknown;
-};
-
-export type LegacyIngestionAccessScope = Omit<
-  ApiAccessScope,
-  "orgId" | "plan" | "rateLimitOverrides"
->;
-
-type LegacyIngestionAuthHeaderVerificationResult =
-  | {
-      validKey: true;
-      scope: LegacyIngestionAccessScope;
-    }
-  | {
-      validKey: false;
-      error: string;
-    };
-
-export const handleBatch = async (
-  events: z.infer<typeof ingestionApiSchema>["batch"],
-  authCheck: LegacyIngestionAuthHeaderVerificationResult,
-  calculateTokenDelegate: (p: TokenCountInput) => number | undefined,
-) => {
-  logger.debug(`handling ingestion ${events.length} events`);
-
-  if (!authCheck.validKey) throw new UnauthorizedError(authCheck.error);
-
-  const results: BatchResult[] = []; // Array to store the results
-
-  const errors: {
-    error: unknown;
-    id: string;
-    type: string;
-  }[] = []; // Array to store the errors
-
-  for (const singleEvent of events) {
-    try {
-      const result = await retry(async () => {
-        return await handleSingleEvent(
-          singleEvent,
-          authCheck.scope,
-          calculateTokenDelegate,
-        );
-      });
-      results.push({
-        result: result,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      }); // Push each result into the array
-    } catch (error) {
-      // Handle or log the error if `handleSingleEvent` fails
-      logger.error("Error handling event:", error);
-      // Decide how to handle the error: rethrow, continue, or push an error object to results
-      // For example, push an error object:
-      errors.push({
-        error,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      });
-    }
-  }
-
-  return { results, errors };
-};
-
-async function retry<T>(request: () => Promise<T>): Promise<T> {
-  return await backOff(request, {
-    numOfAttempts: env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" ? 5 : 3,
-    retry: (e: Error, attemptNumber: number) => {
-      if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
-        logger.info("not retrying auth error");
-        return false;
-      }
-      logger.info(`retrying processing events ${attemptNumber}`);
-      return true;
-    },
-  });
-}
-
-const handleSingleEvent = async (
-  event: IngestionEventType,
-  apiScope: LegacyIngestionAccessScope,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-) => {
-  const { body } = event;
-  let restEvent = body;
-  if ("input" in body) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { input, ...rest } = body;
-    restEvent = rest;
-  }
-  if ("output" in restEvent) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { output, ...rest } = restEvent;
-    restEvent = rest;
-  }
-
-  logger.debug(
-    `handling single event ${event.id} of type ${event.type}:  ${JSON.stringify({ body: restEvent })}`,
-  );
-
-  const cleanedEvent = cleanEvent(event) as IngestionEventType;
-
-  // Deny access to non-score events if the access level is not "all"
-  // This is an additional safeguard to auth checks in EventProcessor
-  if (
-    apiScope.accessLevel !== "all" &&
-    cleanedEvent.type !== eventTypes.SCORE_CREATE
-  ) {
-    throw new ForbiddenError("Access denied. Event type not allowed.");
-  }
-
-  return getProcessorForEvent(cleanedEvent, calculateTokenDelegate).process(
-    apiScope,
-  );
-};
-
-// cleans NULL characters from the event
-export function cleanEvent(obj: unknown): unknown {
-  if (typeof obj === "string") {
-    return obj.replace(/\u0000/g, "");
-  } else if (typeof obj === "object" && obj !== null) {
-    if (Array.isArray(obj)) {
-      return obj.map(cleanEvent);
-    } else {
-      // Here we assert that obj is a Record<string, unknown>
-      const objAsRecord = obj as Record<string, unknown>;
-      const newObj: Record<string, unknown> = {};
-      for (const key in objAsRecord) {
-        newObj[key] = cleanEvent(objAsRecord[key]);
-      }
-      return newObj;
-    }
-  } else {
-    return obj;
-  }
-}
-
-export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
-  val === undefined || val === null;

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -18,16 +18,13 @@ import {
   traceException,
 } from "../instrumentation";
 import { logger } from "../logger";
-import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
-import { LegacyIngestionQueue } from "../redis/legacyIngestion";
 import { redis } from "../redis/redis";
-import { handleBatch } from "./legacy";
 import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
-import { getProcessorForEvent } from "./legacy/EventProcessor";
 import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
 
 export type TokenCountDelegate = (p: {
@@ -54,7 +51,6 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
 export const processEventBatch = async (
   input: unknown[],
   authCheck: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
 ): Promise<{
   successes: { id: string; status: number }[];
   errors: {
@@ -99,7 +95,7 @@ export const processEventBatch = async (
         });
         return [];
       }
-      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+      if (!isAuthorized(parsed.data, authCheck)) {
         authenticationErrors.push({
           id: parsed.data.id,
           error: new UnauthorizedError("Access Scope Denied"),
@@ -156,157 +152,95 @@ export const processEventBatch = async (
    * ASYNC PROCESSING *
    ********************/
   let s3UploadErrored = false;
-  if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-    await instrumentAsync({ name: "s3-upload-events" }, async () => {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      // S3 Event Upload is currently blocking, but non-failing.
-      // If a promise rejects, we log it below, but do not throw an error.
-      // In this case, we upload the full batch into the Redis queue.
-      const results = await Promise.allSettled(
-        Object.keys(sortedBatchByEventBodyId).map(async (id) => {
-          // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
-          // That way we batch updates from the same invocation into a single file and reduce
-          // write operations on S3.
-          const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-          return s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
-            data,
-          );
-        }),
-      );
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          s3UploadErrored = true;
-          logger.error("Failed to upload event to S3", {
-            error: result.reason,
-          });
-        }
-      });
-    });
-  }
-
-  // Send each event individually to IngestionQueue for new processing
-  if (
-    env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" &&
-    env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" &&
-    env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING === "true" &&
-    redis &&
-    !s3UploadErrored
-  ) {
-    const queue = IngestionQueue.getInstance();
+  await instrumentAsync({ name: "s3-upload-events" }, async () => {
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    // S3 Event Upload is blocking, but non-failing.
+    // If a promise rejects, we log it below, but do not throw an error.
+    // In this case, we upload the full batch into the Redis queue.
     const results = await Promise.allSettled(
-      Object.keys(sortedBatchByEventBodyId).map(async (id) =>
-        queue
-          ? queue.add(
-              QueueJobs.IngestionJob,
-              {
-                id: randomUUID(),
-                timestamp: new Date(),
-                name: QueueJobs.IngestionJob as const,
-                payload: {
-                  data: {
-                    type: sortedBatchByEventBodyId[id].type,
-                    eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
-                  },
-                  authCheck,
-                },
-              },
-              {
-                delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
-              },
-            )
-          : Promise.reject("Failed to instantiate queue"),
-      ),
+      Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+        // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+        // That way we batch updates from the same invocation into a single file and reduce
+        // write operations on S3.
+        const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+        return s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+          data,
+        );
+      }),
     );
     results.forEach((result) => {
       if (result.status === "rejected") {
-        logger.error("Failed to add event to IngestionQueue", {
+        s3UploadErrored = true;
+        logger.error("Failed to upload event to S3", {
           error: result.reason,
         });
       }
     });
+  });
+
+  // Send each event individually to IngestionQueue for ClickHouse processing
+  if (s3UploadErrored) {
+    throw new Error(
+      "Failed to upload events to blob storage, aborting event processing",
+    );
   }
 
-  // As part of the legacy processing we sent the entire batch to the worker.
-  if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
-    const queue = LegacyIngestionQueue.getInstance();
-
-    if (queue) {
-      let addToQueueFailed = false;
-
-      const queuePayload: LegacyIngestionEventType =
-        env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" && !s3UploadErrored
-          ? {
-              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
-                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-                return {
-                  type,
-                  eventBodyId,
-                  eventId: key,
-                };
-              }),
-              authCheck,
-              useS3EventStore: true,
-            }
-          : { data: sortedBatch, authCheck, useS3EventStore: false };
-
-      try {
-        await queue.add(QueueJobs.LegacyIngestionJob, {
-          payload: queuePayload,
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.LegacyIngestionJob as const,
-        });
-      } catch (e: unknown) {
-        logger.warn(
-          "Failed to add batch to queue, falling back to sync processing",
-          e,
-        );
-        addToQueueFailed = true;
-      }
-
-      if (!addToQueueFailed) {
-        return aggregateBatchResult(
-          // we are not sending additional server errors to the client in case of early return
-          [...validationErrors, ...authenticationErrors],
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
-        );
-      }
-    } else {
-      logger.error(
-        "Ingestion queue not initialized, falling back to sync processing",
-      );
-    }
+  if (!redis) {
+    throw new Error("Redis not initialized, aborting event processing");
   }
 
-  /*******************
-   * SYNC PROCESSING *
-   *******************/
-  const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
+  const queue = IngestionQueue.getInstance();
+  await Promise.all(
+    Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+      queue
+        ? queue.add(
+            QueueJobs.IngestionJob,
+            {
+              id: randomUUID(),
+              timestamp: new Date(),
+              name: QueueJobs.IngestionJob as const,
+              payload: {
+                data: {
+                  type: sortedBatchByEventBodyId[id].type,
+                  eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                },
+                authCheck,
+              },
+            },
+            {
+              delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+            },
+          )
+        : Promise.reject("Failed to instantiate queue"),
+    ),
+  );
 
-  //  in case we did not return early, we return the result here
   return aggregateBatchResult(
-    [...validationErrors, ...authenticationErrors, ...result.errors],
-    result.results,
+    [...validationErrors, ...authenticationErrors],
+    sortedBatch.map((event) => ({ id: event.id, result: event })),
+    authCheck.scope.projectId,
   );
 };
 
 const isAuthorized = (
   event: IngestionEventType,
   authScope: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
 ): boolean => {
-  try {
-    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+  if (event.type === eventTypes.SDK_LOG) {
     return true;
-  } catch (error) {
-    return false;
   }
+
+  if (event.type === eventTypes.SCORE_CREATE) {
+    return (
+      authScope.scope.accessLevel === "scores" ||
+      authScope.scope.accessLevel === "all"
+    );
+  }
+
+  return authScope.scope.accessLevel === "all";
 };
 
 /**
@@ -336,6 +270,7 @@ const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
 export const aggregateBatchResult = (
   errors: Array<{ id: string; error: unknown }>,
   results: Array<{ id: string; result: unknown }>,
+  projectId?: string,
 ) => {
   const returnedErrors: {
     id: string;
@@ -382,7 +317,10 @@ export const aggregateBatchResult = (
 
   if (returnedErrors.length > 0) {
     traceException(errors);
-    logger.error("Error processing events", returnedErrors);
+    logger.error("Error processing events", {
+      errors: returnedErrors,
+      "langfuse.project.id": projectId,
+    });
   }
 
   results.forEach((result) => {

packages/shared/src/server/ingestion/types.ts

@@ -56,13 +56,76 @@ export const usage = MixedUsage.nullish()
   // ensure output is always of new usage model
   .pipe(Usage.nullish());
 
+const RawUsageOrCostDetails = z.record(
+  z.string(),
+  z.number().nonnegative().nullish(),
+);
+
+const OpenAIUsageSchema = z
+  .object({
+    prompt_tokens: z.number().nonnegative(),
+    completion_tokens: z.number().nonnegative(),
+    total_tokens: z.number().nonnegative(),
+    prompt_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
+    completion_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
+  })
+  .strict()
+  .transform((v) => {
+    if (!v) return;
+
+    const {
+      prompt_tokens,
+      completion_tokens,
+      total_tokens,
+      prompt_tokens_details,
+      completion_tokens_details,
+    } = v;
+    const result: z.infer<typeof RawUsageOrCostDetails> & {
+      input: number;
+      output: number;
+      total: number;
+    } = {
+      input: prompt_tokens,
+      output: completion_tokens,
+      total: total_tokens,
+    };
+
+    if (prompt_tokens_details) {
+      for (const [key, value] of Object.entries(prompt_tokens_details)) {
+        result[`input_${key}`] = value;
+        result.input = Math.max(result.input - (value ?? 0), 0);
+      }
+    }
+
+    if (completion_tokens_details) {
+      for (const [key, value] of Object.entries(completion_tokens_details)) {
+        result[`output_${key}`] = value;
+        result.output = Math.max(result.output - (value ?? 0), 0);
+      }
+    }
+
+    return result;
+  })
+  .pipe(RawUsageOrCostDetails);
+
+export const UsageOrCostDetails = z
+  .union([OpenAIUsageSchema, RawUsageOrCostDetails])
+  .nullish();
+
+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 export const TraceBody = z.object({
   id: z.string().nullish(),
   timestamp: stringDateTime,
-  name: z.string().nullish(),
+  name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
   sessionId: z.string().nullish(),
   userId: z.string().nullish(),
   metadata: jsonSchema.nullish(),
@@ -77,8 +140,8 @@ export const OptionalObservationBody = z.object({
   name: z.string().nullish(),
   startTime: stringDateTime,
   metadata: jsonSchema.nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
   level: z.nativeEnum(ObservationLevel).nullish(),
   statusMessage: z.string().nullish(),
   parentObservationId: z.string().nullish(),
@@ -119,6 +182,8 @@ export const CreateGenerationBody = CreateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -147,6 +212,8 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -298,6 +365,8 @@ export const LegacyObservationBody = z.object({
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   metadata: jsonSchema.nullish(),
   parentObservationId: z.string().nullish(),
   level: z.nativeEnum(ObservationLevel).nullish(),

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -17,7 +17,7 @@ type ValidateAndInflateScoreParams = {
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
 ): Promise<Score> {
-  const { body, projectId } = params;
+  const { body, projectId, scoreId } = params;
 
   if (body.configId) {
     const config = await prisma.scoreConfig.findFirst({
@@ -32,10 +32,22 @@ export async function validateAndInflateScore(
         "The configId you provided does not match a valid config in this project",
       );
 
-    validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+    // Override some fields in the score body with config fields
+    // We ignore the set fields in the body
+    const bodyWithConfigOverrides = {
+      ...body,
+      name: config.name,
+    };
+
+    validateConfigAgainstBody(
+      bodyWithConfigOverrides,
+      config as ValidatedScoreConfig,
+    );
 
     return inflateScoreBody({
-      ...params,
+      projectId,
+      scoreId,
+      body: bodyWithConfigOverrides,
       config: config as ValidatedScoreConfig,
     });
   }

packages/shared/src/server/instrumentation/index.ts

@@ -226,6 +226,14 @@ export const recordHistogram = (
   dd.dogstatsd.histogram(stat, value, tags);
 };
 
+export const recordDistribution = (
+  stat: string,
+  value?: number | undefined,
+  tags?: { [tag: string]: string | number } | undefined,
+) => {
+  dd.dogstatsd.distribution(stat, value, tags);
+};
+
 /**
  * Converts a queue name to the matching datadog metric name.
  * Consumer only needs to append the relevant suffix.

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,11 +1,11 @@
 import type { ZodSchema } from "zod";
 
-import { CallbackHandler } from "langfuse-langchain";
-
 import { ChatAnthropic } from "@langchain/anthropic";
+import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
   AIMessage,
+  BaseMessage,
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
@@ -15,31 +15,24 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
-import {
+import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
-import {
-  processEventBatch,
-  type TokenCountDelegate,
-} from "../ingestion/processEventBatch";
+import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
-import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
-
+import {
+  ChatMessage,
+  ChatMessageRole,
+  LLMAdapter,
+  ModelParams,
+  TraceParams,
+} from "./types";
+import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
 type ProcessTracedEvents = () => Promise<void>;
 
-export type TraceParams = {
-  traceName: string;
-  traceId: string;
-  projectId: string;
-  tags: string[];
-  tokenCountDelegate: TokenCountDelegate;
-  authCheck: AuthHeaderValidVerificationResult;
-};
-
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
@@ -47,9 +40,11 @@ type LLMCompletionParams = {
   callbacks?: BaseCallbackHandler[];
   baseURL?: string;
   apiKey: string;
+  extraHeaders?: Record<string, string>;
   maxRetries?: number;
   config?: Record<string, string> | null;
   traceParams?: TraceParams;
+  throwOnError?: boolean; // default is true
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -98,6 +93,8 @@ export async function fetchLLMCompletion(
     maxRetries,
     config,
     traceParams,
+    extraHeaders,
+    throwOnError = true,
   } = params;
 
   let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
@@ -109,7 +106,6 @@ export async function fetchLLMCompletion(
       _isLocalEventExportEnabled: true,
       tags: traceParams.tags,
     });
-
     finalCallbacks.push(handler);
 
     processTracedEvents = async () => {
@@ -120,7 +116,6 @@ export async function fetchLLMCompletion(
         await processEventBatch(
           JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
           traceParams.authCheck,
-          traceParams.tokenCountDelegate,
         );
       } catch (e) {
         logger.error("Failed to process traced events", { error: e });
@@ -130,16 +125,28 @@ export async function fetchLLMCompletion(
 
   finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
 
-  const finalMessages = messages.map((message) => {
-    if (message.role === ChatMessageRole.User)
-      return new HumanMessage(message.content);
-    if (message.role === ChatMessageRole.System)
-      return new SystemMessage(message.content);
+  let finalMessages: BaseMessage[];
+  // VertexAI requires at least 1 user message
+  if (modelParams.adapter === LLMAdapter.VertexAI && messages.length === 1) {
+    finalMessages = [new HumanMessage(messages[0].content)];
+  } else {
+    finalMessages = messages.map((message) => {
+      if (message.role === ChatMessageRole.User)
+        return new HumanMessage(message.content);
+      if (message.role === ChatMessageRole.System)
+        return new SystemMessage(message.content);
 
-    return new AIMessage(message.content);
-  });
+      return new AIMessage(message.content);
+    });
+  }
 
-  let chatModel: ChatOpenAI | ChatAnthropic | ChatBedrockConverse;
+  finalMessages = finalMessages.filter((m) => m.content.length > 0);
+
+  let chatModel:
+    | ChatOpenAI
+    | ChatAnthropic
+    | ChatBedrockConverse
+    | ChatVertexAI;
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
@@ -149,7 +156,7 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
-      clientOptions: { maxRetries },
+      clientOptions: { maxRetries, timeout: 1000 * 60 * 2 }, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
@@ -163,7 +170,9 @@ export async function fetchLLMCompletion(
       maxRetries,
       configuration: {
         baseURL,
+        defaultHeaders: extraHeaders,
       },
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Azure) {
     chatModel = new ChatOpenAI({
@@ -176,6 +185,7 @@ export async function fetchLLMCompletion(
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
@@ -190,6 +200,24 @@ export async function fetchLLMCompletion(
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
+    });
+  } else if (modelParams.adapter === LLMAdapter.VertexAI) {
+    const credentials = GCPServiceAccountKeySchema.parse(JSON.parse(apiKey));
+
+    // Requests time out after 60 seconds for both public and private endpoints by default
+    // Reference: https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions#send-request
+    chatModel = new ChatVertexAI({
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxOutputTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks: finalCallbacks,
+      maxRetries,
+      authOptions: {
+        projectId: credentials.project_id,
+        credentials,
+      },
     });
   } else {
     // eslint-disable-next-line no-unused-vars
@@ -203,16 +231,17 @@ export async function fetchLLMCompletion(
     runName: traceParams?.traceName,
   };
 
-  if (params.structuredOutputSchema) {
-    return {
-      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-        .withStructuredOutput(params.structuredOutputSchema)
-        .invoke(finalMessages, runConfig),
-      processTracedEvents,
-    };
-  }
+  try {
+    if (params.structuredOutputSchema) {
+      return {
+        completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+          .withStructuredOutput(params.structuredOutputSchema)
+          .invoke(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }
 
-  /*
+    /*
   Workaround OpenAI o1 while in beta:
   
   This is a temporary workaround to avoid sending system messages to OpenAI's O1 models.
@@ -224,42 +253,49 @@ export async function fetchLLMCompletion(
 
   Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
   */
-  if (modelParams.model.startsWith("o1-")) {
-    return {
-      completion: await new ChatOpenAI({
-        openAIApiKey: apiKey,
-        modelName: modelParams.model,
-        temperature: 1,
-        maxTokens: undefined,
-        topP: undefined,
-        callbacks,
-        maxRetries,
-        configuration: {
-          baseURL,
-        },
-      })
-        .pipe(new StringOutputParser())
-        .invoke(
-          finalMessages.filter((message) => message._getType() !== "system"),
-          runConfig,
-        ),
-      processTracedEvents,
-    };
-  }
+    if (modelParams.model.startsWith("o1-")) {
+      return {
+        completion: await new ChatOpenAI({
+          openAIApiKey: apiKey,
+          modelName: modelParams.model,
+          temperature: 1,
+          maxTokens: undefined,
+          topP: undefined,
+          callbacks,
+          maxRetries,
+          configuration: {
+            baseURL,
+          },
+          timeout: 1000 * 60 * 2, // 2 minutes timeout
+        })
+          .pipe(new StringOutputParser())
+          .invoke(
+            finalMessages.filter((message) => message._getType() !== "system"),
+            runConfig,
+          ),
+        processTracedEvents,
+      };
+    }
+
+    if (streaming) {
+      return {
+        completion: await chatModel
+          .pipe(new BytesOutputParser())
+          .stream(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }
 
-  if (streaming) {
     return {
       completion: await chatModel
-        .pipe(new BytesOutputParser())
-        .stream(finalMessages, runConfig),
+        .pipe(new StringOutputParser())
+        .invoke(finalMessages, runConfig),
       processTracedEvents,
     };
+  } catch (error) {
+    if (throwOnError) {
+      throw error;
+    }
+    return { completion: null, processTracedEvents };
   }
-
-  return {
-    completion: await chatModel
-      .pipe(new StringOutputParser())
-      .invoke(finalMessages, runConfig),
-    processTracedEvents,
-  };
 }

packages/shared/src/server/llm/types.ts

@@ -1,6 +1,8 @@
 import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
+import { TokenCountDelegate } from "../ingestion/processEventBatch";
+import { AuthHeaderValidVerificationResult } from "../auth/types";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
 
@@ -16,6 +18,7 @@ export enum LLMAdapter {
   OpenAI = "openai",
   Azure = "azure",
   Bedrock = "bedrock",
+  VertexAI = "google-vertex-ai",
 }
 
 export enum ChatMessageRole {
@@ -70,6 +73,7 @@ export const ExperimentMetadataSchema = z
     provider: z.string(),
     model: z.string(),
     model_params: ZodModelConfig,
+    error: z.string().optional(),
   })
   .strict();
 export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
@@ -114,10 +118,19 @@ export const anthropicModels = [
   "claude-instant-1.2",
 ] as const;
 
+export const vertexAIModels = [
+  "gemini-2.0-flash-exp",
+  "gemini-1.5-pro",
+  "gemini-1.5-flash",
+  "gemini-1.0-pro",
+] as const;
+
 export type AnthropicModel = (typeof anthropicModels)[number];
+export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
   [LLMAdapter.Anthropic]: anthropicModels,
   [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.VertexAI]: vertexAIModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
 } as const;
@@ -138,6 +151,8 @@ export const LLMApiKeySchema = z
     provider: z.string(),
     displaySecretKey: z.string(),
     secretKey: z.string(),
+    extraHeaders: z.string().nullish(),
+    extraHeaderKeys: z.array(z.string()),
     baseURL: z.string().nullable(),
     customModels: z.array(z.string()),
     withDefaultModels: z.boolean(),
@@ -151,3 +166,12 @@ export type LLMApiKey =
   z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
     ? z.infer<typeof LLMApiKeySchema>
     : never;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};

packages/shared/src/server/llm/utils.ts

@@ -0,0 +1,13 @@
+import { z } from "zod";
+
+import { decrypt } from "../../encryption";
+
+const ExtraHeaderSchema = z.record(z.string(), z.string());
+
+export function decryptAndParseExtraHeaders(
+  extraHeaders: string | null | undefined,
+) {
+  if (!extraHeaders) return;
+
+  return ExtraHeaderSchema.parse(JSON.parse(decrypt(extraHeaders)));
+}

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -161,8 +161,8 @@ export class StringOptionsFilter implements Filter {
     return {
       query:
         this.operator === "any of"
-          ? `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`
-          : `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`,
+          ? `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} IN ({${varName}: Array(String)})`
+          : `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} NOT IN ({${varName}: Array(String)})`,
       params: { [varName]: this.values },
     };
   }
@@ -383,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  some(predicate: (filter: Filter) => boolean) {
+    return this.filters.some(predicate);
+  }
+
   length() {
     return this.filters.length;
   }

packages/shared/src/server/queries/clickhouse-sql/orderby-factory.ts

@@ -3,31 +3,53 @@ import { OrderByState } from "../../../interfaces/orderBy";
 import { UiColumnMapping } from "../../../tableDefinitions";
 import { logger } from "../../logger";
 
+type OrderByStateNotNull = Exclude<OrderByState, null>;
+
 export function orderByToClickhouseSql(
-  orderBy: OrderByState,
+  orderBy: OrderByState | OrderByState[] = [],
   tableColumns: UiColumnMapping[],
 ): string {
-  if (!orderBy) {
+  if (
+    !orderBy ||
+    (Array.isArray(orderBy) && orderBy.filter(Boolean).length === 0)
+  ) {
     return "";
   }
-  // Get column definition to map column to internal name, e.g. "t.id"
-  const col = tableColumns.find(
-    (c) => c.uiTableName === orderBy.column || c.uiTableId === orderBy.column,
-  );
 
-  if (!col) {
-    logger.warn("Invalid order by column", orderBy.column);
-    throw new Error("Invalid order by column: " + orderBy.column);
+  if (!Array.isArray(orderBy)) {
+    orderBy = [orderBy];
+  }
+  // Initialize an array to hold order by clauses
+  const orderByClauses: string[] = [];
+
+  // Loop through each orderBy entry
+  for (const ob of orderBy.filter((o): o is OrderByStateNotNull =>
+    Boolean(o),
+  )) {
+    // Get column definition to map column to internal name, e.g. "t.id"
+    const col = tableColumns.find(
+      (c) => c.uiTableName === ob.column || c.uiTableId === ob.column,
+    );
+
+    if (!col) {
+      logger.warn("Invalid order by column", ob.column);
+      throw new Error("Invalid order by column: " + ob.column);
+    }
+
+    // Assert that ob.order is either "asc" or "desc"
+    const orderByOrder = z.enum(["ASC", "DESC"]);
+    const order = orderByOrder.safeParse(ob.order);
+    if (!order.success) {
+      logger.warn("Invalid order", ob.order);
+      throw new Error("Invalid order: " + ob.order);
+    }
+
+    // Append the order by clause to the array
+    orderByClauses.push(
+      `${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`,
+    );
   }
 
-  // Assert that orderBy.order is either "asc" or "desc"
-  const orderByOrder = z.enum(["ASC", "DESC"]);
-  const order = orderByOrder.safeParse(orderBy.order);
-  if (!order.success) {
-    logger.warn("Invalid order", orderBy.order);
-    throw new Error("Invalid order: " + orderBy.order);
-  }
-
-  // Both column and order are safe, can use raw SQL
-  return `ORDER BY ${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`;
+  // Join all order by clauses with a comma and return
+  return `ORDER BY ${orderByClauses.join(", ")}`;
 }

packages/shared/src/server/queries/createGenerationsQuery.ts

@@ -1,15 +1,10 @@
-import { ObservationView, Prisma } from "@prisma/client";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { observationsTableCols } from "../../observationsTable";
-import { TableFilters } from "./types";
+import { ObservationView } from "@prisma/client";
 
 type AdditionalObservationFields = {
   traceName: string | null;
   traceTags: Array<string>;
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
 };
 
 export type FullObservation = AdditionalObservationFields & ObservationView;
@@ -24,147 +19,3 @@ export type IOAndMetadataOmittedObservations = Array<
   Omit<ObservationView, "input" | "output" | "metadata"> &
     AdditionalObservationFields
 >;
-
-export function parseGetAllGenerationsInput(filters: TableFilters) {
-  const searchCondition = filters.searchQuery
-    ? Prisma.sql`AND (
-        o."id" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."name" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."model" ILIKE ${`%${filters.searchQuery}%`} OR
-        t."name" ILIKE ${`%${filters.searchQuery}%`}
-      )`
-    : Prisma.empty;
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filters.filter ?? [],
-    observationsTableCols,
-    "observations",
-  );
-
-  const orderByCondition = orderByToPrismaSql(
-    filters.orderBy,
-    observationsTableCols,
-  );
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const startTimeFilter = filters.filter?.find(
-    (f) => f.column === "Start Time" && f.type === "datetime",
-  );
-
-  const datetimeFilter =
-    startTimeFilter && startTimeFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          startTimeFilter.operator,
-          startTimeFilter.value,
-        )
-      : Prisma.empty;
-
-  return {
-    searchCondition,
-    filterCondition,
-    orderByCondition,
-    datetimeFilter,
-  };
-}
-
-export function createGenerationsQuery({
-  projectId,
-  datetimeFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectIOAndMetadata = false,
-  selectScoreValues = false,
-}: {
-  projectId: string;
-  datetimeFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectIOAndMetadata?: boolean;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  WITH scores_avg AS (
-        SELECT
-          trace_id,
-          observation_id,
-          ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-          jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-        FROM (
-          SELECT
-            trace_id,
-            observation_id,
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            avg(value) avg_value,
-            comment
-          FROM
-            scores
-          WHERE
-            project_id = ${projectId}
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-          GROUP BY
-            trace_id,
-            observation_id,
-            name,
-            comment
-          ORDER BY
-            trace_id
-          ) tmp
-        GROUP BY
-          trace_id, 
-          observation_id
-      )
-      SELECT
-        ${selectScoreValues ? Prisma.sql`s_avg."scores_values" AS "scores",` : Prisma.empty}
-        o.id,
-        o.name,
-        o.model,
-        o."modelParameters",
-        o.start_time as "startTime",
-        o.end_time as "endTime",
-        ${selectIOAndMetadata ? Prisma.sql`o.input, o.output, o.metadata,` : Prisma.empty} 
-        o.trace_id as "traceId",
-        t.name as "traceName",
-        o.completion_start_time as "completionStartTime",
-        o.time_to_first_token as "timeToFirstToken",
-        o.prompt_tokens as "promptTokens",
-        o.completion_tokens as "completionTokens",
-        o.total_tokens as "totalTokens",
-        o.unit,
-        o.level,
-        o.status_message as "statusMessage",
-        o.version,
-        o.model_id as "modelId",
-        o.input_price as "inputPrice",
-        o.output_price as "outputPrice",
-        o.total_price as "totalPrice",
-        o.calculated_input_cost as "calculatedInputCost",
-        o.calculated_output_cost as "calculatedOutputCost",
-        o.calculated_total_cost as "calculatedTotalCost",
-        o."latency",
-        o.prompt_id as "promptId",
-        p.name as "promptName",
-        p.version as "promptVersion",
-        t.tags as "traceTags"
-      FROM observations_view o
-      JOIN traces t ON t.id = o.trace_id AND t.project_id = ${projectId}
-      LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
-      LEFT JOIN prompts p ON p.id = o.prompt_id AND p.project_id = ${projectId}
-      WHERE
-        o.project_id = ${projectId}
-        AND o.type = 'GENERATION'
-        ${datetimeFilter}
-        ${searchCondition}
-        ${filterCondition}
-        ${orderByCondition}
-      ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-      ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-    `;
-}

packages/shared/src/server/queries/createTracesQuery.ts

@@ -1,135 +0,0 @@
-import { Prisma } from "@prisma/client";
-import { TableFilters } from "./types";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { tracesTableCols } from "../../tableDefinitions/tracesTable";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-
-export function parseTraceAllFilters(input: TableFilters) {
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    input.filter ?? [],
-    tracesTableCols,
-    "traces",
-  );
-  const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime",
-  );
-
-  const observationTimeseriesFilter =
-    timeseriesFilter && timeseriesFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          timeseriesFilter.operator,
-          timeseriesFilter.value,
-        )
-      : Prisma.empty;
-
-  const searchCondition = input.searchQuery
-    ? Prisma.sql`AND (
-    t."id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."external_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."user_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."name" ILIKE ${`%${input.searchQuery}%`}
-  )`
-    : Prisma.empty;
-
-  return {
-    filterCondition,
-    orderByCondition,
-    observationTimeseriesFilter,
-    searchCondition,
-  };
-}
-
-export function createTracesQuery({
-  select,
-  projectId,
-  observationTimeseriesFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectScoreValues = false,
-}: {
-  select: Prisma.Sql;
-  projectId: string;
-  observationTimeseriesFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  SELECT
-      ${select}
-  FROM
-    "traces" AS t
-  LEFT JOIN LATERAL (
-    SELECT
-      SUM(prompt_tokens) AS "promptTokens",
-      SUM(completion_tokens) AS "completionTokens",
-      SUM(total_tokens) AS "totalTokens",
-      SUM(calculated_total_cost) AS "calculatedTotalCost",
-      SUM(calculated_input_cost) AS "calculatedInputCost",
-      SUM(calculated_output_cost) AS "calculatedOutputCost"
-    FROM
-      "observations_view"
-    WHERE
-      trace_id = t.id
-      AND "type" = 'GENERATION'
-      AND "project_id" = ${projectId}
-      ${observationTimeseriesFilter}
-  ) AS generation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      COUNT(*) AS "observationCount",
-      EXTRACT(EPOCH FROM COALESCE(MAX("end_time"), MAX("start_time"))) - EXTRACT(EPOCH FROM MIN("start_time"))::double precision AS "latency",
-      COALESCE(  
-        MAX(CASE WHEN level = 'ERROR' THEN 'ERROR' END),  
-        MAX(CASE WHEN level = 'WARNING' THEN 'WARNING' END),  
-        MAX(CASE WHEN level = 'DEFAULT' THEN 'DEFAULT' END),  
-        'DEBUG'  
-      ) AS "level"
-    FROM
-        "observations"
-    WHERE
-        trace_id = t.id
-        AND "project_id" = ${projectId}
-         ${observationTimeseriesFilter}
-  ) AS observation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-      jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-    FROM (
-        SELECT
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            AVG(value) avg_value
-        FROM
-            scores
-        WHERE
-            trace_id = t.id
-            AND t."project_id" = ${projectId}     
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-        GROUP BY
-            name
-    ) tmp
-  ) AS s_avg ON true
-  WHERE 
-    t."project_id" = ${projectId}
-    ${searchCondition}
-    ${filterCondition}
-  ${orderByCondition}
-  ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-`;
-}

packages/shared/src/server/queries/index.ts

@@ -1,8 +1,5 @@
 export { createSessionsAllQuery } from "./createSessionsAllQuery";
-export { createTracesQuery, parseTraceAllFilters } from "./createTracesQuery";
 export {
-  createGenerationsQuery,
-  parseGetAllGenerationsInput,
   type FullObservations,
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
@@ -20,3 +17,4 @@ export {
   NullFilter,
   type ClickhouseOperator,
 } from "./clickhouse-sql/clickhouse-filter";
+export { orderByToClickhouseSql } from "./clickhouse-sql/orderby-factory";

packages/shared/src/server/queues.ts

@@ -1,49 +1,5 @@
 import { z } from "zod";
-import { eventTypes, ingestionBatchEvent } from ".";
-
-export enum EventName {
-  TraceUpsert = "TraceUpsert",
-  BatchExport = "BatchExport",
-  EvaluationExecution = "EvaluationExecution",
-  LegacyIngestion = "LegacyIngestion",
-  CloudUsageMetering = "CloudUsageMetering",
-  ExperimentCreate = "ExperimentCreate",
-}
-
-export const LegacyIngestionEventFull = z.object({
-  useS3EventStore: z.literal(false),
-  data: ingestionBatchEvent,
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEventMeta = z.object({
-  useS3EventStore: z.literal(true),
-  data: z.array(
-    z.object({
-      type: z.nativeEnum(eventTypes),
-      eventBodyId: z.string(),
-      eventId: z.string(),
-    }),
-  ),
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEvent = z.discriminatedUnion("useS3EventStore", [
-  LegacyIngestionEventFull,
-  LegacyIngestionEventMeta,
-]);
+import { eventTypes } from ".";
 
 export const IngestionEvent = z.object({
   data: z.object({
@@ -63,10 +19,18 @@ export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
 });
-export const TraceUpsertEventSchema = z.object({
+export const TraceQueueEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
+export const TracesQueueEventSchema = z.object({
+  projectId: z.string(),
+  traceIds: z.array(z.string()),
+});
+export const ProjectQueueEventSchema = z.object({
+  projectId: z.string(),
+  orgId: z.string(),
+});
 export const DatasetRunItemUpsertEventSchema = z.object({
   projectId: z.string(),
   datasetItemId: z.string(),
@@ -78,76 +42,96 @@ export const EvalExecutionEvent = z.object({
   jobExecutionId: z.string(),
   delay: z.number().nullish(),
 });
-
+export const PostHogIntegrationProcessingEventSchema = z.object({
+  projectId: z.string(),
+});
 export const ExperimentCreateEventSchema = z.object({
   projectId: z.string(),
   datasetId: z.string(),
   runId: z.string(),
   description: z.string().optional(),
 });
+export const DataRetentionProcessingEventSchema = z.object({
+  projectId: z.string(),
+  retention: z.number(),
+});
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
-export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
+export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
+export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
 >;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
-export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
 export type ExperimentCreateEventType = z.infer<
   typeof ExperimentCreateEventSchema
 >;
-
-export const EventBodySchema = z.union([
-  z.object({
-    name: z.literal(EventName.TraceUpsert),
-    payload: z.array(TraceUpsertEventSchema),
-  }),
-  z.object({
-    name: z.literal(EventName.EvaluationExecution),
-    payload: EvalExecutionEvent,
-  }),
-  z.object({
-    name: z.literal(EventName.BatchExport),
-    payload: BatchExportJobSchema,
-  }),
-  z.object({
-    name: z.literal(EventName.ExperimentCreate),
-    payload: ExperimentCreateEventSchema,
-  }),
-]);
-export type EventBodyType = z.infer<typeof EventBodySchema>;
+export type PostHogIntegrationProcessingEventType = z.infer<
+  typeof PostHogIntegrationProcessingEventSchema
+>;
+export type DataRetentionProcessingEventType = z.infer<
+  typeof DataRetentionProcessingEventSchema
+>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
   DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
-  LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
+  IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
   ExperimentCreate = "experiment-create-queue",
+  PostHogIntegrationQueue = "posthog-integration-queue",
+  PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
+  CoreDataS3ExportQueue = "core-data-s3-export-queue",
+  MeteringDataPostgresExportQueue = "metering-data-postgres-export-queue",
+  DataRetentionQueue = "data-retention-queue",
+  DataRetentionProcessingQueue = "data-retention-processing-queue",
 }
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
-  EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
-  LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  IngestionSecondaryJob = "secondary-ingestion-job",
   ExperimentCreateJob = "experiment-create-job",
+  PostHogIntegrationJob = "posthog-integration-job",
+  PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
+  CoreDataS3ExportJob = "core-data-s3-export-job",
+  MeteringDataPostgresExportJob = "metering-data-postgres-export-job",
+  DataRetentionJob = "data-retention-job",
+  DataRetentionProcessingJob = "data-retention-processing-job",
 }
 
 export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
     timestamp: Date;
     id: string;
-    payload: TraceUpsertEventType;
+    payload: TraceQueueEventType;
     name: QueueJobs.TraceUpsert;
   };
+  [QueueName.TraceDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: TracesQueueEventType | TraceQueueEventType;
+    name: QueueJobs.TraceDelete;
+  };
+  [QueueName.ProjectDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: ProjectQueueEventType;
+    name: QueueJobs.ProjectDelete;
+  };
   [QueueName.DatasetRunItemUpsert]: {
     timestamp: Date;
     id: string;
@@ -166,13 +150,13 @@ export type TQueueJobTypes = {
     payload: BatchExportJobType;
     name: QueueJobs.BatchExportJob;
   };
-  [QueueName.LegacyIngestionQueue]: {
+  [QueueName.IngestionQueue]: {
     timestamp: Date;
     id: string;
-    payload: LegacyIngestionEventType;
-    name: QueueJobs.LegacyIngestionJob;
+    payload: IngestionEventQueueType;
+    name: QueueJobs.IngestionJob;
   };
-  [QueueName.IngestionQueue]: {
+  [QueueName.IngestionSecondaryQueue]: {
     timestamp: Date;
     id: string;
     payload: IngestionEventQueueType;
@@ -184,4 +168,16 @@ export type TQueueJobTypes = {
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
   };
+  [QueueName.PostHogIntegrationProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: PostHogIntegrationProcessingEventType;
+    name: QueueJobs.PostHogIntegrationProcessingJob;
+  };
+  [QueueName.DataRetentionProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: DataRetentionProcessingEventType;
+    name: QueueJobs.DataRetentionProcessingJob;
+  };
 };

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -1,8 +1,8 @@
 import { Queue } from "bullmq";
-import { env } from "../..";
-import { logger } from "@azure/storage-blob";
+import { env } from "../../env";
 import { QueueName, QueueJobs } from "../queues";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
 
 export class CloudUsageMeteringQueue {
   private static instance: Queue | null = null;
@@ -45,6 +45,7 @@ export class CloudUsageMeteringQueue {
         QueueJobs.CloudUsageMeteringJob,
         {},
         {
+          // Run at minute 5 of every hour (e.g. 1:05, 2:05, 3:05, etc)
           repeat: { pattern: "5 * * * *" },
         },
       );

packages/shared/src/server/redis/coreDataS3ExportQueue.ts

@@ -0,0 +1,60 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class CoreDataS3ExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (CoreDataS3ExportQueue.instance) {
+      return CoreDataS3ExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CoreDataS3ExportQueue.instance = newRedis
+      ? new Queue(QueueName.CoreDataS3ExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CoreDataS3ExportQueue.instance?.on("error", (err) => {
+      logger.error("CoreDataS3ExportQueue error", err);
+    });
+
+    if (CoreDataS3ExportQueue.instance) {
+      logger.debug("Scheduling jobs for CoreDataS3ExportQueue");
+      CoreDataS3ExportQueue.instance
+        .add(
+          QueueJobs.CoreDataS3ExportJob,
+          {},
+          {
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding CoreDataS3ExportJob schedule", err);
+        });
+    }
+
+    return CoreDataS3ExportQueue.instance;
+  }
+}

packages/shared/src/server/redis/dataRetentionProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionProcessingQueue.instance) {
+      return DataRetentionProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 10000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionProcessingQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionProcessingQueue error", err);
+    });
+
+    return DataRetentionProcessingQueue.instance;
+  }
+}

packages/shared/src/server/redis/dataRetentionQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionQueue.instance) {
+      return DataRetentionQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionQueue error", err);
+    });
+
+    if (DataRetentionQueue.instance) {
+      logger.debug("Scheduling jobs for DataRetentionQueue");
+      DataRetentionQueue.instance
+        .add(
+          QueueJobs.DataRetentionJob,
+          {},
+          {
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding DataRetentionQueue schedule", err);
+        });
+    }
+
+    return DataRetentionQueue.instance;
+  }
+}

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -29,7 +29,7 @@ export class EvalExecutionQueue {
               attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 5000,
+                delay: 1000,
               },
             },
           },

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -26,10 +26,10 @@ export class ExperimentCreateQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 5000,
+                delay: 1000,
               },
             },
           },

packages/shared/src/server/redis/getQueue.ts

@@ -1,18 +1,23 @@
 import { Queue } from "bullmq";
 import { QueueName } from "../queues";
 import { BatchExportQueue } from "./batchExport";
-import { CloudUsageMeteringQueue } from "./CloudUsageMeteringQueue";
+import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
 import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
-import { IngestionQueue } from "./ingestionQueue";
-import { LegacyIngestionQueue } from "./legacyIngestion";
+import { IngestionQueue, SecondaryIngestionQueue } from "./ingestionQueue";
 import { TraceUpsertQueue } from "./traceUpsert";
+import { TraceDeleteQueue } from "./traceDelete";
+import { ProjectDeleteQueue } from "./projectDelete";
+import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
+import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
+import { CoreDataS3ExportQueue } from "./coreDataS3ExportQueue";
+import { MeteringDataPostgresExportQueue } from "./meteringDataPostgresExportQueue";
+import { DataRetentionQueue } from "./dataRetentionQueue";
+import { DataRetentionProcessingQueue } from "./dataRetentionProcessingQueue";
 
 export function getQueue(queueName: QueueName): Queue | null {
   switch (queueName) {
-    case QueueName.LegacyIngestionQueue:
-      return LegacyIngestionQueue.getInstance();
     case QueueName.BatchExport:
       return BatchExportQueue.getInstance();
     case QueueName.CloudUsageMeteringQueue:
@@ -25,8 +30,26 @@ export function getQueue(queueName: QueueName): Queue | null {
       return ExperimentCreateQueue.getInstance();
     case QueueName.TraceUpsert:
       return TraceUpsertQueue.getInstance();
+    case QueueName.TraceDelete:
+      return TraceDeleteQueue.getInstance();
     case QueueName.IngestionQueue:
       return IngestionQueue.getInstance();
+    case QueueName.ProjectDelete:
+      return ProjectDeleteQueue.getInstance();
+    case QueueName.PostHogIntegrationQueue:
+      return PostHogIntegrationQueue.getInstance();
+    case QueueName.PostHogIntegrationProcessingQueue:
+      return PostHogIntegrationProcessingQueue.getInstance();
+    case QueueName.IngestionSecondaryQueue:
+      return SecondaryIngestionQueue.getInstance();
+    case QueueName.CoreDataS3ExportQueue:
+      return CoreDataS3ExportQueue.getInstance();
+    case QueueName.MeteringDataPostgresExportQueue:
+      return MeteringDataPostgresExportQueue.getInstance();
+    case QueueName.DataRetentionQueue:
+      return DataRetentionQueue.getInstance();
+    case QueueName.DataRetentionProcessingQueue:
+      return DataRetentionProcessingQueue.getInstance();
     default:
       const exhaustiveCheckDefault: never = queueName;
       throw new Error(`Queue ${queueName} not found`);

packages/shared/src/server/redis/ingestionQueue.ts

@@ -43,3 +43,45 @@ export class IngestionQueue {
     return IngestionQueue.instance;
   }
 }
+
+export class SecondaryIngestionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null {
+    if (SecondaryIngestionQueue.instance)
+      return SecondaryIngestionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    SecondaryIngestionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(
+          QueueName.IngestionSecondaryQueue,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    SecondaryIngestionQueue.instance?.on("error", (err) => {
+      logger.error("SecondaryIngestionQueue error", err);
+    });
+
+    return SecondaryIngestionQueue.instance;
+  }
+}

packages/shared/src/server/redis/meteringDataPostgresExportQueue.ts

@@ -0,0 +1,63 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class MeteringDataPostgresExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      return MeteringDataPostgresExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    MeteringDataPostgresExportQueue.instance = newRedis
+      ? new Queue(QueueName.MeteringDataPostgresExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    MeteringDataPostgresExportQueue.instance?.on("error", (err) => {
+      logger.error("MeteringDataPostgresExportQueue error", err);
+    });
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      logger.debug("Scheduling jobs for MeteringDataPostgresExportQueue");
+      MeteringDataPostgresExportQueue.instance
+        .add(
+          QueueJobs.MeteringDataPostgresExportJob,
+          {},
+          {
+            repeat: { pattern: "30 2 * * *" }, // every day at 2:30am UTC
+          },
+        )
+        .catch((err) => {
+          logger.error(
+            "Error adding MeteringDataPostgresExportJob schedule",
+            err,
+          );
+        });
+    }
+
+    return MeteringDataPostgresExportQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationProcessingQueue.instance) {
+      return PostHogIntegrationProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100_000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationProcessingQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationProcessingQueue error", err);
+    });
+
+    return PostHogIntegrationProcessingQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationQueue.instance) {
+      return PostHogIntegrationQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationQueue error", err);
+    });
+
+    if (PostHogIntegrationQueue.instance) {
+      logger.debug("Scheduling jobs for PostHogIntegrationQueue");
+      PostHogIntegrationQueue.instance
+        .add(
+          QueueJobs.PostHogIntegrationJob,
+          {},
+          {
+            repeat: { pattern: "30 * * * *" }, // every hour at 30 minutes past
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding PostHogIntegrationJob schedule", err);
+        });
+    }
+
+    return PostHogIntegrationQueue.instance;
+  }
+}

packages/shared/src/server/redis/projectDelete.ts

@@ -1,31 +1,31 @@
-import { Queue } from "bullmq";
 import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";
 
-export class LegacyIngestionQueue {
+export class ProjectDeleteQueue {
   private static instance: Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.ProjectDelete]
   > | null = null;
 
   public static getInstance(): Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.ProjectDelete]
   > | null {
-    if (LegacyIngestionQueue.instance) return LegacyIngestionQueue.instance;
+    if (ProjectDeleteQueue.instance) return ProjectDeleteQueue.instance;
 
     const newRedis = createNewRedisInstance({
       enableOfflineQueue: false,
       ...redisQueueRetryOptions,
     });
 
-    LegacyIngestionQueue.instance = newRedis
-      ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
-          QueueName.LegacyIngestionQueue,
+    ProjectDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ProjectDelete]>(
+          QueueName.ProjectDelete,
           {
             connection: newRedis,
             defaultJobOptions: {
               removeOnComplete: true,
-              removeOnFail: 500_000,
+              removeOnFail: 100_000,
               attempts: 5,
               backoff: {
                 type: "exponential",
@@ -36,10 +36,10 @@ export class LegacyIngestionQueue {
         )
       : null;
 
-    LegacyIngestionQueue.instance?.on("error", (err) => {
-      logger.error("LegacyIngestionQueue error", err);
+    ProjectDeleteQueue.instance?.on("error", (err) => {
+      logger.error("ProjectDeleteQueue error", err);
     });
 
-    return LegacyIngestionQueue.instance;
+    return ProjectDeleteQueue.instance;
   }
 }

packages/shared/src/server/redis/traceDelete.ts

@@ -0,0 +1,44 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class TraceDeleteQueue {
+  private static instance: Queue<TQueueJobTypes[QueueName.TraceDelete]> | null =
+    null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.TraceDelete]
+  > | null {
+    if (TraceDeleteQueue.instance) return TraceDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    TraceDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.TraceDelete]>(
+          QueueName.TraceDelete,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    TraceDeleteQueue.instance?.on("error", (err) => {
+      logger.error("TraceDeleteQueue error", err);
+    });
+
+    return TraceDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -1,10 +1,4 @@
-import { randomUUID } from "crypto";
-import {
-  QueueJobs,
-  QueueName,
-  TQueueJobTypes,
-  TraceUpsertEventType,
-} from "../queues";
+import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";
@@ -32,7 +26,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
-              delay: 10_000, // 10 seconds
+              delay: 15_000, // 15 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/repositories/clickhouse.ts

@@ -4,7 +4,7 @@ import {
   convertDateToClickhouseDateTime,
 } from "../clickhouse/client";
 import { logger } from "../logger";
-import { instrumentAsync } from "../instrumentation";
+import { getTracer, instrumentAsync } from "../instrumentation";
 import {
   StorageService,
   StorageServiceFactory,
@@ -12,6 +12,7 @@ import {
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
+import { context, trace } from "@opentelemetry/api";
 
 let s3StorageServiceClient: StorageService;
 
@@ -40,37 +41,31 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // If event upload is enabled, we store all rows in S3 to have a backup
-    if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      await Promise.all(
-        opts.records.map((record) => {
-          // drop trailing s and pretend it's always a create.
-          // Only applicable to scores and traces.
-          let eventType = `${opts.table.slice(0, -1)}-create`;
-          if (opts.table === "observations") {
-            // @ts-ignore - If it's an observation we now that `type` is a string
-            eventType = `${record["type"].toLowerCase()}-create`;
-          }
-          s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
-            [
-              {
-                id: randomUUID(),
-                timestamp: new Date().toISOString(),
-                type: eventType,
-                body: opts.eventBodyMapper(record),
-              },
-            ],
-          );
-        }),
-      );
-    }
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    await Promise.all(
+      opts.records.map((record) => {
+        // drop trailing s and pretend it's always a create.
+        // Only applicable to scores and traces.
+        let eventType = `${opts.table.slice(0, -1)}-create`;
+        if (opts.table === "observations") {
+          // @ts-ignore - If it's an observation we now that `type` is a string
+          eventType = `${record["type"].toLowerCase()}-create`;
+        }
+        s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+          [
+            {
+              id: randomUUID(),
+              timestamp: new Date().toISOString(),
+              type: eventType,
+              body: opts.eventBodyMapper(record),
+            },
+          ],
+        );
+      }),
+    );
 
     const res = await clickhouseClient().insert({
       table: opts.table,
@@ -107,6 +102,64 @@ export async function upsertClickhouse<
   });
 }
 
+export async function* queryClickhouseStream<T>(opts: {
+  query: string;
+  params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
+}): AsyncGenerator<T> {
+  const tracer = getTracer("clickhouse-query-stream");
+  const span = tracer.startSpan("clickhouse-query-stream");
+
+  try {
+    const res = await context.with(
+      trace.setSpan(context.active(), span),
+      async () => {
+        // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+        span.setAttribute("ch.query.text", opts.query);
+
+        const res = await clickhouseClient(opts.clickhouseConfigs).query({
+          query: opts.query,
+          format: "JSONEachRow",
+          query_params: opts.params,
+        });
+        // same logic as for prisma. we want to see queries in development
+        if (env.NODE_ENV === "development") {
+          logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+        }
+
+        span.setAttribute("ch.queryId", res.query_id);
+
+        // add summary headers to the span. Helps to tune performance
+        const summaryHeader = res.response_headers["x-clickhouse-summary"];
+        if (summaryHeader) {
+          try {
+            const summary = Array.isArray(summaryHeader)
+              ? JSON.parse(summaryHeader[0])
+              : JSON.parse(summaryHeader);
+            for (const key in summary) {
+              span.setAttribute(`ch.${key}`, summary[key]);
+            }
+          } catch (error) {
+            logger.debug(
+              `Failed to parse clickhouse summary header ${summaryHeader}`,
+              error,
+            );
+          }
+        }
+        return res;
+      },
+    );
+
+    for await (const rows of res.stream<T>()) {
+      for (const row of rows) {
+        yield row.json();
+      }
+    }
+  } finally {
+    span.end();
+  }
+}
+
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;

packages/shared/src/server/repositories/dashboards.ts

@@ -1,4 +1,7 @@
-import { queryClickhouse } from "./clickhouse";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "./clickhouse";
 import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
 import { FilterState } from "../../types";
 import {
@@ -142,23 +145,26 @@ export const getScoreAggregate = async (
 export const groupTracesByTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
   ).apply();
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
   const query = `
     SELECT 
-      ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
       count(*) as count
     FROM traces t FINAL
     WHERE project_id = {projectId: String}
     AND ${chFilter.query}
     GROUP BY timestamp
-    ${orderByTimeSeries(groupBy, "timestamp")}
+    ${orderByQuery}
     `;
-
   const result = await queryClickhouse<{
     timestamp: string;
     count: string;
@@ -167,11 +173,12 @@ export const groupTracesByTime = async (
     params: {
       projectId,
       ...chFilter.params,
+      ...orderByParams,
     },
   });
 
   return result.map((row) => ({
-    timestamp: new Date(row.timestamp),
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
     countTraceId: Number(row.count),
   }));
 };
@@ -179,7 +186,6 @@ export const groupTracesByTime = async (
 export const getObservationUsageByTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -197,11 +203,16 @@ export const getObservationUsageByTime = async (
       ) as DateTimeFilter | undefined)
     : undefined;
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time",
+  );
+
   const query = `
     SELECT 
-      ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
-      sumMap(usage_details)['total'] as sum_usage_details,
-      sumMap(cost_details)['total'] as sum_cost_details,
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
+      sumMap(usage_details) as units,
+      sumMap(cost_details) as cost, 
       provided_model_name
     FROM observations o FINAL
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -209,19 +220,20 @@ export const getObservationUsageByTime = async (
     AND ${appliedFilter.query}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY start_time, provided_model_name
-    ${orderByTimeSeries(groupBy, "start_time")}
+    ${orderByQuery}
     `;
 
   const result = await queryClickhouse<{
     start_time: string;
-    sum_usage_details: string;
-    sum_cost_details: number;
+    units: Record<string, number>;
+    cost: Record<string, number>;
     provided_model_name: string;
   }>({
     query,
     params: {
       projectId,
       ...appliedFilter.params,
+      ...orderByParams,
       ...(timeFilter
         ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
         : {}),
@@ -229,9 +241,19 @@ export const getObservationUsageByTime = async (
   });
 
   return result.map((row) => ({
-    start_time: new Date(row.start_time),
-    sum_usage_details: Number(row.sum_usage_details),
-    sum_cost_details: row.sum_cost_details,
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time),
+    units: Object.fromEntries(
+      Object.entries(row.units ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
+    cost: Object.fromEntries(
+      Object.entries(row.cost ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
     provided_model_name: row.provided_model_name,
   }));
 };
@@ -283,7 +305,6 @@ export const getDistinctModels = async (
 export const getScoresAggregateOverTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -294,9 +315,14 @@ export const getScoresAggregateOverTime = async (
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
   // TODO: Validate whether we can filter traces on timestamp here.
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
   const query = `
   SELECT 
-    ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
     name,
     data_type,
     source,
@@ -311,7 +337,7 @@ export const getScoresAggregateOverTime = async (
     name,
     data_type,
     source
-  ${orderByTimeSeries(groupBy, "timestamp")};
+  ${orderByQuery};
 `;
 
   const result = await queryClickhouse<{
@@ -325,11 +351,12 @@ export const getScoresAggregateOverTime = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...orderByParams,
     },
   });
 
   return result.map((row) => ({
-    scoreTimestamp: new Date(row.timestamp),
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
     scoreName: row.name,
     scoreDataType: row.data_type,
     scoreSource: row.source,
@@ -405,7 +432,7 @@ export const getObservationLatencies = async (
   // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
     SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles,
       name
     FROM observations o
     ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -452,7 +479,7 @@ export const getTracesLatencies = async (
       select o.trace_id,
              t.name,
              o.project_id,
-             date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
+             date_diff('millisecond', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
       FROM traces t 
       JOIN observations o
       ON o.trace_id = t.id AND o.project_id = t.project_id
@@ -493,7 +520,6 @@ export const getTracesLatencies = async (
 export const getModelLatenciesOverTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -503,25 +529,33 @@ export const getModelLatenciesOverTime = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
   // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
   SELECT 
-    ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket")},
     provided_model_name,
-    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles
   FROM observations o
   ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
   AND ${appliedFilter.query}
   GROUP BY provided_model_name, start_time_bucket
-  ${orderByTimeSeries(groupBy, "start_time_bucket")};
+  ${orderByQuery};
 `;
 
   const result = await queryClickhouse<{
     start_time_bucket: string;
     provided_model_name: string;
     quantiles: string[];
-  }>({ query, params: { projectId, ...appliedFilter.params } });
+  }>({
+    query,
+    params: { projectId, ...appliedFilter.params, ...orderByParams },
+  });
 
   return result.map((row) => ({
     p50: Number(row.quantiles[0]) / 1000,
@@ -530,14 +564,13 @@ export const getModelLatenciesOverTime = async (
     p95: Number(row.quantiles[3]) / 1000,
     p99: Number(row.quantiles[4]) / 1000,
     model: row.provided_model_name,
-    start_time: new Date(row.start_time_bucket),
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time_bucket),
   }));
 };
 
 export const getNumericScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -546,9 +579,14 @@ export const getNumericScoreTimeSeries = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
   const query = `
     SELECT
-    ${selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp")},
     s.name as score_name,
     AVG(s.value) as avg_value
     FROM scores s final
@@ -556,11 +594,11 @@ export const getNumericScoreTimeSeries = async (
     WHERE s.project_id = {projectId: String}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     GROUP BY score_name, score_timestamp
-    ${orderByTimeSeries(groupBy, "score_timestamp")}
+    ${orderByQuery}
   `;
 
-  return queryClickhouse<{
-    score_timestamp: Date;
+  const result = await queryClickhouse<{
+    score_timestamp: string;
     score_name: string;
     avg_value: number;
   }>({
@@ -568,14 +606,20 @@ export const getNumericScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
     },
   });
+
+  return result.map((row) => ({
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.score_timestamp),
+    scoreName: row.score_name,
+    avgValue: Number(row.avg_value),
+  }));
 };
 
 export const getCategoricalScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc | undefined,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -584,9 +628,14 @@ export const getCategoricalScoreTimeSeries = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
   const query = `
     SELECT
-    ${groupBy ? selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp") + ", " : ""}
+    ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp") + ", " : ""}
     s.name as score_name,
     s.data_type as score_data_type,
     s.source as score_source,
@@ -596,12 +645,12 @@ export const getCategoricalScoreTimeSeries = async (
     ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
     WHERE s.project_id = {projectId: String}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
-    GROUP BY score_name, score_data_type, score_source, score_value ${groupBy ? ", score_timestamp" : ""}
-    ${groupBy ? orderByTimeSeries(groupBy, "score_timestamp") : ""}
+    GROUP BY score_name, score_data_type, score_source, score_value ${bucketSizeInSeconds ? ", score_timestamp" : ""}
+      ${orderByQuery}
   `;
 
-  return queryClickhouse<{
-    score_timestamp?: Date;
+  const result = await queryClickhouse<{
+    score_timestamp?: string;
     score_name: string;
     score_data_type: string;
     score_source: string;
@@ -612,65 +661,138 @@ export const getCategoricalScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
     },
   });
+
+  return result.map((row) => ({
+    scoreTimestamp: row.score_timestamp
+      ? parseClickhouseUTCDateTimeFormat(row.score_timestamp)
+      : undefined,
+    scoreName: row.score_name,
+    scoreDataType: row.score_data_type,
+    scoreSource: row.score_source,
+    scoreValue: row.score_value,
+    count: Number(row.count),
+  }));
 };
 
-const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toIntervalYear(1)";
-      break;
-    case "month":
-      interval = "toIntervalMonth(1)";
-      break;
-    case "week":
-      interval = "toIntervalWeek(1)";
-      break;
-    case "day":
-      interval = "toIntervalDay(1)";
-      break;
-    case "hour":
-      interval = "toIntervalHour(1)";
-      break;
-    case "minute":
-      interval = "toIntervalMinute(1)";
-      break;
-    default:
-      return undefined;
+export const getObservationsStatusTimeSeries = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
+  const query = `
+    SELECT 
+      ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket") + ", " : ""}
+      count(*) as observation_count,
+      level as level
+    FROM observations o
+    ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND o.level IS NOT NULL
+    AND ${chFilterRes?.query}
+    GROUP BY level ${bucketSizeInSeconds ? ", start_time_bucket" : ""}
+    ${orderByQuery}
+  `;
+
+  const result = await queryClickhouse<{
+    start_time_bucket?: string;
+    observation_count: string;
+    level: string;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
+    },
+  });
+
+  return result.map((row) => ({
+    start_time_bucket: row.start_time_bucket
+      ? parseClickhouseUTCDateTimeFormat(row.start_time_bucket)
+      : undefined,
+    count: Number(row.observation_count),
+    level: row.level,
+  }));
+};
+
+export const orderByTimeSeries = (
+  filter: FilterState,
+  col: string,
+): [string, { fromTime: number; toTime: number }, number] => {
+  const potentialBucketSizesSeconds = [
+    5, 10, 30, 60, 300, 600, 1800, 3600, 18000, 36000, 86400, 604800, 2592000,
+  ];
+
+  // Calculate time difference in seconds
+  const [from, to] = extractFromAndToTimestampsFromFilter(filter);
+
+  if (!from || !to) {
+    throw new Error("Time Filter is required for time series queries");
   }
 
-  return `ORDER BY ${col} ASC WITH FILL STEP ${interval}`;
+  const fromDate = new Date(from.value as Date);
+  const toDate = new Date(to.value as Date);
+
+  const diffInSeconds = Math.abs(toDate.getTime() - fromDate.getTime()) / 1000;
+
+  // choose the bucket size that is the closest to the desired number of buckets
+  const bucketSizeInSeconds = potentialBucketSizesSeconds.reduce(
+    (closest, size) => {
+      const diffFromDesiredBuckets = Math.abs(diffInSeconds / size - 50);
+      return diffFromDesiredBuckets < closest.diffFromDesiredBuckets
+        ? { size, diffFromDesiredBuckets }
+        : closest;
+    },
+    { size: 0, diffFromDesiredBuckets: Infinity },
+  ).size;
+
+  // Convert to interval string
+  const interval = `toIntervalSecond(${bucketSizeInSeconds})`;
+
+  return [
+    `ORDER BY ${col} ASC 
+    WITH FILL
+    FROM toStartOfInterval(toDateTime({fromTime: DateTime64(3)}), INTERVAL ${bucketSizeInSeconds} SECOND)
+    TO toDateTime({toTime: DateTime64(3)}) + INTERVAL ${bucketSizeInSeconds} SECOND
+    STEP ${interval}`,
+    { fromTime: fromDate.getTime(), toTime: toDate.getTime() },
+    bucketSizeInSeconds,
+  ];
 };
 
-const selectTimeseriesColumn = (
-  dateTrunc: DateTrunc,
+export const selectTimeseriesColumn = (
+  bucketSizeInSeconds: number,
   col: string,
   as: String,
 ) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toStartOfYear";
-      break;
-    case "month":
-      interval = "toStartOfMonth";
-      break;
-    case "week":
-      interval = "toStartOfWeek";
-      break;
-    case "day":
-      interval = "toStartOfDay";
-      break;
-    case "hour":
-      interval = "toStartOfHour";
-      break;
-    case "minute":
-      interval = "toStartOfMinute";
-      break;
-    default:
-      return undefined;
-  }
-  return `${interval}(${col}) as ${as}`;
+  return `toStartOfInterval(${col}, INTERVAL ${bucketSizeInSeconds} SECOND) as ${as}`;
+};
+
+export const extractFromAndToTimestampsFromFilter = (filter?: FilterState) => {
+  if (!filter)
+    throw new Error("Time Filter is required for time series queries");
+
+  const fromTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === ">" || f.operator === ">="),
+  );
+
+  const toTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === "<" || f.operator === "<="),
+  );
+
+  return [fromTimestamp[0], toTimestamp[0]];
 };

packages/shared/src/server/repositories/index.ts

@@ -7,3 +7,5 @@ export * from "./traces_converters";
 export * from "./scores_converters";
 export * from "./observations_converters";
 export * from "./clickhouse";
+export * from "./constants";
+export * from "./trace-sessions";

packages/shared/src/server/repositories/observations.ts

@@ -2,6 +2,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  queryClickhouseStream,
   upsertClickhouse,
 } from "./clickhouse";
 import { ObservationLevel } from "@prisma/client";
@@ -38,6 +39,7 @@ import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
+import { env } from "../../env";
 
 export const checkObservationExists = async (
   projectId: string,
@@ -210,11 +212,13 @@ export const getObservationById = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const records = await getObservationByIdInternal(
     id,
     projectId,
     fetchWithInputOutput,
+    startTime,
   );
   const mapped = records.map(convertObservation);
 
@@ -311,6 +315,7 @@ const getObservationByIdInternal = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const query = `
   SELECT
@@ -345,11 +350,18 @@ const getObservationByIdInternal = async (
   FROM observations
   WHERE id = {id: String}
   AND project_id = {projectId: String}
+  ${startTime ? `AND start_time = {startTime: DateTime64(3)}` : ""}
   ORDER BY event_ts desc
   LIMIT 1 by id, project_id`;
   return await queryClickhouse<ObservationRecordReadType>({
     query,
-    params: { id, projectId },
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
   });
 };
 
@@ -389,7 +401,7 @@ export type ObservationsTableQueryResult = ObservationRecordReadType & {
 export const getObservationsTableCount = async (opts: ObservationTableQuery) =>
   getObservationsTableInternal<TableCount>({
     ...opts,
-    select: "count(*) as count",
+    select: "count",
   });
 
 export type ObservationsTableRow = Omit<
@@ -403,33 +415,11 @@ export const getObservationsTable = async (
   const observationRecords = await getObservationsTableInternal<
     Omit<
       ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const traces = await getTracesByIds(
@@ -442,7 +432,7 @@ export const getObservationsTable = async (
   return observationRecords.map((o) => {
     const trace = traces.find((t) => t.id === o.trace_id);
     return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
       latency: o.latency ? Number(o.latency) / 1000 : null,
       timeToFirstToken: o.time_to_first_token
         ? Number(o.time_to_first_token) / 1000
@@ -460,33 +450,11 @@ export const getObservationsTableWithModelData = async (
   const observationRecords = await getObservationsTableInternal<
     Omit<
       ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const uniqueModels: string[] = Array.from(
@@ -523,7 +491,7 @@ export const getObservationsTableWithModelData = async (
     const trace = traces.find((t) => t.id === o.trace_id);
     const model = models.find((m) => m.id === o.internal_model_id);
     return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
       latency: o.latency ? Number(o.latency) / 1000 : null,
       timeToFirstToken: o.time_to_first_token
         ? Number(o.time_to_first_token) / 1000
@@ -543,17 +511,49 @@ export const getObservationsTableWithModelData = async (
 };
 
 const getObservationsTableInternal = async <T>(
-  opts: ObservationTableQuery & { select: string },
+  opts: ObservationTableQuery & { select: "count" | "rows" },
 ): Promise<Array<T>> => {
+  const select =
+    opts.select === "count"
+      ? "count(*) as count"
+      : `
+        o.id as id,
+        o.type as type,
+        o.project_id as "project_id",
+        o.name as name,
+        o."model_parameters" as model_parameters,
+        o.start_time as "start_time",
+        o.end_time as "end_time",
+        o.trace_id as "trace_id",
+        o.completion_start_time as "completion_start_time",
+        o.provided_usage_details as "provided_usage_details",
+        o.usage_details as "usage_details",
+        o.provided_cost_details as "provided_cost_details",
+        o.cost_details as "cost_details",
+        o.level as level,
+        o.status_message as "status_message",
+        o.version as version,
+        o.parent_observation_id as "parent_observation_id",
+        o.created_at as "created_at",
+        o.updated_at as "updated_at",
+        o.provided_model_name as "provided_model_name",
+        o.total_cost as "total_cost",
+        o.prompt_id as "prompt_id",
+        o.prompt_name as "prompt_name",
+        o.prompt_version as "prompt_version",
+        internal_model_id as "internal_model_id",
+        if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time)) as "time_to_first_token"`;
+
   const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
     opts;
 
   const selectString = selectIOAndMetadata
     ? `
-    ${opts.select},
+    ${select},
     ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
   `
-    : opts.select;
+    : select;
 
   const scoresFilter = new FilterList([
     new StringFilter({
@@ -580,6 +580,10 @@ const getObservationsTableInternal = async <T>(
         .includes(f.column),
   );
 
+  const hasScoresFilter = filter.some(
+    (f) => f.column === "Scores" || f.column === "scores",
+  );
+
   const orderByTraces = opts.orderBy
     ? observationsTableTraceUiColumnDefinitions
         .map((c) => c.uiTableId)
@@ -650,63 +654,58 @@ const getObservationsTableInternal = async <T>(
       observation_id
   )`;
 
-  if (traceTableFilter.length > 0 || orderByTraces) {
-    // joins with traces are very expensive. We need to filter by time as well.
-    // We assume that a trace has to have been within the last 2 days to be relevant.
+  // if we have default ordering by time, we order by toDate(o.start_time) first and then by
+  // o.start_time. This way, clickhouse is able to read more efficiently directly from disk without ordering
+  const newDefaultOrder =
+    orderBy?.column === "startTime"
+      ? [{ column: "order_by_date", order: orderBy.order }, orderBy]
+      : [orderBy ?? null];
 
-    const query = `
+  const chOrderBy = orderByToClickhouseSql(newDefaultOrder, [
+    ...observationsTableUiColumnDefinitions,
+    {
+      uiTableName: "order_by_date",
+      uiTableId: "order_by_date",
+      clickhouseTableName: "observation",
+      clickhouseSelect: "toDate(o.start_time)",
+    },
+  ]);
+
+  // joins with traces are very expensive. We need to filter by time as well.
+  // We assume that a trace has to have been within the last 2 days to be relevant.
+
+  const query = `
       ${scoresCte}
       SELECT
        ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
+      FROM observations o 
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${hasScoresFilter ? `LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
-        AND o.type = 'GENERATION'
-        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        
+        ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
+      ${chOrderBy}
+      ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-        ...(timeFilter
-          ? {
-              tracesTimestampFilter: convertDateToClickhouseDateTime(
-                timeFilter.value as Date,
-              ),
-            }
-          : {}),
-        ...search.params,
-      },
-    });
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedScoresFilter.params,
+      ...appliedObservationsFilter.params,
+      ...(timeFilter
+        ? {
+            tracesTimestampFilter: convertDateToClickhouseDateTime(
+              timeFilter.value as Date,
+            ),
+          }
+        : {}),
+      ...search.params,
+    },
+  });
 
-    return res;
-  } else {
-    // we query by T, which could also be {count: string}.
-    const query = `
-      ${scoresCte}
-      SELECT
-       ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
-      WHERE ${appliedObservationsFilter.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
-      ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
-
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-      },
-    });
-
-    return res;
-  }
+  return res;
 };
 
 export const getObservationsGroupedByModel = async (
@@ -753,6 +752,50 @@ export const getObservationsGroupedByModel = async (
   return res.map((r) => ({ model: r.name }));
 };
 
+export const getObservationsGroupedByModelId = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const observationsFilter = new FilterList([
+    new StringFilter({
+      clickhouseTable: "observations",
+      field: "project_id",
+      operator: "=",
+      value: projectId,
+      tablePrefix: "o",
+    }),
+  ]);
+
+  observationsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      observationsTableUiColumnDefinitions,
+    ),
+  );
+
+  const appliedObservationsFilter = observationsFilter.apply();
+
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
+  const query = `
+    SELECT o.internal_model_id as modelId
+    FROM observations o
+    WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
+    GROUP BY o.internal_model_id
+    ORDER BY count() DESC
+    LIMIT 1000;
+  `;
+
+  const res = await queryClickhouse<{ modelId: string }>({
+    query,
+    params: {
+      ...appliedObservationsFilter.params,
+    },
+  });
+  return res.map((r) => ({ modelId: r.modelId }));
+};
+
 export const getObservationsGroupedByName = async (
   projectId: string,
   filter: FilterState,
@@ -907,6 +950,46 @@ export const deleteObservationsByTraceIds = async (
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteObservationsByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteObservationsOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String}
+    AND start_time < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -950,7 +1033,7 @@ export const getObservationMetricsForPrompts = async (
                   end_time,
                   usage_details,
                   cost_details,
-                  dateDiff('milliseconds', start_time, end_time) AS latency_ms
+                  dateDiff('millisecond', start_time, end_time) AS latency_ms
               FROM observations
               FINAL
               WHERE (type = 'GENERATION') 
@@ -964,8 +1047,8 @@ export const getObservationMetricsForPrompts = async (
           prompt_version,
           min(start_time) AS first_observation,
           max(start_time) AS last_observation,
-          medianExact(usage_details['input']) AS median_input_usage,
-          medianExact(usage_details['output']) AS median_output_usage,
+          medianExact(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))) AS median_input_usage,
+          medianExact(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))) AS median_output_usage,
           medianExact(cost_details['total']) AS median_total_cost,
           medianExact(latency_ms) AS median_latency_ms
       FROM latencies
@@ -1013,7 +1096,7 @@ export const getLatencyAndTotalCostForObservations = async (
     SELECT
         id,
         cost_details['total'] AS total_cost,
-        dateDiff('milliseconds', start_time, end_time) AS latency_ms
+        dateDiff('millisecond', start_time, end_time) AS latency_ms
     FROM observations FINAL 
     WHERE project_id = {projectId: String} 
     AND id IN ({observationIds: Array(String)})
@@ -1045,7 +1128,7 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     SELECT
         trace_id,
         sumMap(cost_details)['total'] AS total_cost,
-        dateDiff('milliseconds', min(start_time), max(end_time)) AS latency_ms
+        dateDiff('millisecond', min(start_time), max(end_time)) AS latency_ms
     FROM observations FINAL
     WHERE project_id = {projectId: String} 
     AND trace_id IN ({traceIds: Array(String)})
@@ -1069,3 +1152,167 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     latency: Number(r.latency_ms) / 1000,
   }));
 };
+
+export const getObservationCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM observations
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
+export const getObservationCountOfProjectsSinceCreationDate = async ({
+  projectIds,
+  start,
+}: {
+  projectIds: string[];
+  start: Date;
+}) => {
+  const query = `
+    SELECT 
+      count(*) as count
+    FROM observations
+    WHERE project_id IN ({projectIds: Array(String)})
+    AND created_at >= {start: DateTime64(3)}
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectIds,
+      start: convertDateToClickhouseDateTime(start),
+    },
+  });
+
+  return Number(rows[0]?.count ?? 0);
+};
+
+export const getTraceIdsForObservations = async (
+  projectId: string,
+  observationIds: string[],
+) => {
+  const query = `
+    SELECT 
+      trace_id,
+      id
+    FROM observations
+    WHERE project_id = {projectId: String}
+    AND id IN ({observationIds: Array(String)})
+  `;
+
+  const rows = await queryClickhouse<{ id: string; trace_id: string }>({
+    query,
+    params: {
+      projectId,
+      observationIds,
+    },
+  });
+
+  return rows.map((row) => ({
+    id: row.id,
+    traceId: row.trace_id,
+  }));
+};
+
+export const getGenerationsForPostHog = async function* (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) {
+  const query = `
+    SELECT
+      o.name as name,
+      o.start_time as start_time,
+      o.id as id,
+      o.total_cost as total_cost,
+      if(isNull(completion_start_time), NULL, date_diff('millisecond', start_time, completion_start_time)) as time_to_first_token,
+      o.usage_details['total'] as input_tokens,
+      o.usage_details['output'] as output_tokens,
+      o.cost_details['total'] as total_tokens,
+      o.project_id as project_id,
+      if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000) as latency,
+      o.provided_model_name as model,
+      o.level as level,
+      o.version as version,
+      t.id as trace_id,
+      t.name as trace_name,
+      t.session_id as trace_session_id,
+      t.user_id as trace_user_id,
+      t.release as trace_release,
+      t.tags as trace_tags,
+      t.metadata['$posthog_session_id'] as posthog_session_id
+    FROM observations o FINAL
+    LEFT JOIN traces t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
+    WHERE o.project_id = {projectId: String}
+    AND t.project_id = {projectId: String}
+    AND o.start_time >= {minTimestamp: DateTime64(3)}
+    AND o.start_time <= {maxTimestamp: DateTime64(3)}
+    AND t.timestamp >= {minTimestamp: DateTime64(3)} - INTERVAL 7 DAY
+    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
+    AND o.type = 'GENERATION'
+  `;
+
+  const records = queryClickhouseStream<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+  });
+
+  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
+  for await (const record of records) {
+    yield {
+      timestamp: record.start_time,
+      langfuse_generation_name: record.name,
+      langfuse_trace_name: record.trace_name,
+      langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
+      langfuse_id: record.id,
+      langfuse_cost_usd: record.total_cost,
+      langfuse_input_units: record.input_tokens,
+      langfuse_output_units: record.output_tokens,
+      langfuse_total_units: record.total_tokens,
+      langfuse_session_id: record.trace_session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+      langfuse_latency: record.latency,
+      langfuse_time_to_first_token: record.time_to_first_token,
+      langfuse_release: record.trace_release,
+      langfuse_version: record.version,
+      langfuse_model: record.model,
+      langfuse_level: record.level,
+      langfuse_tags: record.trace_tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
+};