chore: release v3.6.2

* fix: fix trace exports

* push

* fix: fix trace exports

.env.dev-azure.example

@@ -0,0 +1,84 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_BATCH_EXPORT_REGION=auto
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.dev.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -34,17 +34,18 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
-# # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+# S3 Media Upload LOCAL
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -56,7 +57,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -79,11 +79,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=false
-LANGFUSE_RETURN_FROM_CLICKHOUSE=true
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
-
-LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
-LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.legacy.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -34,17 +34,18 @@ SALT="salt"
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
 
-# S3 storage
-S3_ENDPOINT=http://localhost:9090
-S3_ACCESS_KEY_ID=minio
-S3_SECRET_ACCESS_KEY=miniosecret
-S3_BUCKET_NAME=langfuse
-S3_REGION=us-east-1
+# S3 Batch Exports
+LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
+LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
+LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
+LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
+LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
 ## Necessary for minio compatibility
-S3_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
 
 # # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -56,7 +57,6 @@ LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
 
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
 LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
 LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
@@ -78,4 +78,9 @@ REDIS_AUTH="myredissecret"
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=true
+LANGFUSE_RETURN_FROM_CLICKHOUSE=false
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false

.env.local.example

@@ -1,22 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@db:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@db:5432/postgres"
-# Next Auth
-NEXTAUTH_SECRET="secret"
-NEXTAUTH_URL="http://localhost:3000"
-
-# feature flag to enable experimental features locally
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="false"
-SALT="salt"
-
-# Redis
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,11 +92,16 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_CLIENT_AUTH_METHOD="client_secret_basic" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
 
 # Transactional email, optional
@@ -100,16 +110,16 @@ OTEL_SERVICE_NAME="langfuse"
 # Defines the connection url for smtp server. 
 # SMTP_CONNECTION_URL=
 
-# S3 storage, optional, used for exports from the UI
-# S3_ENDPOINT=
-# S3_ACCESS_KEY_ID=
-# S3_SECRET_ACCESS_KEY=
-# S3_BUCKET_NAME=
-# S3_REGION=
-# BATCH_EXPORT_DOWNLOAD_LINK_EXPIRATION_HOURS=
+# S3 Batch Exports
+# LANGFUSE_S3_BATCH_EXPORT_ENABLED=
+# LANGFUSE_S3_BATCH_EXPORT_BUCKET=
+# LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=
+# LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=
+# LANGFUSE_S3_BATCH_EXPORT_REGION=
+# LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=
+# LANGFUSE_S3_BATCH_EXPORT_PREFIX=
 
 # S3 storage for events, optional, used to persist all incoming events
-# LANGFUSE_S3_EVENT_UPLOAD_ENABLED="true"
 # LANGFUSE_S3_EVENT_UPLOAD_BUCKET=
 # Optional prefix to be used within the bucket. Must end with `/` if set
 # LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
@@ -235,6 +245,7 @@ OTEL_SERVICE_NAME="langfuse"
 # CLICKHOUSE_URL=
 # CLICKHOUSE_USER=
 # CLICKHOUSE_PASSWORD=
+# CLICKHOUSE_DB=
 
 # Ingestion
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=
@@ -242,7 +253,6 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
 # LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
 ## END Langfuse V3 Ingestion

.github/workflows/_deploy_ecs_service.yml

@@ -58,7 +58,6 @@ jobs:
             --build-arg SENTRY_PROJECT=${{ vars.SENTRY_PROJECT }} \
             .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
-
       - name: Render AWS ECS Task Definition
         id: render-task-definition
         uses: aws-actions/amazon-ecs-render-task-definition@v1

.github/workflows/codespell.yml

@@ -6,6 +6,7 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -10,7 +10,11 @@ on:
   merge_group:
   pull_request:
     branches:
-      - "*"
+      - "**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   lint:
@@ -41,6 +45,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -89,6 +94,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -105,7 +111,8 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
+          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -133,15 +140,18 @@ jobs:
           LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run test-sync
         run: pnpm --filter=web run test-sync
+      - name: run test-client
+        run: pnpm --filter=web run test-client
 
   tests-web-async:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -157,6 +167,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -172,11 +183,11 @@ jobs:
           pnpm install
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
@@ -205,11 +216,12 @@ jobs:
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -220,6 +232,7 @@ jobs:
         with:
           version: 9.5.0
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -240,12 +253,12 @@ jobs:
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status
@@ -279,6 +292,7 @@ jobs:
           cache: "pnpm"
           cache-dependency-path: "pnpm-lock.yaml"
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -311,6 +325,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
@@ -326,18 +341,16 @@ jobs:
       - name: install dependencies
         run: |
           pnpm install
-      - name: Load default env
-        run: |
-          cp .env.dev.example .env
-          echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
-          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
-          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Install golang-migrate for Clickhouse migrations
         run: |
           curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
-
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -439,13 +452,16 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
@@ -464,13 +480,16 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse-worker # GitHub
             langfuse/langfuse-worker # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.github/workflows/snyk.yml

@@ -6,6 +6,11 @@ on:
   push:
     branches:
       - main
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   snyk:
@@ -15,13 +20,13 @@ jobs:
       - name: Build a Docker image
         run: docker compose -f docker-compose.build.yml up -d
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-server)
+      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
         continue-on-error: true
         uses: snyk/actions/docker@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-server
+          image: langfuse-langfuse-web
           args: --file=web/Dockerfile
 
       - name: Upload result to GitHub Code Scanning
@@ -36,7 +41,7 @@ jobs:
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-worker
+          image: langfuse-langfuse-worker
           args: --file=worker/Dockerfile
 
       - name: Upload result to GitHub Code Scanning

.gitignore

@@ -37,7 +37,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env*
 !.env.dev.example
-!.env.local.example
+!.env.dev-azure.example
 !.env.prod.example
 !.env.dev.legacy.example
 

LICENSE

@@ -2,8 +2,8 @@ Copyright (c) 2023--2024 Langfuse GmbH
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and/or "web/src/ee" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
-- All third party components incorporated into the Finto Technologies Software are licensed under the original license provided by the owner of the applicable component.
+- All content that resides under the "ee/", "web/src/ee/", and/or "worker/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All third party components incorporated into the Langfuse Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

README.md

@@ -42,9 +42,7 @@
 
 ## Langfuse Overview
 
-_Unmute video for voice-over_
-
-https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+[![Langfuse Overview Video](https://github.com/user-attachments/assets/3926b288-ff61-4b95-8aa1-45d041c70866)](https://langfuse.com/watch-demo)
 
 ### Develop
 
@@ -191,3 +189,54 @@ You can opt-out by setting `TELEMETRY_ENABLED=false`.
    <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=langfuse/langfuse&type=Date" />
  </picture>
 </a>
+
+### Open Source Projects Using Langfuse
+
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents)):
+
+| Repository                                                                                                                                                                                                                                                          | Stars |
+| :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 54865 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 51531 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/131470832?s=40&v=4" width="20" height="20" alt=""> &nbsp; [lobehub](https://github.com/lobehub) / [lobe-chat](https://github.com/lobehub/lobe-chat)                                           | 49003 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 39093 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37368 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/139558948?s=40&v=4" width="20" height="20" alt=""> &nbsp; [chatchat-space](https://github.com/chatchat-space) / [Langchain-Chatchat](https://github.com/chatchat-space/Langchain-Chatchat)    | 32486 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/128289781?s=40&v=4" width="20" height="20" alt=""> &nbsp; [FlowiseAI](https://github.com/FlowiseAI) / [Flowise](https://github.com/FlowiseAI/Flowise)                                         | 32448 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26931 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/119600397?s=40&v=4" width="20" height="20" alt=""> &nbsp; [twentyhq](https://github.com/twentyhq) / [twenty](https://github.com/twentyhq/twenty)                                              | 24195 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22618 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 15151 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/179202840?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mediar-ai](https://github.com/mediar-ai) / [screenpipe](https://github.com/mediar-ai/screenpipe)                                   | 11037 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/105877416?s=40&v=4" width="20" height="20" alt=""> &nbsp; [formbricks](https://github.com/formbricks) / [formbricks](https://github.com/formbricks/formbricks)                                |  9386 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/76263028?s=40&v=4" width="20" height="20" alt=""> &nbsp; [anthropics](https://github.com/anthropics) / [courses](https://github.com/anthropics/courses)                                       |  8385 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7374 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5391 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/137907881?s=40&v=4" width="20" height="20" alt=""> &nbsp; [promptfoo](https://github.com/promptfoo) / [promptfoo](https://github.com/promptfoo/promptfoo)                                     |  4976 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/157326433?s=40&v=4" width="20" height="20" alt=""> &nbsp; [onlook-dev](https://github.com/onlook-dev) / [onlook](https://github.com/onlook-dev/onlook)                                        |  4141 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2526 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  2061 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1161 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1100 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/18422723?s=40&v=4" width="20" height="20" alt=""> &nbsp; [alishobeiri](https://github.com/alishobeiri) / [thread](https://github.com/alishobeiri/thread)                                      |  1074 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   971 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   823 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   677 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   639 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   514 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   394 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   384 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   370 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   305 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/154247157?s=40&v=4" width="20" height="20" alt=""> &nbsp; [vespperhq](https://github.com/vespperhq) / [vespper](https://github.com/vespperhq/vespper)                                         |   304 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   295 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   291 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/2357342?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dmayboroda](https://github.com/dmayboroda) / [minima](https://github.com/dmayboroda/minima)                                          |   221 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   172 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/148684274?s=40&v=4" width="20" height="20" alt=""> &nbsp; [i-am-alice](https://github.com/i-am-alice) / [3rd-devs](https://github.com/i-am-alice/3rd-devs)                                    |   148 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   138 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/104478511?s=40&v=4" width="20" height="20" alt=""> &nbsp; [babelcloud](https://github.com/babelcloud) / [LLM-RGB](https://github.com/babelcloud/LLM-RGB)                                      |   135 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/15125613?s=40&v=4" width="20" height="20" alt=""> &nbsp; [souzatharsis](https://github.com/souzatharsis) / [tamingLLMs](https://github.com/souzatharsis/tamingLLMs)                           |   129 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169401942?s=40&v=4" width="20" height="20" alt=""> &nbsp; [LibreChat-AI](https://github.com/LibreChat-AI) / [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)                      |   128 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   126 |

docker-compose.build.yml

@@ -1,34 +1,53 @@
-version: "3.5"
-
 services:
-  server:
+  langfuse-web:
     build:
       dockerfile: ./web/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: &langfuse-depends-on
+      postgres:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
     ports:
       - "3000:3000"
-    environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+    environment: &langfuse-web-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      NEXTAUTH_SECRET: mysecret
+      SALT: mysalt
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: http://localhost:3000
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3000/api/public/health"]
@@ -36,26 +55,17 @@ services:
       timeout: 10s
       retries: 3
 
-  worker:
+  langfuse-worker:
     build:
       dockerfile: ./worker/Dockerfile
       context: .
       args:
         - NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${NEXT_PUBLIC_LANGFUSE_CLOUD_REGION}
-    depends_on:
-      - db
-      - redis
+    depends_on: *langfuse-depends-on
     ports:
       - "3030:3030"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - PORT=${PORT:-3030}
-      - REDIS_HOST=${REDIS_HOST:-redis}
-      - REDIS_PORT=${REDIS_PORT:-6379}
-      - REDIS_AUTH=${REDIS_AUTH:-myredissecret}
+      <<: *langfuse-web-env
     restart: always
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:3030/api/health"]
@@ -63,26 +73,85 @@ services:
       timeout: 10s
       retries: 3
 
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
   redis:
-    image: redis:7.2.4
+    image: redis:7
+    restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
-    restart: always
     ports:
       - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
 
-  db:
-    image: postgres
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.v3preview.yml

@@ -1,145 +0,0 @@
-services:
-  langfuse-worker:
-    image: langfuse/langfuse-worker:latest
-    depends_on: &langfuse-depends-on
-      postgres:
-        condition: service_healthy
-      minio:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
-    ports:
-      - "3030:3030"
-    environment: &langfuse-worker-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      SALT: "mysalt"
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
-      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
-      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
-      LANGFUSE_ASYNC_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_INGESTION_PROCESSING:-true}
-      LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:-true}
-      LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: ${LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE:-true}
-      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
-      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
-      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
-      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
-      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
-      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      CLICKHOUSE_MIGRATION_CLUSTER_DISABLED: ${CLICKHOUSE_MIGRATION_CLUSTER_DISABLED:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENABLED: ${LANGFUSE_S3_MEDIA_UPLOAD_ENABLED:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
-      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-us-east-1}
-      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
-      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
-      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
-      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
-      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
-      REDIS_HOST: ${REDIS_HOST:-redis}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
-
-  langfuse-web:
-    image: langfuse/langfuse:latest
-    depends_on: *langfuse-depends-on
-    ports:
-      - "3000:3000"
-    environment:
-      <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
-      NEXTAUTH_SECRET: mysecret
-      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
-      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
-      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
-      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
-      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
-      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
-      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-
-  clickhouse:
-    image: clickhouse/clickhouse-server
-    user: "101:101"
-    container_name: clickhouse
-    hostname: clickhouse
-    environment:
-      CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
-    volumes:
-      - langfuse_clickhouse_data:/var/lib/clickhouse
-      - langfuse_clickhouse_logs:/var/log/clickhouse-server
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    depends_on:
-      - postgres
-
-  minio:
-    image: minio/minio
-    container_name: minio
-    entrypoint: sh
-    # create the 'langfuse' bucket before starting the service
-    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
-    environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
-    ports:
-      - "9090:9000"
-      - "9091:9001"
-    volumes:
-      - langfuse_minio_data:/data
-    healthcheck:
-      test: ["CMD", "mc", "ready", "local"]
-      interval: 1s
-      timeout: 5s
-      retries: 5
-      start_period: 1s
-
-  redis:
-    image: redis:7
-    restart: always
-    command: >
-      --requirepass ${REDIS_AUTH:-myredissecret}
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 3s
-      timeout: 10s
-      retries: 10
-
-  postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
-    restart: always
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 3s
-      timeout: 3s
-      retries: 10
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
-    ports:
-      - 5432:5432
-    volumes:
-      - langfuse_postgres_data:/var/lib/postgresql/data
-
-volumes:
-  langfuse_postgres_data:
-    driver: local
-  langfuse_clickhouse_data:
-    driver: local
-  langfuse_clickhouse_logs:
-    driver: local
-  langfuse_minio_data:
-    driver: local

docker-compose.yml

@@ -1,31 +1,133 @@
 services:
-  langfuse-server:
-    image: langfuse/langfuse:2
-    depends_on:
-      db:
+  langfuse-worker:
+    image: langfuse/langfuse-worker:3
+    restart: always
+    depends_on: &langfuse-depends-on
+      postgres:
         condition: service_healthy
+      minio:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    ports:
+      - "3030:3030"
+    environment: &langfuse-worker-env
+      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
+      SALT: "mysalt"
+      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
+      TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
+      LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
+      LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-auto}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
+      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
+      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_PORT: ${REDIS_PORT:-6379}
+      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
+
+  langfuse-web:
+    image: langfuse/langfuse:3
+    restart: always
+    depends_on: *langfuse-depends-on
     ports:
       - "3000:3000"
     environment:
-      - DATABASE_URL=postgresql://postgres:postgres@db:5432/postgres
-      - NEXTAUTH_SECRET=mysecret
-      - SALT=mysalt
-      - ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 # generate via `openssl rand -hex 32`
-      - NEXTAUTH_URL=http://localhost:3000
-      - TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
-      - LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
-      - LANGFUSE_INIT_ORG_ID=${LANGFUSE_INIT_ORG_ID:-}
-      - LANGFUSE_INIT_ORG_NAME=${LANGFUSE_INIT_ORG_NAME:-}
-      - LANGFUSE_INIT_PROJECT_ID=${LANGFUSE_INIT_PROJECT_ID:-}
-      - LANGFUSE_INIT_PROJECT_NAME=${LANGFUSE_INIT_PROJECT_NAME:-}
-      - LANGFUSE_INIT_PROJECT_PUBLIC_KEY=${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
-      - LANGFUSE_INIT_PROJECT_SECRET_KEY=${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
-      - LANGFUSE_INIT_USER_EMAIL=${LANGFUSE_INIT_USER_EMAIL:-}
-      - LANGFUSE_INIT_USER_NAME=${LANGFUSE_INIT_USER_NAME:-}
-      - LANGFUSE_INIT_USER_PASSWORD=${LANGFUSE_INIT_USER_PASSWORD:-}
+      <<: *langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
+      NEXTAUTH_SECRET: mysecret
+      LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
+      LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
+      LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
+      LANGFUSE_INIT_PROJECT_NAME: ${LANGFUSE_INIT_PROJECT_NAME:-}
+      LANGFUSE_INIT_PROJECT_PUBLIC_KEY: ${LANGFUSE_INIT_PROJECT_PUBLIC_KEY:-}
+      LANGFUSE_INIT_PROJECT_SECRET_KEY: ${LANGFUSE_INIT_PROJECT_SECRET_KEY:-}
+      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
+      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
+      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
+      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
 
-  db:
-    image: postgres
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    restart: always
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:8123/ping || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 1s
+
+  minio:
+    image: minio/minio
+    restart: always
+    container_name: minio
+    entrypoint: sh
+    # create the 'langfuse' bucket before starting the service
+    command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: miniosecret
+    ports:
+      - "9090:9000"
+      - "9091:9001"
+    volumes:
+      - langfuse_minio_data:/data
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 1s
+      timeout: 5s
+      retries: 5
+      start_period: 1s
+
+  redis:
+    image: redis:7
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 3s
+      timeout: 10s
+      retries: 10
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -33,14 +135,20 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
     ports:
       - 5432:5432
     volumes:
-      - database_data:/var/lib/postgresql/data
+      - langfuse_postgres_data:/var/lib/postgresql/data
 
 volumes:
-  database_data:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_minio_data:
     driver: local

ee/package.json

@@ -27,8 +27,8 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +47,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8"
     }
   }
 }

ee/src/env.ts

@@ -1,12 +1,9 @@
 import { z } from "zod";
-import { env as sharedEnv, removeEmptyEnvVariables } from "@langfuse/shared";
+import { removeEmptyEnvVariables } from "@langfuse/shared";
 
 const EnvSchema = z.object({
   NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: z.string().optional(),
   LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
 });
 
-export const env = {
-  ...sharedEnv,
-  ...EnvSchema.parse(removeEmptyEnvVariables(process.env)),
-};
+export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

fern/apis/server/definition/comments.yml

@@ -58,7 +58,7 @@ types:
         docs: The id of the object to attach the comment to. If this does not reference a valid existing object, an error will be thrown.
       content:
         type: string
-        docs: The content of the comment. May include markdown. Currently limited to 500 characters.
+        docs: The content of the comment. May include markdown. Currently limited to 3000 characters.
       authorUserId:
         type: optional<string>
         docs: The id of the user who created the comment.

fern/apis/server/definition/commons.yml

@@ -126,7 +126,7 @@ types:
         docs: The output data of the observation
       usage:
         type: optional<Usage>
-        docs: The usage data of the observation
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The usage data of the observation
       level:
         type: ObservationLevel
         docs: The level of the observation
@@ -139,6 +139,12 @@ types:
       promptId:
         type: optional<string>
         docs: The prompt ID associated with the observation
+      usageDetails:
+        type: optional<map<string, integer>>
+        docs: The usage details of the observation. Key is the name of the usage metric, value is the number of units consumed. The total key is the sum of all (non-total) usage metrics or the total value ingested.
+      costDetails:
+        type: optional<map<string, double>>
+        docs: The cost details of the observation. Key is the name of the cost metric, value is the cost in USD. The total key is the sum of all (non-total) cost metrics or the total value ingested.
 
   ObservationsView:
     extends: Observation
@@ -163,13 +169,13 @@ types:
         docs: The total price in USD.
       calculatedInputCost:
         type: optional<double>
-        docs: The calculated cost of the input in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the input in USD
       calculatedOutputCost:
         type: optional<double>
-        docs: The calculated cost of the output in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated cost of the output in USD
       calculatedTotalCost:
         type: optional<double>
-        docs: The calculated total cost in USD
+        docs: (Deprecated. Use usageDetails and costDetails instead.) The calculated total cost in USD
       latency:
         type: optional<double>
         docs: The latency in seconds.
@@ -178,7 +184,7 @@ types:
         docs: The time to the first token in seconds
 
   Usage:
-    docs: Standard interface for usage and cost
+    docs: (Deprecated. Use usageDetails and costDetails instead.) Standard interface for usage and cost
     properties:
       input:
         docs: Number of input units (e.g. tokens)

fern/apis/server/definition/ingestion.yml

@@ -11,6 +11,7 @@ service:
         Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
 
         Notes:
+        - Introduction to data model: https://langfuse.com/docs/tracing-data-model
         - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
         - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
@@ -126,6 +127,8 @@ types:
       model: optional<string>
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptName: optional<string>
       promptVersion: optional<integer>
 
@@ -137,6 +140,8 @@ types:
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
       promptName: optional<string>
+      usageDetails: optional<UsageDetails>
+      costDetails: optional<map<string, float>>
       promptVersion: optional<integer>
 
   ObservationBody:
@@ -312,3 +317,17 @@ types:
     properties:
       successes: list<IngestionSuccess>
       errors: list<IngestionError>
+
+  OpenAIUsageSchema:
+    properties:
+      prompt_tokens: integer
+      completion_tokens: integer
+      total_tokens: integer
+      prompt_tokens_details: optional<map<string, integer>>
+      completion_tokens_details: optional<map<string, integer>>
+
+  UsageDetails:
+    discriminated: false
+    union:
+      - map<string, integer>
+      - OpenAIUsageSchema

fern/apis/server/definition/media.yml

@@ -99,5 +99,63 @@ types:
         docs: The unique langfuse identifier of a media record
 
   MediaContentType:
-    type: literal<"image/png","image/jpeg","image/jpg","image/webp","audio/mpeg","audio/mp3","audio/wav","text/plain","application/pdf">
+    enum:
+      - value: image/png
+        name: IMAGE_PNG
+      - value: image/jpeg
+        name: IMAGE_JPEG
+      - value: image/jpg
+        name: IMAGE_JPG
+      - value: image/webp
+        name: IMAGE_WEBP
+      - value: image/gif
+        name: IMAGE_GIF
+      - value: image/svg+xml
+        name: IMAGE_SVG_XML
+      - value: image/tiff
+        name: IMAGE_TIFF
+      - value: image/bmp
+        name: IMAGE_BMP
+      - value: audio/mpeg
+        name: AUDIO_MPEG
+      - value: audio/mp3
+        name: AUDIO_MP3
+      - value: audio/wav
+        name: AUDIO_WAV
+      - value: audio/ogg
+        name: AUDIO_OGG
+      - value: audio/oga
+        name: AUDIO_OGA
+      - value: audio/aac
+        name: AUDIO_AAC
+      - value: audio/mp4
+        name: AUDIO_MP4
+      - value: audio/flac
+        name: AUDIO_FLAC
+      - value: video/mp4
+        name: VIDEO_MP4
+      - value: video/webm
+        name: VIDEO_WEBM
+      - value: text/plain
+        name: TEXT_PLAIN
+      - value: text/html
+        name: TEXT_HTML
+      - value: text/css
+        name: TEXT_CSS
+      - value: text/csv
+        name: TEXT_CSV
+      - value: application/pdf
+        name: APPLICATION_PDF
+      - value: application/msword
+        name: APPLICATION_MSWORD
+      - value: application/vnd.ms-excel
+        name: APPLICATION_MS_EXCEL
+      - value: application/zip
+        name: APPLICATION_ZIP
+      - value: application/json
+        name: APPLICATION_JSON
+      - value: application/xml
+        name: APPLICATION_XML
+      - value: application/octet-stream
+        name: APPLICATION_OCTET_STREAM
     docs: The MIME type of the media record

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.92.0",
+  "version": "3.6.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -81,10 +81,11 @@
       }
     }
   },
+  "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8"
     }
-  },
-  "packageManager": "pnpm@9.5.0"
+  }
 }

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/clustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ADD INDEX IF NOT EXISTS idx_user_id user_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces MATERIALIZE INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/down.sh

@@ -18,23 +18,33 @@ then
     exit 1
 fi
 
-# Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
-else
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+else
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
 fi

packages/shared/clickhouse/scripts/drop.sh

@@ -12,11 +12,16 @@ then
     exit 1
 fi
 
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
 fi
 # Execute the drop command
 migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/seed.ts

@@ -2,7 +2,7 @@ import {
   clickhouseClient,
   ObservationRecordReadType,
 } from "@langfuse/shared/src/server";
-import { Prisma, prisma } from "../../src/db";
+import { prisma } from "../../src/db";
 import { redis } from "@langfuse/shared/src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
 import { createDatasets } from "../../prisma/seed";

packages/shared/clickhouse/scripts/up.sh

@@ -18,23 +18,33 @@ then
     exit 1
 fi
 
-# Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
-  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
-  fi
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
 
-  # Execute the up command
-  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
-else
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
+# Construct the database URL
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+else
+if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
 fi

packages/shared/package.json

@@ -59,10 +59,12 @@
     "@aws-sdk/client-s3": "^3.675.0",
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
     "@langchain/anthropic": "^0.3.8",
     "@langchain/aws": "^0.1.2",
     "@langchain/core": "^0.3.18",
+    "@langchain/google-vertexai": "^0.1.3",
     "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "@prisma/client": "^5.22.0",
@@ -78,9 +80,9 @@
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
     "langchain": "^0.3.6",
-    "langfuse-langchain": "3.30.1",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
@@ -119,7 +121,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.0.7",
+      "nanoid": "^3.3.8"
     }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -453,6 +453,7 @@ export type Project = {
     org_id: string;
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
+    deleted_at: Timestamp | null;
     name: string;
 };
 export type ProjectMembership = {
@@ -478,6 +479,13 @@ export type Prompt = {
     tags: Generated<string[]>;
     labels: Generated<string[]>;
 };
+export type QueueBackUp = {
+    id: string;
+    project_id: string | null;
+    queue_name: string;
+    content: unknown;
+    created_at: Generated<Timestamp>;
+};
 export type Score = {
     id: string;
     timestamp: Generated<Timestamp>;
@@ -627,6 +635,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    queue_backups: QueueBackUp;
     score_configs: ScoreConfig;
     scores: Score;
     Session: Session;

packages/shared/prisma/migrations/20241124115100_add_projects_deleted_at/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "projects" ADD COLUMN "deleted_at" TIMESTAMP(3);

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/prisma/migrations/20241206115829_remove_trace_score_observation_constraints/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_output_score_id_fkey";
+
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/migrations/20250108220721_add_queue_backup_table/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "queue_backups" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT,
+    "queue_name" TEXT NOT NULL,
+    "content" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "queue_backups_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_project_id_fkey" FOREIGN KEY ("session_id", "project_id") REFERENCES "trace_sessions"("id", "project_id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250109083346_drop_trace_tracesession_fk/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/schema.prisma

@@ -113,6 +113,7 @@ model Project {
   orgId               String                 @map("org_id")
   createdAt           DateTime               @default(now()) @map("created_at")
   updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
+  deletedAt           DateTime?              @map("deleted_at")
   name                String
   projectMembers      ProjectMembership[]
   organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
@@ -271,7 +272,6 @@ model TraceSession {
   project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
   bookmarked Boolean  @default(false)
   public     Boolean  @default(false)
-  traces     Trace[]
 
   @@id([id, projectId])
   @@index([projectId])
@@ -283,25 +283,24 @@ model TraceSession {
 // Update TraceView below when making changes to this model!
 
 model Trace {
-  id         String        @id @default(cuid())
-  externalId String?       @map("external_id")
-  timestamp  DateTime      @default(now())
+  id         String   @id @default(cuid())
+  externalId String?  @map("external_id")
+  timestamp  DateTime @default(now())
   name       String?
-  userId     String?       @map("user_id")
+  userId     String?  @map("user_id")
   metadata   Json?
   release    String?
   version    String?
-  projectId  String        @map("project_id")
-  project    Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  public     Boolean       @default(false)
-  bookmarked Boolean       @default(false)
-  tags       String[]      @default([])
+  projectId  String   @map("project_id")
+  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  public     Boolean  @default(false)
+  bookmarked Boolean  @default(false)
+  tags       String[] @default([])
   input      Json?
   output     Json?
-  sessionId  String?       @map("session_id")
-  session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
-  createdAt  DateTime      @default(now()) @map("created_at")
-  updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
+  sessionId  String?  @map("session_id")
+  createdAt  DateTime @default(now()) @map("created_at")
+  updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")
 
   @@index([projectId, timestamp])
   @@index([sessionId])
@@ -468,25 +467,24 @@ enum ObservationLevel {
 }
 
 model Score {
-  id            String         @id @default(cuid())
-  timestamp     DateTime       @default(now())
-  projectId     String         @map("project_id")
-  project       Project        @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  id            String        @id @default(cuid())
+  timestamp     DateTime      @default(now())
+  projectId     String        @map("project_id")
+  project       Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
   name          String
   value         Float? // always defined if data type is NUMERIC or BOOLEAN, optional for CATEGORICAL
   source        ScoreSource
-  authorUserId  String?        @map("author_user_id")
+  authorUserId  String?       @map("author_user_id")
   comment       String?
-  traceId       String         @map("trace_id")
-  observationId String?        @map("observation_id")
-  configId      String?        @map("config_id")
-  stringValue   String?        @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
-  queueId       String?        @map("queue_id")
-  createdAt     DateTime       @default(now()) @map("created_at")
-  updatedAt     DateTime       @default(now()) @updatedAt @map("updated_at")
-  dataType      ScoreDataType  @default(NUMERIC) @map("data_type")
-  JobExecution  JobExecution[]
-  scoreConfig   ScoreConfig?   @relation(fields: [configId], references: [id], onDelete: SetNull)
+  traceId       String        @map("trace_id")
+  observationId String?       @map("observation_id")
+  configId      String?       @map("config_id")
+  stringValue   String?       @map("string_value") // always defined if data type is CATEGORICAL or BOOLEAN, null for NUMERIC
+  queueId       String?       @map("queue_id")
+  createdAt     DateTime      @default(now()) @map("created_at")
+  updatedAt     DateTime      @default(now()) @updatedAt @map("updated_at")
+  dataType      ScoreDataType @default(NUMERIC) @map("data_type")
+  scoreConfig   ScoreConfig?  @relation(fields: [configId], references: [id], onDelete: SetNull)
 
   @@unique([id, projectId]) // used for upserts via prisma
   @@index(timestamp)
@@ -900,10 +898,9 @@ model JobExecution {
 
   jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
 
-  jobInputDatasetItemId String?      @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
+  jobInputDatasetItemId String? @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
-  score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
 
   @@index([projectId, status])
   @@index([projectId, id])
@@ -1018,3 +1015,14 @@ model ObservationMedia {
   @@index([projectId, observationId])
   @@map("observation_media")
 }
+
+model QueueBackUp {
+  id        String  @id @default(cuid())
+  projectId String? @map("project_id")
+  queueName String  @map("queue_name")
+  content   Json
+
+  createdAt DateTime @default(now()) @map("created_at")
+
+  @@map("queue_backups")
+}

packages/shared/prisma/seed.ts

@@ -450,6 +450,9 @@ export async function createDatasets(
           Math.random() > 0.3
             ? observations[Math.floor(Math.random() * observations.length)]
             : undefined;
+        if (!sourceObservation) {
+          continue;
+        }
         const datasetItem = await prisma.datasetItem.create({
           data: {
             projectId,
@@ -510,6 +513,9 @@ export async function createDatasets(
               Math.floor(Math.random() * relevantObservations.length)
             ];
 
+          if (!observation) {
+            continue;
+          }
           await prisma.datasetRunItems.create({
             data: {
               projectId,

packages/shared/scripts/prepareClickhouse.ts

@@ -110,7 +110,7 @@ export const prepareClickhouse = async (
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
       toDecimal64(randUniform(0, 2000), 12) AS total_cost,
-      start_time AS completion_start_time,
+      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
       array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
         ",",
       )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,

packages/shared/src/env.ts

@@ -5,6 +5,7 @@ const EnvSchema = z.object({
   NODE_ENV: z
     .enum(["development", "test", "production"])
     .default("development"),
+  NEXTAUTH_URL: z.string().url().optional(),
   REDIS_HOST: z.string().nullish(),
   REDIS_PORT: z.coerce
     .number({
@@ -27,15 +28,21 @@ const EnvSchema = z.object({
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
-  CLICKHOUSE_URL: z.string().url().optional(),
-  CLICKHOUSE_USER: z.string().optional(),
-  CLICKHOUSE_PASSWORD: z.string().optional(),
-  LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
+  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
+  CLICKHOUSE_DB: z.string().default("default"),
+  CLICKHOUSE_USER: z.string(),
+  CLICKHOUSE_PASSWORD: z.string(),
+  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_ASYNC_INGESTION_PROCESSING: z
+  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
+    .enum(["true", "false"])
+    .default("true"),
+  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
     .enum(["true", "false"])
     .default("false"),
+
   LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
     .number()
     .nonnegative()
@@ -48,8 +55,9 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
+  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
+    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
+  }),
   LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
   LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
   LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
@@ -58,6 +66,15 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
+
+  LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
 });
 
-export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));
+export const env: z.infer<typeof EnvSchema> =
+  process.env.DOCKER_BUILD === "1"
+    ? (process.env as any)
+    : EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/comments/types.ts

@@ -1,5 +1,7 @@
 import { z } from "zod";
 
+const MAX_COMMENT_LENGTH = 3000;
+
 const COMMENT_OBJECT_TYPES = [
   "TRACE",
   "OBSERVATION",
@@ -9,7 +11,7 @@ const COMMENT_OBJECT_TYPES = [
 
 export const CreateCommentData = z.object({
   projectId: z.string(),
-  content: z.string().trim().min(1).max(500),
+  content: z.string().trim().min(1).max(MAX_COMMENT_LENGTH),
   objectId: z.string(),
   objectType: z.enum(COMMENT_OBJECT_TYPES),
 });

packages/shared/src/features/entitlements/plans.ts

@@ -3,15 +3,17 @@ export const planLabels = {
   "cloud:hobby": "Hobby",
   "cloud:pro": "Pro",
   "cloud:team": "Team",
-  "self-hosted:enterprise": "Enterprise",
+  "self-hosted:pro": "Pro (self-hosted)",
+  "self-hosted:enterprise": "Enterprise (self-hosted)",
 } as const;
 
 export type Plan = keyof typeof planLabels;
 
 export const plans = Object.keys(planLabels) as Plan[];
 
-// This function is kept here to ensure consistency when updating plan names in the future.
+// These functions are kept here to ensure consistency when updating plan names in the future.
 export const isCloudPlan = (plan: Plan) => plan.startsWith("cloud");
+export const isSelfHostedPlan = (plan: Plan) => plan.startsWith("self-hosted");
 
 export const isPlan = (value: string): value is Plan =>
   plans.includes(value as Plan);

packages/shared/src/features/experiments/utils.ts

@@ -0,0 +1,15 @@
+import { Prisma } from "../../db";
+
+export const datasetItemMatchesVariable = (
+  input: Prisma.JsonValue,
+  variable: string,
+) => {
+  if (
+    input === null ||
+    input === undefined ||
+    typeof input !== "object" ||
+    Array.isArray(input)
+  )
+    return false;
+  return Object.keys(input).includes(variable);
+};

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -6,7 +6,7 @@ import { isPresent } from "../../utils/typeChecks";
 import {
   jsonSchema,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 
 /**
@@ -17,7 +17,7 @@ export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
 
 const validateCategories = (
   categories: ConfigCategory[],
-  ctx: z.RefinementCtx
+  ctx: z.RefinementCtx,
 ) => {
   const uniqueNames = new Set<string>();
   const uniqueValues = new Set<number>();
@@ -94,7 +94,7 @@ const BooleanScoreConfig = z.object({
       return categories.every(
         (category, index) =>
           category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value
+          category.value === expectedCategories[index].value,
       );
     }),
 });
@@ -123,7 +123,7 @@ const ValidatedScoreConfigSchema = z
         minValue: z.undefined().nullish(),
         dataType: z.literal("CATEGORICAL"),
         categories: Categories.superRefine(validateCategories),
-      })
+      }),
     ),
     ScoreConfigBase.merge(BooleanScoreConfig),
   ])
@@ -150,7 +150,7 @@ const ValidatedScoreConfigSchema = z
  */
 export const filterAndValidateDbScoreConfigList = (
   scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): ValidatedScoreConfig[] =>
   scoreConfigs.reduce((acc, ts) => {
     const result = ValidatedScoreConfigSchema.safeParse(ts);
@@ -170,7 +170,7 @@ export const filterAndValidateDbScoreConfigList = (
  * @throws error if score fails validation
  */
 export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType
+  scoreConfig: ScoreConfigDbType,
 ): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
 
 /**
@@ -205,7 +205,7 @@ export const PostScoreConfigBody = z
       z.object({
         dataType: z.literal("BOOLEAN"),
         categories: z.undefined().nullish(),
-      })
+      }),
     ),
   ])
   .superRefine((data, ctx) => {
@@ -227,7 +227,7 @@ export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
 
 // GET /score-configs
 export const GetScoreConfigsQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
 });
 
 export const GetScoreConfigsResponse = z.object({

packages/shared/src/features/scores/scoreTypes.ts

@@ -6,7 +6,7 @@ import { isPresent, stringDateTime } from "../../utils/typeChecks";
 import {
   NonEmptyString,
   paginationMetaResponseZod,
-  paginationZod,
+  publicApiPaginationZod,
 } from "../../utils/zod";
 import { Category as ConfigCategory } from "./scoreConfigTypes";
 
@@ -84,13 +84,13 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
     z.object({
       value: z.number(),
       dataType: z.literal("NUMERIC"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -98,7 +98,7 @@ export const ScoreBodyWithoutConfig = z.discriminatedUnion("dataType", [
         message: "Value must be either 0 or 1",
       }),
       dataType: z.literal("BOOLEAN"),
-    })
+    }),
   ),
 ]);
 
@@ -162,7 +162,7 @@ export const ScorePropsAgainstConfig = z.union([
  */
 export const filterAndValidateDbScoreList = (
   scores: Score[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): APIScore[] =>
   scores.reduce((acc, ts) => {
     const result = APIScoreSchema.safeParse(ts);
@@ -199,14 +199,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       value: z.number(),
       dataType: z.literal("NUMERIC"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.string(),
       dataType: z.literal("CATEGORICAL"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
@@ -216,14 +216,14 @@ export const PostScoresBody = z.discriminatedUnion("dataType", [
       }),
       dataType: z.literal("BOOLEAN"),
       configId: z.string().nullish(),
-    })
+    }),
   ),
   BaseScoreBody.merge(
     z.object({
       value: z.union([z.string(), z.number()]),
       dataType: z.undefined(),
       configId: z.string().nullish(),
-    })
+    }),
   ),
 ]);
 
@@ -231,7 +231,7 @@ export const PostScoresResponse = z.object({ id: z.string() });
 
 // GET /scores
 export const GetScoresQuery = z.object({
-  ...paginationZod,
+  ...publicApiPaginationZod,
   userId: z.string().nullish(),
   dataType: z.enum(ScoreDataType).nullish(),
   configId: z.string().nullish(),
@@ -260,7 +260,7 @@ const LegacyGetScoreResponseDataV1 = z.intersection(
       userId: z.string().nullish(),
       tags: z.array(z.string()).nullish(),
     }),
-  })
+  }),
 );
 export const GetScoresResponse = z.object({
   data: z.array(LegacyGetScoreResponseDataV1),
@@ -269,7 +269,7 @@ export const GetScoresResponse = z.object({
 
 export const legacyFilterAndValidateV1GetScoreList = (
   scores: unknown[],
-  onParseError?: (error: z.ZodError) => void
+  onParseError?: (error: z.ZodError) => void,
 ): z.infer<typeof LegacyGetScoreResponseDataV1>[] =>
   scores.reduce(
     (acc: z.infer<typeof LegacyGetScoreResponseDataV1>[], ts) => {
@@ -282,7 +282,7 @@ export const legacyFilterAndValidateV1GetScoreList = (
       }
       return acc;
     },
-    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[]
+    [] as z.infer<typeof LegacyGetScoreResponseDataV1>[],
   );
 
 // GET /scores/{scoreId}

packages/shared/src/index.ts

@@ -7,15 +7,14 @@ export * from "./interfaces/customLLMProviderConfigSchemas";
 export * from "./tableDefinitions";
 export * from "./types";
 export * from "./tableDefinitions/tracesTable";
-export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
-export { env } from "./env";
 
 // llm api
 export * from "./server/llm/types";
@@ -33,6 +32,9 @@ export * from "./features/scores";
 // comments
 export * from "./features/comments/types";
 
+// experiments
+export * from "./features/experiments/utils";
+
 // export db types only
 export * from "@prisma/client";
 export { type DB } from "../prisma/generated/types";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -10,3 +10,19 @@ export const BedrockCredentialSchema = z
   })
   .optional();
 export type BedrockCredential = z.infer<typeof BedrockCredentialSchema>;
+
+export const GCPServiceAccountKeySchema = z.object({
+  type: z.literal("service_account"),
+  project_id: z.string(),
+  private_key_id: z.string(),
+  private_key: z.string(),
+  client_email: z.string(),
+  client_id: z.string(),
+  auth_uri: z.string(),
+  token_uri: z.string(),
+  auth_provider_x509_cert_url: z.string(),
+  client_x509_cert_url: z.string(),
+});
+
+export type GCPServiceAccountKey = z.infer<typeof GCPServiceAccountKeySchema>;
+export default GCPServiceAccountKeySchema;

packages/shared/src/observationsTable.ts

@@ -112,6 +112,14 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "Model ID",
+    id: "modelId",
+    type: "stringOptions",
+    internal: 'o."internal_model_id"',
+    options: [], // to be added at runtime
+    nullable: true,
+  },
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -186,6 +194,7 @@ export const observationsTableCols: ColumnDefinition[] = [
 // allows for undefined options, to offer filters while options are still loading
 export type ObservationOptions = {
   model: Array<OptionsDefinition>;
+  modelId: Array<OptionsDefinition>;
   name: Array<OptionsDefinition>;
   traceName: Array<OptionsDefinition>;
   scores_avg: Array<string>;
@@ -194,12 +203,15 @@ export type ObservationOptions = {
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions
+  options?: ObservationOptions,
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {
       return { ...col, options: options?.model ?? [] };
     }
+    if (col.id === "modelId") {
+      return { ...col, options: options?.modelId ?? [] };
+    }
     if (col.id === "name") {
       return { ...col, options: options?.name ?? [] };
     }

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/clickhouse/client.ts

@@ -10,7 +10,7 @@ export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
     url: env.CLICKHOUSE_URL,
     username: env.CLICKHOUSE_USER,
     password: env.CLICKHOUSE_PASSWORD,
-    database: "default",
+    database: env.CLICKHOUSE_DB,
     clickhouse_settings: {
       async_insert: 1,
       wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse

packages/shared/src/server/index.ts

@@ -1,4 +1,4 @@
-export * from "./services/S3StorageService";
+export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
@@ -6,6 +6,7 @@ export * from "./services/PromptService";
 export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -21,10 +22,18 @@ export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
+export * from "./redis/cloudUsageMeteringQueue";
+export * from "./redis/getQueue";
+export * from "./redis/traceDelete";
+export * from "./redis/projectDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/postHogIntegrationQueue";
+export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/coreDataS3ExportQueue";
+export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
 export * from "./ingestion/legacy/index";
 export * from "./queues";
@@ -36,3 +45,7 @@ export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
+export * from "./services/sessions-ui-table-service";
+
+// test utils
+export * from "./test-utils";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -136,7 +136,7 @@ export class ObservationProcessor implements EventProcessor {
 
     // Token counts
     const [newInputCount, newOutputCount] =
-      "usage" in this.event.body
+      "usage" in this.event.body || "usageDetails" in this.event.body
         ? await this.calculateTokenCounts(
             apiScope.projectId,
             this.event.body,
@@ -147,8 +147,9 @@ export class ObservationProcessor implements EventProcessor {
         : [undefined, undefined];
 
     const newTotalCount =
-      "usage" in this.event.body
+      "usage" in this.event.body || "usageDetails" in this.event.body
         ? (this.event.body.usage?.total ??
+          this.event.body.usageDetails?.total ??
           (newInputCount != null || newOutputCount != null
             ? (newInputCount ?? 0) + (newOutputCount ?? 0)
             : undefined))
@@ -158,15 +159,24 @@ export class ObservationProcessor implements EventProcessor {
       inputCost:
         "usage" in this.event.body && this.event.body.usage?.inputCost != null // inputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.inputCost)
-          : existingObservation?.inputCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.input != null
+            ? new Decimal(this.event.body.costDetails?.input)
+            : existingObservation?.inputCost,
       outputCost:
         "usage" in this.event.body && this.event.body.usage?.outputCost != null // outputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.outputCost)
-          : existingObservation?.outputCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.output != null
+            ? new Decimal(this.event.body.costDetails?.output)
+            : existingObservation?.outputCost,
       totalCost:
         "usage" in this.event.body && this.event.body.usage?.totalCost != null // totalCost can be explicitly 0. Note only one equal sign to capture null AND undefined
           ? new Decimal(this.event.body.usage?.totalCost)
-          : existingObservation?.totalCost,
+          : "costDetails" in this.event.body &&
+              this.event.body.costDetails?.total != null
+            ? new Decimal(this.event.body.costDetails?.total)
+            : existingObservation?.totalCost,
     };
 
     const tokenCounts = {
@@ -384,7 +394,7 @@ export class ObservationProcessor implements EventProcessor {
     model?: Model,
     existingObservation?: Omit<Observation, "input" | "output">,
   ) {
-    let newPromptTokens = body.usage?.input;
+    let newPromptTokens = body.usage?.input ?? body.usageDetails?.input;
     if (newPromptTokens === undefined && model && model.tokenizerId) {
       if (body.input) {
         newPromptTokens = calculateTokenDelegate({
@@ -409,7 +419,7 @@ export class ObservationProcessor implements EventProcessor {
       }
     }
 
-    let newCompletionTokens = body.usage?.output;
+    let newCompletionTokens = body.usage?.output ?? body.usageDetails?.output;
 
     if (newCompletionTokens === undefined && model && model.tokenizerId) {
       if (body.output) {

packages/shared/src/server/ingestion/legacy/index.ts

@@ -3,13 +3,7 @@ import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
 import { getProcessorForEvent } from "./EventProcessor";
-import { TraceUpsertEventType } from "../../queues";
-import {
-  convertTraceUpsertEventsToRedisEvents,
-  TraceUpsertQueue,
-} from "../../redis/traceUpsert";
 import { ApiAccessScope } from "../../auth/types";
-import { redis } from "../../redis/redis";
 import { backOff } from "exponential-backoff";
 import { Model } from "../../..";
 import { logger } from "../../logger";
@@ -89,7 +83,7 @@ export const handleBatch = async (
 
 async function retry<T>(request: () => Promise<T>): Promise<T> {
   return await backOff(request, {
-    numOfAttempts: env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" ? 5 : 3,
+    numOfAttempts: 5,
     retry: (e: Error, attemptNumber: number) => {
       if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
         logger.info("not retrying auth error");
@@ -163,45 +157,5 @@ export function cleanEvent(obj: unknown): unknown {
   }
 }
 
-export const isNotNullOrUndefined = <T>(
-  val?: T | null,
-): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
-
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
-
-export const addTracesToTraceUpsertQueue = async (
-  batchResults: BatchResult[],
-  projectId: string,
-): Promise<void> => {
-  const traceEvents: TraceUpsertEventType[] = batchResults
-    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
-    .map((result) =>
-      result.result &&
-      typeof result.result === "object" &&
-      "id" in result.result
-        ? // ingestion API only gets traces for one projectId
-          {
-            traceId: result.result.id as string,
-            projectId,
-          }
-        : null,
-    )
-    .filter(isNotNullOrUndefined);
-
-  try {
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
-
-      const queue = TraceUpsertQueue.getInstance();
-      if (!queue) {
-        logger.error("TraceUpsertQueue not initialized");
-        return;
-      }
-
-      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    }
-  } catch (error) {
-    logger.error("Error sending events to worker", error);
-  }
-};

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -22,8 +22,11 @@ import { LegacyIngestionEventType, QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
 import { LegacyIngestionQueue } from "../redis/legacyIngestion";
 import { redis } from "../redis/redis";
-import { S3StorageService } from "../services/S3StorageService";
-import { addTracesToTraceUpsertQueue, handleBatch } from "./legacy";
+import { handleBatch } from "./legacy";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { getProcessorForEvent } from "./legacy/EventProcessor";
 import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
 
@@ -32,11 +35,11 @@ export type TokenCountDelegate = (p: {
   text: unknown;
 }) => number | undefined;
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -153,91 +156,106 @@ export const processEventBatch = async (
    * ASYNC PROCESSING *
    ********************/
   let s3UploadErrored = false;
-  if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-    await instrumentAsync({ name: "s3-upload-events" }, async () => {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      // S3 Event Upload is currently blocking, but non-failing.
-      // If a promise rejects, we log it below, but do not throw an error.
-      // In this case, we upload the full batch into the Redis queue.
-      const results = await Promise.allSettled(
-        Object.keys(sortedBatchByEventBodyId).map(async (id) => {
-          // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
-          // That way we batch updates from the same invocation into a single file and reduce
-          // write operations on S3.
-          const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-          return s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
-            data,
-          );
-        }),
-      );
-      results.forEach((result) => {
-        if (result.status === "rejected") {
-          s3UploadErrored = true;
-          logger.error("Failed to upload event to S3", {
-            error: result.reason,
-          });
-        }
-      });
-    });
-  }
-
-  // Send each event individually to IngestionQueue for new processing
-  if (
-    env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" &&
-    env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" &&
-    env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING === "true" &&
-    redis &&
-    !s3UploadErrored
-  ) {
-    const queue = IngestionQueue.getInstance();
+  await instrumentAsync({ name: "s3-upload-events" }, async () => {
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    // S3 Event Upload is blocking, but non-failing.
+    // If a promise rejects, we log it below, but do not throw an error.
+    // In this case, we upload the full batch into the Redis queue.
     const results = await Promise.allSettled(
-      Object.keys(sortedBatchByEventBodyId).map(async (id) =>
-        queue
-          ? queue.add(
-              QueueJobs.IngestionJob,
-              {
-                id: randomUUID(),
-                timestamp: new Date(),
-                name: QueueJobs.IngestionJob as const,
-                payload: {
-                  data: {
-                    type: sortedBatchByEventBodyId[id].type,
-                    eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
-                  },
-                  authCheck,
-                },
-              },
-              {
-                delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
-              },
-            )
-          : Promise.reject("Failed to instantiate queue"),
-      ),
+      Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+        // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+        // That way we batch updates from the same invocation into a single file and reduce
+        // write operations on S3.
+        const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+        return s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+          data,
+        );
+      }),
     );
     results.forEach((result) => {
       if (result.status === "rejected") {
-        logger.error("Failed to add event to IngestionQueue", {
+        s3UploadErrored = true;
+        logger.error("Failed to upload event to S3", {
           error: result.reason,
         });
       }
     });
+  });
+
+  // This is a workaround to allow us to disable async ingestion processing for SDK CI testing
+  // TODO: remove this block after SDKs are ready for V3 async ingestion processing
+  if (env.LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED === "true") {
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
   }
 
-  // As part of the legacy processing we sent the entire batch to the worker.
-  if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
-    const queue = LegacyIngestionQueue.getInstance();
+  // Send each event individually to IngestionQueue for ClickHouse processing
+  if (env.LANGFUSE_CLICKHOUSE_INGESTION_ENABLED === "true") {
+    if (s3UploadErrored) {
+      throw new Error(
+        "Failed to upload events to blob storage, aborting event processing",
+      );
+    }
 
-    if (queue) {
-      let addToQueueFailed = false;
+    if (redis) {
+      const queue = IngestionQueue.getInstance();
+      await Promise.all(
+        Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+          queue
+            ? queue.add(
+                QueueJobs.IngestionJob,
+                {
+                  id: randomUUID(),
+                  timestamp: new Date(),
+                  name: QueueJobs.IngestionJob as const,
+                  payload: {
+                    data: {
+                      type: sortedBatchByEventBodyId[id].type,
+                      eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                    },
+                    authCheck,
+                  },
+                },
+                {
+                  delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+                },
+              )
+            : Promise.reject("Failed to instantiate queue"),
+        ),
+      );
+      if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED !== "true") {
+        // If postgres ingestion is disabled, we return early
+        return aggregateBatchResult(
+          [...validationErrors, ...authenticationErrors],
+          sortedBatch.map((event) => ({ id: event.id, result: event })),
+          authCheck.scope.projectId,
+        );
+      }
+    }
+  }
 
-      const queuePayload: LegacyIngestionEventType =
-        env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" && !s3UploadErrored
+  if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED === "true") {
+    // As part of the legacy processing we sent the entire batch to the worker.
+    if (redis) {
+      const queue = LegacyIngestionQueue.getInstance();
+
+      if (queue) {
+        let addToQueueFailed = false;
+
+        const queuePayload: LegacyIngestionEventType = !s3UploadErrored
           ? {
               data: Object.keys(sortedBatchByEventBodyId).map((id) => {
                 const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
@@ -252,46 +270,55 @@ export const processEventBatch = async (
             }
           : { data: sortedBatch, authCheck, useS3EventStore: false };
 
-      try {
-        await queue.add(QueueJobs.LegacyIngestionJob, {
-          payload: queuePayload,
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.LegacyIngestionJob as const,
-        });
-      } catch (e: unknown) {
-        logger.warn(
-          "Failed to add batch to queue, falling back to sync processing",
-          e,
-        );
-        addToQueueFailed = true;
-      }
+        try {
+          await queue.add(QueueJobs.LegacyIngestionJob, {
+            payload: queuePayload,
+            id: randomUUID(),
+            timestamp: new Date(),
+            name: QueueJobs.LegacyIngestionJob as const,
+          });
+        } catch (e: unknown) {
+          logger.warn(
+            "Failed to add batch to queue, falling back to sync processing",
+            e,
+          );
+          addToQueueFailed = true;
+        }
 
-      if (!addToQueueFailed) {
-        return aggregateBatchResult(
-          // we are not sending additional server errors to the client in case of early return
-          [...validationErrors, ...authenticationErrors],
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
+        if (!addToQueueFailed) {
+          return aggregateBatchResult(
+            // we are not sending additional server errors to the client in case of early return
+            [...validationErrors, ...authenticationErrors],
+            sortedBatch.map((event) => ({ id: event.id, result: event })),
+            authCheck.scope.projectId,
+          );
+        }
+      } else {
+        logger.warn(
+          "Ingestion queue not initialized, falling back to sync processing",
         );
       }
-    } else {
-      logger.error(
-        "Ingestion queue not initialized, falling back to sync processing",
-      );
     }
+
+    /*******************
+     * SYNC PROCESSING *
+     *******************/
+    const result = await handleBatch(
+      sortedBatch,
+      authCheck,
+      tokenCountDelegate,
+    );
+
+    //  in case we did not return early, we return the result here
+    return aggregateBatchResult(
+      [...validationErrors, ...authenticationErrors, ...result.errors],
+      result.results,
+      authCheck.scope.projectId,
+    );
   }
 
-  /*******************
-   * SYNC PROCESSING *
-   *******************/
-  const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
-
-  await addTracesToTraceUpsertQueue(result.results, authCheck.scope.projectId);
-
-  //  in case we did not return early, we return the result here
-  return aggregateBatchResult(
-    [...validationErrors, ...authenticationErrors, ...result.errors],
-    result.results,
+  throw new Error(
+    "Either Clickhouse or Postgres ingestion (or both) must be enabled",
   );
 };
 
@@ -335,6 +362,7 @@ const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
 export const aggregateBatchResult = (
   errors: Array<{ id: string; error: unknown }>,
   results: Array<{ id: string; result: unknown }>,
+  projectId?: string,
 ) => {
   const returnedErrors: {
     id: string;
@@ -381,7 +409,10 @@ export const aggregateBatchResult = (
 
   if (returnedErrors.length > 0) {
     traceException(errors);
-    logger.error("Error processing events", returnedErrors);
+    logger.error("Error processing events", {
+      errors: returnedErrors,
+      "langfuse.project.id": projectId,
+    });
   }
 
   results.forEach((result) => {

packages/shared/src/server/ingestion/types.ts

@@ -56,10 +56,66 @@ export const usage = MixedUsage.nullish()
   // ensure output is always of new usage model
   .pipe(Usage.nullish());
 
+const RawUsageOrCostDetails = z.record(
+  z.string(),
+  z.number().nonnegative().nullish(),
+);
+
+const OpenAIUsageSchema = z
+  .object({
+    prompt_tokens: z.number().nonnegative(),
+    completion_tokens: z.number().nonnegative(),
+    total_tokens: z.number().nonnegative(),
+    prompt_tokens_details: z.record(z.string(), z.number().nonnegative()),
+    completion_tokens_details: z.record(z.string(), z.number().nonnegative()),
+  })
+  .strict()
+  .transform((v) => {
+    if (!v) return;
+
+    const {
+      prompt_tokens,
+      completion_tokens,
+      total_tokens,
+      prompt_tokens_details,
+      completion_tokens_details,
+    } = v;
+    const result: z.infer<typeof RawUsageOrCostDetails> & {
+      input: number;
+      output: number;
+      total: number;
+    } = {
+      input: prompt_tokens,
+      output: completion_tokens,
+      total: total_tokens,
+    };
+
+    if (prompt_tokens_details) {
+      for (const [key, value] of Object.entries(prompt_tokens_details)) {
+        result[`input_${key}`] = value;
+        result.input = Math.max(result.input - (value ?? 0), 0);
+      }
+    }
+
+    if (completion_tokens_details) {
+      for (const [key, value] of Object.entries(completion_tokens_details)) {
+        result[`output_${key}`] = value;
+        result.output = Math.max(result.output - (value ?? 0), 0);
+      }
+    }
+
+    return result;
+  })
+  .pipe(RawUsageOrCostDetails);
+
+export const UsageOrCostDetails = z
+  .union([OpenAIUsageSchema, RawUsageOrCostDetails])
+  .nullish();
+
 export const TraceBody = z.object({
   id: z.string().nullish(),
   timestamp: stringDateTime,
-  name: z.string().nullish(),
+  name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
@@ -119,6 +175,8 @@ export const CreateGenerationBody = CreateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -147,6 +205,8 @@ export const UpdateGenerationBody = UpdateSpanBody.extend({
     )
     .nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   promptName: z.string().nullish(),
   promptVersion: z.number().int().nullish(),
 }).refine((value) => {
@@ -298,6 +358,8 @@ export const LegacyObservationBody = z.object({
   input: jsonSchema.nullish(),
   output: jsonSchema.nullish(),
   usage: usage,
+  usageDetails: UsageOrCostDetails,
+  costDetails: UsageOrCostDetails,
   metadata: jsonSchema.nullish(),
   parentObservationId: z.string().nullish(),
   level: z.nativeEnum(ObservationLevel).nullish(),

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -17,7 +17,7 @@ type ValidateAndInflateScoreParams = {
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
 ): Promise<Score> {
-  const { body, projectId } = params;
+  const { body, projectId, scoreId } = params;
 
   if (body.configId) {
     const config = await prisma.scoreConfig.findFirst({
@@ -32,10 +32,22 @@ export async function validateAndInflateScore(
         "The configId you provided does not match a valid config in this project",
       );
 
-    validateConfigAgainstBody(body, config as ValidatedScoreConfig);
+    // Override some fields in the score body with config fields
+    // We ignore the set fields in the body
+    const bodyWithConfigOverrides = {
+      ...body,
+      name: config.name,
+    };
+
+    validateConfigAgainstBody(
+      bodyWithConfigOverrides,
+      config as ValidatedScoreConfig,
+    );
 
     return inflateScoreBody({
-      ...params,
+      projectId,
+      scoreId,
+      body: bodyWithConfigOverrides,
       config: config as ValidatedScoreConfig,
     });
   }

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -3,9 +3,11 @@ import type { ZodSchema } from "zod";
 import { CallbackHandler } from "langfuse-langchain";
 
 import { ChatAnthropic } from "@langchain/anthropic";
+import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
   AIMessage,
+  BaseMessage,
   HumanMessage,
   SystemMessage,
 } from "@langchain/core/messages";
@@ -15,7 +17,7 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI } from "@langchain/openai";
-import {
+import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
@@ -31,7 +33,7 @@ import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
 type ProcessTracedEvents = () => Promise<void>;
 
-type TraceParams = {
+export type TraceParams = {
   traceName: string;
   traceId: string;
   projectId: string;
@@ -106,6 +108,7 @@ export async function fetchLLMCompletion(
   if (traceParams) {
     const handler = new CallbackHandler({
       _projectId: traceParams.projectId,
+      _isLocalEventExportEnabled: true,
       tags: traceParams.tags,
     });
 
@@ -129,16 +132,28 @@ export async function fetchLLMCompletion(
 
   finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
 
-  const finalMessages = messages.map((message) => {
-    if (message.role === ChatMessageRole.User)
-      return new HumanMessage(message.content);
-    if (message.role === ChatMessageRole.System)
-      return new SystemMessage(message.content);
+  let finalMessages: BaseMessage[];
+  // VertexAI requires at least 1 user message
+  if (modelParams.adapter === LLMAdapter.VertexAI && messages.length === 1) {
+    finalMessages = [new HumanMessage(messages[0].content)];
+  } else {
+    finalMessages = messages.map((message) => {
+      if (message.role === ChatMessageRole.User)
+        return new HumanMessage(message.content);
+      if (message.role === ChatMessageRole.System)
+        return new SystemMessage(message.content);
 
-    return new AIMessage(message.content);
-  });
+      return new AIMessage(message.content);
+    });
+  }
 
-  let chatModel: ChatOpenAI | ChatAnthropic | ChatBedrockConverse;
+  finalMessages = finalMessages.filter((m) => m.content.length > 0);
+
+  let chatModel:
+    | ChatOpenAI
+    | ChatAnthropic
+    | ChatBedrockConverse
+    | ChatVertexAI;
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
@@ -148,7 +163,7 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
-      clientOptions: { maxRetries },
+      clientOptions: { maxRetries, timeout: 1000 * 60 * 2 }, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
@@ -163,6 +178,7 @@ export async function fetchLLMCompletion(
       configuration: {
         baseURL,
       },
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Azure) {
     chatModel = new ChatOpenAI({
@@ -175,6 +191,7 @@ export async function fetchLLMCompletion(
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
@@ -189,6 +206,24 @@ export async function fetchLLMCompletion(
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
       maxRetries,
+      timeout: 1000 * 60 * 2, // 2 minutes timeout
+    });
+  } else if (modelParams.adapter === LLMAdapter.VertexAI) {
+    const credentials = GCPServiceAccountKeySchema.parse(JSON.parse(apiKey));
+
+    // Requests time out after 60 seconds for both public and private endpoints by default
+    // Reference: https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions#send-request
+    chatModel = new ChatVertexAI({
+      modelName: modelParams.model,
+      temperature: modelParams.temperature,
+      maxOutputTokens: modelParams.max_tokens,
+      topP: modelParams.top_p,
+      callbacks: finalCallbacks,
+      maxRetries,
+      authOptions: {
+        projectId: credentials.project_id,
+        credentials,
+      },
     });
   } else {
     // eslint-disable-next-line no-unused-vars
@@ -196,15 +231,17 @@ export async function fetchLLMCompletion(
     throw new Error("This model provider is not supported.");
   }
 
+  const runConfig = {
+    callbacks: finalCallbacks,
+    runId: traceParams?.traceId,
+    runName: traceParams?.traceName,
+  };
+
   if (params.structuredOutputSchema) {
     return {
       completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
         .withStructuredOutput(params.structuredOutputSchema)
-        .invoke(finalMessages, {
-          callbacks: finalCallbacks,
-          runId: traceParams?.traceId,
-          runName: traceParams?.traceName,
-        }),
+        .invoke(finalMessages, runConfig),
       processTracedEvents,
     };
   }
@@ -234,10 +271,12 @@ export async function fetchLLMCompletion(
         configuration: {
           baseURL,
         },
+        timeout: 1000 * 60 * 2, // 2 minutes timeout
       })
         .pipe(new StringOutputParser())
         .invoke(
           finalMessages.filter((message) => message._getType() !== "system"),
+          runConfig,
         ),
       processTracedEvents,
     };
@@ -247,7 +286,7 @@ export async function fetchLLMCompletion(
     return {
       completion: await chatModel
         .pipe(new BytesOutputParser())
-        .stream(finalMessages),
+        .stream(finalMessages, runConfig),
       processTracedEvents,
     };
   }
@@ -255,7 +294,7 @@ export async function fetchLLMCompletion(
   return {
     completion: await chatModel
       .pipe(new StringOutputParser())
-      .invoke(finalMessages),
+      .invoke(finalMessages, runConfig),
     processTracedEvents,
   };
 }

packages/shared/src/server/llm/types.ts

@@ -16,6 +16,7 @@ export enum LLMAdapter {
   OpenAI = "openai",
   Azure = "azure",
   Bedrock = "bedrock",
+  VertexAI = "vertex-ai",
 }
 
 export enum ChatMessageRole {
@@ -26,6 +27,20 @@ export enum ChatMessageRole {
 
 export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
 
+const ChatMessageSchema = z.object({
+  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+
+export const PromptContentSchema = z.union([
+  ChatMessageListSchema,
+  TextPromptSchema,
+]);
+export type PromptContent = z.infer<typeof PromptContentSchema>;
+
 export type ModelParams = {
   provider: string;
   adapter: LLMAdapter;
@@ -49,6 +64,17 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
+// Experiment config
+export const ExperimentMetadataSchema = z
+  .object({
+    prompt_id: z.string(),
+    provider: z.string(),
+    model: z.string(),
+    model_params: ZodModelConfig,
+  })
+  .strict();
+export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
+
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",
@@ -89,10 +115,18 @@ export const anthropicModels = [
   "claude-instant-1.2",
 ] as const;
 
+export const vertexAIModels = [
+  "gemini-1.5-pro",
+  "gemini-1.5-flash",
+  "gemini-1.0-pro",
+] as const;
+
 export type AnthropicModel = (typeof anthropicModels)[number];
+export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
   [LLMAdapter.Anthropic]: anthropicModels,
   [LLMAdapter.OpenAI]: openAIModels,
+  [LLMAdapter.VertexAI]: vertexAIModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
 } as const;

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -1,10 +1,13 @@
 import { filterOperators } from "../../../interfaces/filters";
 import { clickhouseCompliantRandomCharacters } from "../../repositories";
 
+export type ClickhouseOperator =
+  | (typeof filterOperators)[keyof typeof filterOperators][number]
+  | "!=";
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
-  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  operator: ClickhouseOperator;
   field: string;
 }
 type ClickhouseFilter = {
@@ -69,28 +72,32 @@ export class NumberFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
   public value: number;
-  public operator: (typeof filterOperators)["number"][number];
+  public operator: (typeof filterOperators)["number"][number] | "!=";
+  public clickhouseTypeOverwrite?: string;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["number"][number];
+    operator: (typeof filterOperators)["number"][number] | "!=";
     value: number;
     tablePrefix?: string;
+    clickhouseTypeOverwrite?: string;
   }) {
     this.clickhouseTable = opts.clickhouseTable;
     this.field = opts.field;
     this.value = opts.value;
     this.operator = opts.operator;
     this.tablePrefix = opts.tablePrefix;
+    this.clickhouseTypeOverwrite = opts.clickhouseTypeOverwrite;
   }
 
   apply(): ClickhouseFilter {
     const uid = clickhouseCompliantRandomCharacters();
     const varName = `numberFilter${uid}`;
+    const type = this.clickhouseTypeOverwrite ?? "Decimal64(12)";
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Decimal64(12)}`,
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: ${type}}`,
       params: { [varName]: this.value.toString() },
     };
   }
@@ -154,8 +161,8 @@ export class StringOptionsFilter implements Filter {
     return {
       query:
         this.operator === "any of"
-          ? `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`
-          : `has({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`,
+          ? `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} IN ({${varName}: Array(String)})`
+          : `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} NOT IN ({${varName}: Array(String)})`,
       params: { [varName]: this.values },
     };
   }
@@ -256,7 +263,7 @@ export class ArrayOptionsFilter implements Filter {
         query = `hasAny({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`;
         break;
       case "all of":
-        query = `arrayAll(x -> has({${varName}: Array(String)}, x), ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`;
+        query = `hasAll(${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}, {${varName}: Array(String)}) = True`;
         break;
       default:
         throw new Error(`Unsupported operator: ${this.operator}`);
@@ -300,13 +307,13 @@ export class NumberObjectFilter implements Filter {
   public field: string;
   public key: string;
   public value: number;
-  public operator: (typeof filterOperators)["numberObject"][number];
+  public operator: (typeof filterOperators)["numberObject"][number] | "!=";
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["numberObject"][number];
+    operator: (typeof filterOperators)["numberObject"][number] | "!=";
     key: string;
     value: number;
     tablePrefix?: string;
@@ -364,7 +371,7 @@ export class BooleanFilter implements Filter {
 export class FilterList {
   private filters: Filter[];
 
-  constructor(filters: Filter[]) {
+  constructor(filters: Filter[] = []) {
     this.filters = filters;
   }
 
@@ -376,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  length() {
+    return this.filters.length;
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -67,6 +67,7 @@ export const createFilterFromFilterState = (
           operator: frontEndFilter.operator,
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
+          clickhouseTypeOverwrite: column.clickhouseTypeOverwrite,
         });
       case "arrayOptions":
         return new ArrayOptionsFilter({

packages/shared/src/server/queries/clickhouse-sql/orderby-factory.ts

@@ -3,31 +3,53 @@ import { OrderByState } from "../../../interfaces/orderBy";
 import { UiColumnMapping } from "../../../tableDefinitions";
 import { logger } from "../../logger";
 
+type OrderByStateNotNull = Exclude<OrderByState, null>;
+
 export function orderByToClickhouseSql(
-  orderBy: OrderByState,
+  orderBy: OrderByState | OrderByState[] = [],
   tableColumns: UiColumnMapping[],
 ): string {
-  if (!orderBy) {
+  if (
+    !orderBy ||
+    (Array.isArray(orderBy) && orderBy.filter(Boolean).length === 0)
+  ) {
     return "";
   }
-  // Get column definition to map column to internal name, e.g. "t.id"
-  const col = tableColumns.find(
-    (c) => c.uiTableName === orderBy.column || c.uiTableId === orderBy.column,
-  );
 
-  if (!col) {
-    logger.warn("Invalid order by column", orderBy.column);
-    throw new Error("Invalid order by column: " + orderBy.column);
+  if (!Array.isArray(orderBy)) {
+    orderBy = [orderBy];
+  }
+  // Initialize an array to hold order by clauses
+  const orderByClauses: string[] = [];
+
+  // Loop through each orderBy entry
+  for (const ob of orderBy.filter((o): o is OrderByStateNotNull =>
+    Boolean(o),
+  )) {
+    // Get column definition to map column to internal name, e.g. "t.id"
+    const col = tableColumns.find(
+      (c) => c.uiTableName === ob.column || c.uiTableId === ob.column,
+    );
+
+    if (!col) {
+      logger.warn("Invalid order by column", ob.column);
+      throw new Error("Invalid order by column: " + ob.column);
+    }
+
+    // Assert that ob.order is either "asc" or "desc"
+    const orderByOrder = z.enum(["ASC", "DESC"]);
+    const order = orderByOrder.safeParse(ob.order);
+    if (!order.success) {
+      logger.warn("Invalid order", ob.order);
+      throw new Error("Invalid order: " + ob.order);
+    }
+
+    // Append the order by clause to the array
+    orderByClauses.push(
+      `${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`,
+    );
   }
 
-  // Assert that orderBy.order is either "asc" or "desc"
-  const orderByOrder = z.enum(["ASC", "DESC"]);
-  const order = orderByOrder.safeParse(orderBy.order);
-  if (!order.success) {
-    logger.warn("Invalid order", orderBy.order);
-    throw new Error("Invalid order: " + orderBy.order);
-  }
-
-  // Both column and order are safe, can use raw SQL
-  return `ORDER BY ${col.queryPrefix ? col.queryPrefix + "." : ""}${col.clickhouseSelect} ${order.data}`;
+  // Join all order by clauses with a comma and return
+  return `ORDER BY ${orderByClauses.join(", ")}`;
 }

packages/shared/src/server/queries/createGenerationsQuery.ts

@@ -10,6 +10,8 @@ import { TableFilters } from "./types";
 type AdditionalObservationFields = {
   traceName: string | null;
   traceTags: Array<string>;
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
 };
 
 export type FullObservation = AdditionalObservationFields & ObservationView;

packages/shared/src/server/queries/index.ts

@@ -7,3 +7,17 @@ export {
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
+export {
+  FilterList,
+  StringFilter,
+  DateTimeFilter,
+  StringOptionsFilter,
+  NumberFilter,
+  ArrayOptionsFilter,
+  BooleanFilter,
+  NumberObjectFilter,
+  StringObjectFilter,
+  NullFilter,
+  type ClickhouseOperator,
+} from "./clickhouse-sql/clickhouse-filter";
+export { orderByToClickhouseSql } from "./clickhouse-sql/orderby-factory";

packages/shared/src/server/queues.ts

@@ -1,14 +1,6 @@
 import { z } from "zod";
 import { eventTypes, ingestionBatchEvent } from ".";
 
-export enum EventName {
-  TraceUpsert = "TraceUpsert",
-  BatchExport = "BatchExport",
-  EvaluationExecution = "EvaluationExecution",
-  LegacyIngestion = "LegacyIngestion",
-  CloudUsageMetering = "CloudUsageMetering",
-}
-
 export const LegacyIngestionEventFull = z.object({
   useS3EventStore: z.literal(false),
   data: ingestionBatchEvent,
@@ -62,10 +54,18 @@ export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
 });
-export const TraceUpsertEventSchema = z.object({
+export const TraceQueueEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
+export const TracesQueueEventSchema = z.object({
+  projectId: z.string(),
+  traceIds: z.array(z.string()),
+});
+export const ProjectQueueEventSchema = z.object({
+  projectId: z.string(),
+  orgId: z.string(),
+});
 export const DatasetRunItemUpsertEventSchema = z.object({
   projectId: z.string(),
   datasetItemId: z.string(),
@@ -77,60 +77,86 @@ export const EvalExecutionEvent = z.object({
   jobExecutionId: z.string(),
   delay: z.number().nullish(),
 });
+export const PostHogIntegrationProcessingEventSchema = z.object({
+  projectId: z.string(),
+});
+export const ExperimentCreateEventSchema = z.object({
+  projectId: z.string(),
+  datasetId: z.string(),
+  runId: z.string(),
+  description: z.string().optional(),
+});
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
-export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
+export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
+export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
 >;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
 export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
-
-export const EventBodySchema = z.union([
-  z.object({
-    name: z.literal(EventName.TraceUpsert),
-    payload: z.array(TraceUpsertEventSchema),
-  }),
-  z.object({
-    name: z.literal(EventName.EvaluationExecution),
-    payload: EvalExecutionEvent,
-  }),
-  z.object({
-    name: z.literal(EventName.BatchExport),
-    payload: BatchExportJobSchema,
-  }),
-]);
-export type EventBodyType = z.infer<typeof EventBodySchema>;
+export type ExperimentCreateEventType = z.infer<
+  typeof ExperimentCreateEventSchema
+>;
+export type PostHogIntegrationProcessingEventType = z.infer<
+  typeof PostHogIntegrationProcessingEventSchema
+>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
   DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
+  IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
   LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  ExperimentCreate = "experiment-create-queue",
+  PostHogIntegrationQueue = "posthog-integration-queue",
+  PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
+  CoreDataS3ExportQueue = "core-data-s3-export-queue",
 }
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
+  TraceDelete = "trace-delete",
+  ProjectDelete = "project-delete",
   DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
-  EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
   LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  IngestionSecondaryJob = "secondary-ingestion-job",
+  ExperimentCreateJob = "experiment-create-job",
+  PostHogIntegrationJob = "posthog-integration-job",
+  PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
+  CoreDataS3ExportJob = "core-data-s3-export-job",
 }
 
 export type TQueueJobTypes = {
   [QueueName.TraceUpsert]: {
     timestamp: Date;
     id: string;
-    payload: TraceUpsertEventType;
+    payload: TraceQueueEventType;
     name: QueueJobs.TraceUpsert;
   };
+  [QueueName.TraceDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: TracesQueueEventType | TraceQueueEventType;
+    name: QueueJobs.TraceDelete;
+  };
+  [QueueName.ProjectDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: ProjectQueueEventType;
+    name: QueueJobs.ProjectDelete;
+  };
   [QueueName.DatasetRunItemUpsert]: {
     timestamp: Date;
     id: string;
@@ -161,4 +187,22 @@ export type TQueueJobTypes = {
     payload: IngestionEventQueueType;
     name: QueueJobs.IngestionJob;
   };
+  [QueueName.IngestionSecondaryQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: IngestionEventQueueType;
+    name: QueueJobs.IngestionJob;
+  };
+  [QueueName.ExperimentCreate]: {
+    timestamp: Date;
+    id: string;
+    payload: ExperimentCreateEventType;
+    name: QueueJobs.ExperimentCreateJob;
+  };
+  [QueueName.PostHogIntegrationProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: PostHogIntegrationProcessingEventType;
+    name: QueueJobs.PostHogIntegrationProcessingJob;
+  };
 };

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -0,0 +1,61 @@
+import { Queue } from "bullmq";
+import { env } from "../../env";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class CloudUsageMeteringQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudUsageMeteringQueue.instance) {
+      return CloudUsageMeteringQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudUsageMeteringQueue.instance = newRedis
+      ? new Queue(QueueName.CloudUsageMeteringQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudUsageMeteringQueue.instance?.on("error", (err) => {
+      logger.error("CloudUsageMeteringQueue error", err);
+    });
+
+    if (CloudUsageMeteringQueue.instance) {
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {
+          repeat: { pattern: "5 * * * *" },
+        },
+      );
+
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {},
+      );
+    }
+
+    return CloudUsageMeteringQueue.instance;
+  }
+}

packages/shared/src/server/redis/coreDataS3ExportQueue.ts

@@ -0,0 +1,60 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class CoreDataS3ExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (CoreDataS3ExportQueue.instance) {
+      return CoreDataS3ExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CoreDataS3ExportQueue.instance = newRedis
+      ? new Queue(QueueName.CoreDataS3ExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CoreDataS3ExportQueue.instance?.on("error", (err) => {
+      logger.error("CoreDataS3ExportQueue error", err);
+    });
+
+    if (CoreDataS3ExportQueue.instance) {
+      logger.debug("Scheduling jobs for CoreDataS3ExportQueue");
+      CoreDataS3ExportQueue.instance
+        .add(
+          QueueJobs.CoreDataS3ExportJob,
+          {},
+          {
+            repeat: { pattern: "30 4 * * *" }, // every day at 4:30am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding CoreDataS3ExportJob schedule", err);
+        });
+    }
+
+    return CoreDataS3ExportQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -27,7 +27,8 @@ export class DatasetRunItemUpsertQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 5,
+              delay: 30_000, // 30 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -26,10 +26,10 @@ export class EvalExecutionQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 5000,
+                delay: 1000,
               },
             },
           },

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class ExperimentCreateQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null {
+    if (ExperimentCreateQueue.instance) return ExperimentCreateQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ExperimentCreateQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ExperimentCreate]>(
+          QueueName.ExperimentCreate,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 10,
+              backoff: {
+                type: "exponential",
+                delay: 1000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ExperimentCreateQueue.instance?.on("error", (err) => {
+      logger.error("ExperimentCreateQueue error", err);
+    });
+
+    return ExperimentCreateQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -0,0 +1,51 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { BatchExportQueue } from "./batchExport";
+import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
+import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
+import { EvalExecutionQueue } from "./evalExecutionQueue";
+import { ExperimentCreateQueue } from "./experimentCreateQueue";
+import { IngestionQueue, SecondaryIngestionQueue } from "./ingestionQueue";
+import { LegacyIngestionQueue } from "./legacyIngestion";
+import { TraceUpsertQueue } from "./traceUpsert";
+import { TraceDeleteQueue } from "./traceDelete";
+import { ProjectDeleteQueue } from "./projectDelete";
+import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
+import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
+import { CoreDataS3ExportQueue } from "./coreDataS3ExportQueue";
+
+export function getQueue(queueName: QueueName): Queue | null {
+  switch (queueName) {
+    case QueueName.LegacyIngestionQueue:
+      return LegacyIngestionQueue.getInstance();
+    case QueueName.BatchExport:
+      return BatchExportQueue.getInstance();
+    case QueueName.CloudUsageMeteringQueue:
+      return CloudUsageMeteringQueue.getInstance();
+    case QueueName.DatasetRunItemUpsert:
+      return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.EvaluationExecution:
+      return EvalExecutionQueue.getInstance();
+    case QueueName.ExperimentCreate:
+      return ExperimentCreateQueue.getInstance();
+    case QueueName.TraceUpsert:
+      return TraceUpsertQueue.getInstance();
+    case QueueName.TraceDelete:
+      return TraceDeleteQueue.getInstance();
+    case QueueName.IngestionQueue:
+      return IngestionQueue.getInstance();
+    case QueueName.ProjectDelete:
+      return ProjectDeleteQueue.getInstance();
+    case QueueName.PostHogIntegrationQueue:
+      return PostHogIntegrationQueue.getInstance();
+    case QueueName.PostHogIntegrationProcessingQueue:
+      return PostHogIntegrationProcessingQueue.getInstance();
+    case QueueName.IngestionSecondaryQueue:
+      return SecondaryIngestionQueue.getInstance();
+    case QueueName.CoreDataS3ExportQueue:
+      return CoreDataS3ExportQueue.getInstance();
+    default:
+      const exhaustiveCheckDefault: never = queueName;
+      throw new Error(`Queue ${queueName} not found`);
+  }
+}

packages/shared/src/server/redis/ingestionQueue.ts

@@ -43,3 +43,45 @@ export class IngestionQueue {
     return IngestionQueue.instance;
   }
 }
+
+export class SecondaryIngestionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null {
+    if (SecondaryIngestionQueue.instance)
+      return SecondaryIngestionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    SecondaryIngestionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(
+          QueueName.IngestionSecondaryQueue,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    SecondaryIngestionQueue.instance?.on("error", (err) => {
+      logger.error("SecondaryIngestionQueue error", err);
+    });
+
+    return SecondaryIngestionQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationProcessingQueue.instance) {
+      return PostHogIntegrationProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100_000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationProcessingQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationProcessingQueue error", err);
+    });
+
+    return PostHogIntegrationProcessingQueue.instance;
+  }
+}

packages/shared/src/server/redis/postHogIntegrationQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class PostHogIntegrationQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (PostHogIntegrationQueue.instance) {
+      return PostHogIntegrationQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    PostHogIntegrationQueue.instance = newRedis
+      ? new Queue(QueueName.PostHogIntegrationQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    PostHogIntegrationQueue.instance?.on("error", (err) => {
+      logger.error("PostHogIntegrationQueue error", err);
+    });
+
+    if (PostHogIntegrationQueue.instance) {
+      logger.debug("Scheduling jobs for PostHogIntegrationQueue");
+      PostHogIntegrationQueue.instance
+        .add(
+          QueueJobs.PostHogIntegrationJob,
+          {},
+          {
+            repeat: { pattern: "30 * * * *" }, // every hour at 30 minutes past
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding PostHogIntegrationJob schedule", err);
+        });
+    }
+
+    return PostHogIntegrationQueue.instance;
+  }
+}

packages/shared/src/server/redis/projectDelete.ts

@@ -0,0 +1,45 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class ProjectDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ProjectDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ProjectDelete]
+  > | null {
+    if (ProjectDeleteQueue.instance) return ProjectDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ProjectDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ProjectDelete]>(
+          QueueName.ProjectDelete,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ProjectDeleteQueue.instance?.on("error", (err) => {
+      logger.error("ProjectDeleteQueue error", err);
+    });
+
+    return ProjectDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/traceDelete.ts

@@ -0,0 +1,44 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class TraceDeleteQueue {
+  private static instance: Queue<TQueueJobTypes[QueueName.TraceDelete]> | null =
+    null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.TraceDelete]
+  > | null {
+    if (TraceDeleteQueue.instance) return TraceDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    TraceDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.TraceDelete]>(
+          QueueName.TraceDelete,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    TraceDeleteQueue.instance?.on("error", (err) => {
+      logger.error("TraceDeleteQueue error", err);
+    });
+
+    return TraceDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -1,10 +1,4 @@
-import { randomUUID } from "crypto";
-import {
-  QueueJobs,
-  QueueName,
-  TQueueJobTypes,
-  TraceUpsertEventType,
-} from "../queues";
+import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";
@@ -32,6 +26,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
+              delay: 15_000, // 15 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,
@@ -48,43 +43,3 @@ export class TraceUpsertQueue {
     return TraceUpsertQueue.instance;
   }
 }
-
-export function convertTraceUpsertEventsToRedisEvents(
-  events: TraceUpsertEventType[],
-) {
-  const uniqueTracesPerProject = events.reduce((acc, event) => {
-    if (!acc.get(event.projectId)) {
-      acc.set(event.projectId, new Set());
-    }
-    acc.get(event.projectId)?.add(event.traceId);
-    return acc;
-  }, new Map<string, Set<string>>());
-
-  return [...uniqueTracesPerProject.entries()]
-    .map((tracesPerProject) => {
-      const [projectId, traceIds] = tracesPerProject;
-
-      return [...traceIds].map((traceId) => ({
-        name: QueueJobs.TraceUpsert,
-        data: {
-          payload: {
-            projectId,
-            traceId,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.TraceUpsert as const,
-        },
-        opts: {
-          removeOnFail: 1_000,
-          removeOnComplete: true,
-          attempts: 5,
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-        },
-      }));
-    })
-    .flat();
-}

packages/shared/src/server/repositories/clickhouse.ts

@@ -5,16 +5,19 @@ import {
 } from "../clickhouse/client";
 import { logger } from "../logger";
 import { instrumentAsync } from "../instrumentation";
-import { S3StorageService } from "../services/S3StorageService";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -29,7 +32,7 @@ const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces"; // TODO: Modify eventType logic to support more tables going forward
+  table: "scores" | "traces" | "observations";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>;
 }): Promise<void> {
@@ -37,34 +40,31 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // drop trailing s and pretend it's always a create.
-    // Only applicable to scores and traces.
-    const eventType = `${opts.table.slice(0, -1)}-create`;
-
-    // If event upload is enabled, we store all rows in S3 to have a backup
-    if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
-      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
-        throw new Error("S3 event store is enabled but no bucket is set");
-      }
-      const s3Client = getS3StorageServiceClient(
-        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-      );
-      await Promise.all(
-        opts.records.map((record) => {
-          s3Client.uploadJson(
-            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
-            [
-              {
-                id: randomUUID(),
-                timestamp: new Date().toISOString(),
-                type: eventType,
-                body: opts.eventBodyMapper(record),
-              },
-            ],
-          );
-        }),
-      );
-    }
+    const s3Client = getS3StorageServiceClient(
+      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+    );
+    await Promise.all(
+      opts.records.map((record) => {
+        // drop trailing s and pretend it's always a create.
+        // Only applicable to scores and traces.
+        let eventType = `${opts.table.slice(0, -1)}-create`;
+        if (opts.table === "observations") {
+          // @ts-ignore - If it's an observation we now that `type` is a string
+          eventType = `${record["type"].toLowerCase()}-create`;
+        }
+        s3Client.uploadJson(
+          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+          [
+            {
+              id: randomUUID(),
+              timestamp: new Date().toISOString(),
+              type: eventType,
+              body: opts.eventBodyMapper(record),
+            },
+          ],
+        );
+      }),
+    );
 
     const res = await clickhouseClient().insert({
       table: opts.table,

packages/shared/src/server/repositories/dashboards.ts

@@ -1,4 +1,7 @@
-import { queryClickhouse } from "./clickhouse";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "./clickhouse";
 import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
 import { FilterState } from "../../types";
 import {
@@ -200,8 +203,8 @@ export const getObservationUsageByTime = async (
   const query = `
     SELECT 
       ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
-      sumMap(usage_details)['total'] as sum_usage_details,
-      sumMap(cost_details)['total'] as sum_cost_details,
+      sumMap(usage_details) as units,
+      sumMap(cost_details) as cost, 
       provided_model_name
     FROM observations o FINAL
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -214,8 +217,8 @@ export const getObservationUsageByTime = async (
 
   const result = await queryClickhouse<{
     start_time: string;
-    sum_usage_details: string;
-    sum_cost_details: number;
+    units: Record<string, number>;
+    cost: Record<string, number>;
     provided_model_name: string;
   }>({
     query,
@@ -230,8 +233,18 @@ export const getObservationUsageByTime = async (
 
   return result.map((row) => ({
     start_time: new Date(row.start_time),
-    sum_usage_details: Number(row.sum_usage_details),
-    sum_cost_details: row.sum_cost_details,
+    units: Object.fromEntries(
+      Object.entries(row.units ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
+    cost: Object.fromEntries(
+      Object.entries(row.cost ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
     provided_model_name: row.provided_model_name,
   }));
 };
@@ -405,7 +418,7 @@ export const getObservationLatencies = async (
   // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
     SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles,
       name
     FROM observations o
     ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -452,7 +465,7 @@ export const getTracesLatencies = async (
       select o.trace_id,
              t.name,
              o.project_id,
-             date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
+             date_diff('millisecond', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
       FROM traces t 
       JOIN observations o
       ON o.trace_id = t.id AND o.project_id = t.project_id
@@ -508,7 +521,7 @@ export const getModelLatenciesOverTime = async (
   SELECT 
     ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
     provided_model_name,
-    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles
   FROM observations o
   ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
@@ -616,6 +629,50 @@ export const getCategoricalScoreTimeSeries = async (
   });
 };
 
+export const getObservationsStatusTimeSeries = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc | undefined,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+    SELECT 
+      ${groupBy ? selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket") + ", " : ""}
+      count(*) as observation_count,
+      level as level
+    FROM observations o
+    ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    WHERE project_id = {projectId: String}
+    AND o.level IS NOT NULL
+    AND ${chFilterRes?.query}
+    GROUP BY level ${groupBy ? ", start_time_bucket" : ""}
+    ${groupBy ? orderByTimeSeries(groupBy, "start_time_bucket") : ""}
+  `;
+
+  const result = await queryClickhouse<{
+    start_time_bucket?: string;
+    observation_count: string;
+    level: string;
+  }>({
+    query,
+    params: { projectId, ...(chFilterRes ? chFilterRes.params : {}) },
+  });
+
+  return result.map((row) => ({
+    start_time_bucket: row.start_time_bucket
+      ? parseClickhouseUTCDateTimeFormat(row.start_time_bucket)
+      : undefined,
+    count: Number(row.observation_count),
+    level: row.level,
+  }));
+};
+
 const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
   let interval;
   switch (dateTrunc) {

packages/shared/src/server/repositories/definitions.ts

@@ -129,8 +129,8 @@ export const scoreRecordBaseSchema = z.object({
   project_id: z.string(),
   trace_id: z.string(),
   observation_id: z.string().nullish(),
-  name: z.string().nullish(),
-  value: z.union([z.number(), z.string()]).nullish(),
+  name: z.string(),
+  value: z.number().nullish(),
   source: z.string(),
   comment: z.string().nullish(),
   author_user_id: z.string().nullish(),

packages/shared/src/server/repositories/index.ts

@@ -7,3 +7,5 @@ export * from "./traces_converters";
 export * from "./scores_converters";
 export * from "./observations_converters";
 export * from "./clickhouse";
+export * from "./constants";
+export * from "./trace-sessions";

packages/shared/src/server/repositories/observations.ts

@@ -2,6 +2,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  upsertClickhouse,
 } from "./clickhouse";
 import { ObservationLevel } from "@prisma/client";
 import { logger } from "../logger";
@@ -37,6 +38,59 @@ import {
   OBSERVATIONS_TO_TRACE_INTERVAL,
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
+import { env } from "../../env";
+
+export const checkObservationExists = async (
+  projectId: string,
+  id: string,
+  startTime: Date | undefined,
+): Promise<boolean> => {
+  const query = `
+    SELECT id, project_id
+    FROM observations o
+    WHERE project_id = {projectId: String}
+    AND id = {id: String}
+    ${startTime ? `AND start_time >= {startTime: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    ORDER BY event_ts DESC
+    LIMIT 1 BY id, project_id
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
+
+/**
+ * Accepts a trace in a Clickhouse-ready format.
+ * id, project_id, and timestamp must always be provided.
+ */
+export const upsertObservation = async (
+  observation: Partial<ObservationRecordReadType>,
+) => {
+  if (
+    !["id", "project_id", "start_time", "type"].every(
+      (key) => key in observation,
+    )
+  ) {
+    throw new Error(
+      "Identifier fields must be provided to upsert Observation.",
+    );
+  }
+  await upsertClickhouse({
+    table: "observations",
+    records: [observation as ObservationRecordReadType],
+    eventBodyMapper: convertObservation,
+  });
+};
 
 export const getObservationsViewForTrace = async (
   traceId: string,
@@ -94,15 +148,76 @@ export const getObservationsViewForTrace = async (
   return records.map(convertObservationToView);
 };
 
+export const getObservationForTraceIdByName = async (
+  traceId: string,
+  projectId: string,
+  name: string,
+  timestamp?: Date,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations 
+  WHERE trace_id = {traceId: String}
+  AND project_id = {projectId: String}
+  AND name = {name: String}
+   ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  ORDER BY event_ts DESC
+  LIMIT 1 BY id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: {
+      traceId,
+      projectId,
+      name,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return records.map(convertObservationToView);
+};
+
 export const getObservationById = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const records = await getObservationByIdInternal(
     id,
     projectId,
     fetchWithInputOutput,
+    startTime,
   );
   const mapped = records.map(convertObservation);
 
@@ -199,6 +314,7 @@ const getObservationByIdInternal = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+  startTime?: Date,
 ) => {
   const query = `
   SELECT
@@ -233,11 +349,18 @@ const getObservationByIdInternal = async (
   FROM observations
   WHERE id = {id: String}
   AND project_id = {projectId: String}
+  ${startTime ? `AND start_time = {startTime: DateTime64(3)}` : ""}
   ORDER BY event_ts desc
   LIMIT 1 by id, project_id`;
   return await queryClickhouse<ObservationRecordReadType>({
     query,
-    params: { id, projectId },
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
   });
 };
 
@@ -277,7 +400,7 @@ export type ObservationsTableQueryResult = ObservationRecordReadType & {
 export const getObservationsTableCount = async (opts: ObservationTableQuery) =>
   getObservationsTableInternal<TableCount>({
     ...opts,
-    select: "count(*) as count",
+    select: "count",
   });
 
 export type ObservationsTableRow = Omit<
@@ -295,29 +418,7 @@ export const getObservationsTable = async (
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const traces = await getTracesByIds(
@@ -352,29 +453,7 @@ export const getObservationsTableWithModelData = async (
     >
   >({
     ...opts,
-    select: `
-        o.id as id,
-        o.name as name,
-        o."model_parameters" as model_parameters,
-        o.start_time as "start_time",
-        o.end_time as "end_time",
-        o.trace_id as "trace_id",
-        o.completion_start_time as "completion_start_time",
-        o.provided_usage_details as "provided_usage_details",
-        o.usage_details as "usage_details",
-        o.provided_cost_details as "provided_cost_details",
-        o.cost_details as "cost_details",
-        o.level as level,
-        o.status_message as "status_message",
-        o.version as version,
-        o.parent_observation_id as "parent_observation_id",
-        o.created_at as "created_at",
-        o.updated_at as "updated_at",
-        o.provided_model_name as "provided_model_name",
-        o.total_cost as "total_cost",
-        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+    select: "rows",
   });
 
   const uniqueModels: string[] = Array.from(
@@ -431,17 +510,48 @@ export const getObservationsTableWithModelData = async (
 };
 
 const getObservationsTableInternal = async <T>(
-  opts: ObservationTableQuery & { select: string },
+  opts: ObservationTableQuery & { select: "count" | "rows" },
 ): Promise<Array<T>> => {
+  const select =
+    opts.select === "count"
+      ? "count(*) as count"
+      : `
+        o.id as id,
+        o.project_id as "project_id",
+        o.name as name,
+        o."model_parameters" as model_parameters,
+        o.start_time as "start_time",
+        o.end_time as "end_time",
+        o.trace_id as "trace_id",
+        o.completion_start_time as "completion_start_time",
+        o.provided_usage_details as "provided_usage_details",
+        o.usage_details as "usage_details",
+        o.provided_cost_details as "provided_cost_details",
+        o.cost_details as "cost_details",
+        o.level as level,
+        o.status_message as "status_message",
+        o.version as version,
+        o.parent_observation_id as "parent_observation_id",
+        o.created_at as "created_at",
+        o.updated_at as "updated_at",
+        o.provided_model_name as "provided_model_name",
+        o.total_cost as "total_cost",
+        o.prompt_id as "prompt_id",
+        o.prompt_name as "prompt_name",
+        o.prompt_version as "prompt_version",
+        internal_model_id as "internal_model_id",
+        if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time)) as "time_to_first_token"`;
+
   const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
     opts;
 
   const selectString = selectIOAndMetadata
     ? `
-    ${opts.select},
+    ${select},
     ${selectIOAndMetadata ? `o.input, o.output, o.metadata` : ""}
   `
-    : opts.select;
+    : select;
 
   const scoresFilter = new FilterList([
     new StringFilter({
@@ -468,6 +578,10 @@ const getObservationsTableInternal = async <T>(
         .includes(f.column),
   );
 
+  const hasScoresFilter = filter.some(
+    (f) => f.column === "Scores" || f.column === "scores",
+  );
+
   const orderByTraces = opts.orderBy
     ? observationsTableTraceUiColumnDefinitions
         .map((c) => c.uiTableId)
@@ -538,63 +652,58 @@ const getObservationsTableInternal = async <T>(
       observation_id
   )`;
 
-  if (traceTableFilter.length > 0 || orderByTraces) {
-    // joins with traces are very expensive. We need to filter by time as well.
-    // We assume that a trace has to have been within the last 2 days to be relevant.
+  // if we have default ordering by time, we order by toDate(o.start_time) first and then by
+  // o.start_time. This way, clickhouse is able to read more efficiently directly from disk without ordering
+  const newDefaultOrder =
+    orderBy?.column === "startTime"
+      ? [{ column: "order_by_date", order: orderBy.order }, orderBy]
+      : [orderBy ?? null];
 
-    const query = `
+  const chOrderBy = orderByToClickhouseSql(newDefaultOrder, [
+    ...observationsTableUiColumnDefinitions,
+    {
+      uiTableName: "order_by_date",
+      uiTableId: "order_by_date",
+      clickhouseTableName: "observation",
+      clickhouseSelect: "toDate(o.start_time)",
+    },
+  ]);
+
+  // joins with traces are very expensive. We need to filter by time as well.
+  // We assume that a trace has to have been within the last 2 days to be relevant.
+
+  const query = `
       ${scoresCte}
       SELECT
        ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
+      FROM observations o 
+        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
+        ${hasScoresFilter ? `LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
         AND o.type = 'GENERATION'
-        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
+      ${chOrderBy}
+      ${opts.select === "rows" ? "LIMIT 1 BY o.id, o.project_id" : ""}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
 
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-        ...(timeFilter
-          ? {
-              tracesTimestampFilter: convertDateToClickhouseDateTime(
-                timeFilter.value as Date,
-              ),
-            }
-          : {}),
-        ...search.params,
-      },
-    });
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...appliedScoresFilter.params,
+      ...appliedObservationsFilter.params,
+      ...(timeFilter
+        ? {
+            tracesTimestampFilter: convertDateToClickhouseDateTime(
+              timeFilter.value as Date,
+            ),
+          }
+        : {}),
+      ...search.params,
+    },
+  });
 
-    return res;
-  } else {
-    // we query by T, which could also be {count: string}.
-    const query = `
-      ${scoresCte}
-      SELECT
-       ${selectString}
-      FROM observations o FINAL 
-        LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
-      WHERE ${appliedObservationsFilter.query}
-      ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
-      ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
-
-    const res = await queryClickhouse<T>({
-      query,
-      params: {
-        ...appliedScoresFilter.params,
-        ...appliedObservationsFilter.params,
-      },
-    });
-
-    return res;
-  }
+  return res;
 };
 
 export const getObservationsGroupedByModel = async (
@@ -641,6 +750,50 @@ export const getObservationsGroupedByModel = async (
   return res.map((r) => ({ model: r.name }));
 };
 
+export const getObservationsGroupedByModelId = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const observationsFilter = new FilterList([
+    new StringFilter({
+      clickhouseTable: "observations",
+      field: "project_id",
+      operator: "=",
+      value: projectId,
+      tablePrefix: "o",
+    }),
+  ]);
+
+  observationsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      observationsTableUiColumnDefinitions,
+    ),
+  );
+
+  const appliedObservationsFilter = observationsFilter.apply();
+
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
+  const query = `
+    SELECT o.internal_model_id as modelId
+    FROM observations o
+    WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
+    GROUP BY o.internal_model_id
+    ORDER BY count() DESC
+    LIMIT 1000;
+  `;
+
+  const res = await queryClickhouse<{ modelId: string }>({
+    query,
+    params: {
+      ...appliedObservationsFilter.params,
+    },
+  });
+  return res.map((r) => ({ modelId: r.modelId }));
+};
+
 export const getObservationsGroupedByName = async (
   projectId: string,
   filter: FilterState,
@@ -795,6 +948,25 @@ export const deleteObservationsByTraceIds = async (
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteObservationsByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -838,7 +1010,7 @@ export const getObservationMetricsForPrompts = async (
                   end_time,
                   usage_details,
                   cost_details,
-                  dateDiff('milliseconds', start_time, end_time) AS latency_ms
+                  dateDiff('millisecond', start_time, end_time) AS latency_ms
               FROM observations
               FINAL
               WHERE (type = 'GENERATION') 
@@ -852,8 +1024,8 @@ export const getObservationMetricsForPrompts = async (
           prompt_version,
           min(start_time) AS first_observation,
           max(start_time) AS last_observation,
-          medianExact(usage_details['input']) AS median_input_usage,
-          medianExact(usage_details['output']) AS median_output_usage,
+          medianExact(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, usage_details)))) AS median_input_usage,
+          medianExact(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details)))) AS median_output_usage,
           medianExact(cost_details['total']) AS median_total_cost,
           medianExact(latency_ms) AS median_latency_ms
       FROM latencies
@@ -901,7 +1073,7 @@ export const getLatencyAndTotalCostForObservations = async (
     SELECT
         id,
         cost_details['total'] AS total_cost,
-        dateDiff('milliseconds', start_time, end_time) AS latency_ms
+        dateDiff('millisecond', start_time, end_time) AS latency_ms
     FROM observations FINAL 
     WHERE project_id = {projectId: String} 
     AND id IN ({observationIds: Array(String)})
@@ -933,7 +1105,7 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     SELECT
         trace_id,
         sumMap(cost_details)['total'] AS total_cost,
-        dateDiff('milliseconds', min(start_time), max(end_time)) AS latency_ms
+        dateDiff('millisecond', min(start_time), max(end_time)) AS latency_ms
     FROM observations FINAL
     WHERE project_id = {projectId: String} 
     AND trace_id IN ({traceIds: Array(String)})
@@ -957,3 +1129,165 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     latency: Number(r.latency_ms) / 1000,
   }));
 };
+
+export const getObservationCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM observations
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
+export const getObservationCountOfProjectsSinceCreationDate = async ({
+  projectIds,
+  start,
+}: {
+  projectIds: string[];
+  start: Date;
+}) => {
+  const query = `
+    SELECT 
+      count(*) as count
+    FROM observations
+    WHERE project_id IN ({projectIds: Array(String)})
+    AND created_at >= {start: DateTime64(3)}
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectIds,
+      start: convertDateToClickhouseDateTime(start),
+    },
+  });
+
+  return Number(rows[0]?.count ?? 0);
+};
+
+export const getTraceIdsForObservations = async (
+  projectId: string,
+  observationIds: string[],
+) => {
+  const query = `
+    SELECT 
+      trace_id,
+      id
+    FROM observations
+    WHERE project_id = {projectId: String}
+    AND id IN ({observationIds: Array(String)})
+  `;
+
+  const rows = await queryClickhouse<{ id: string; trace_id: string }>({
+    query,
+    params: {
+      projectId,
+      observationIds,
+    },
+  });
+
+  return rows.map((row) => ({
+    id: row.id,
+    traceId: row.trace_id,
+  }));
+};
+
+export const getGenerationsForPostHog = async (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) => {
+  const query = `
+    SELECT
+      o.name as name,
+      o.start_time as start_time,
+      o.id as id,
+      o.total_cost as total_cost,
+      if(isNull(completion_start_time), NULL, date_diff('millisecond', start_time, completion_start_time)) as time_to_first_token,
+      o.usage_details['total'] as input_tokens,
+      o.usage_details['output'] as output_tokens,
+      o.cost_details['total'] as total_tokens,
+      o.project_id as project_id,
+      if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000) as latency,
+      o.provided_model_name as model,
+      o.level as level,
+      o.version as version,
+      t.id as trace_id,
+      t.name as trace_name,
+      t.session_id as trace_session_id,
+      t.user_id as trace_user_id,
+      t.release as trace_release,
+      t.tags as trace_tags,
+      t.metadata['$posthog_session_id'] as posthog_session_id
+    FROM observations o FINAL
+    LEFT JOIN traces t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
+    WHERE o.project_id = {projectId: String}
+    AND t.project_id = {projectId: String}
+    AND o.start_time >= {minTimestamp: DateTime64(3)}
+    AND o.start_time <= {maxTimestamp: DateTime64(3)}
+    AND t.timestamp >= {minTimestamp: DateTime64(3)} - INTERVAL 7 DAY
+    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
+    AND o.type = 'GENERATION'
+  `;
+
+  const records = await queryClickhouse<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+  });
+
+  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
+  return records.map((record) => ({
+    timestamp: record.start_time,
+    langfuse_generation_name: record.name,
+    langfuse_trace_name: record.trace_name,
+    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
+    langfuse_id: record.id,
+    langfuse_cost_usd: record.total_cost,
+    langfuse_input_units: record.input_tokens,
+    langfuse_output_units: record.output_tokens,
+    langfuse_total_units: record.total_tokens,
+    langfuse_session_id: record.trace_session_id,
+    langfuse_project_id: projectId,
+    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+    langfuse_latency: record.latency,
+    langfuse_time_to_first_token: record.time_to_first_token,
+    langfuse_release: record.trace_release,
+    langfuse_version: record.version,
+    langfuse_model: record.model,
+    langfuse_level: record.level,
+    langfuse_tags: record.trace_tags,
+    langfuse_event_version: "1.0.0",
+    $session_id: record.posthog_session_id ?? null,
+    $set: {
+      langfuse_user_url: record.user_id
+        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+        : null,
+    },
+  }));
+};

packages/shared/src/server/repositories/observations_converters.ts

@@ -3,15 +3,19 @@ import {
   ObservationView,
   ObservationType,
   ObservationLevel,
+  Prisma,
 } from "@prisma/client";
 import Decimal from "decimal.js";
-import { jsonSchema } from "../../utils/zod";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
+import { parseJsonPrioritised } from "../../utils/json";
 
 export const convertObservationToView = (
   record: ObservationRecordReadType,
-): Omit<ObservationView, "inputPrice" | "outputPrice" | "totalPrice"> => {
+): Omit<ObservationView, "inputPrice" | "outputPrice" | "totalPrice"> & {
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
+} => {
   // these cost are not used from the view. They are in the select statement but not in the
   // Prisma file. We will not clean this up but keep it as it is for now.
   // eslint-disable-next-line no-unused-vars
@@ -23,10 +27,7 @@ export const convertObservationToView = (
       ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
-    timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
-      : null,
+
     promptName: record.prompt_name ?? null,
     promptVersion: record.prompt_version ?? null,
     modelId: record.internal_model_id ?? null,
@@ -40,7 +41,15 @@ export const convertObservation = (
   promptVersion: number | null;
   latency: number | null;
   timeToFirstToken: number | null;
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
 } => {
+  const reducedUsageDetails = reduceUsageOrCostDetails(record.usage_details);
+  const reducedCostDetails = reduceUsageOrCostDetails(record.cost_details);
+  const reducedProvidedCostDetails = reduceUsageOrCostDetails(
+    record.provided_cost_details,
+  );
+
   return {
     id: record.id,
     traceId: record.trace_id ?? null,
@@ -52,12 +61,23 @@ export const convertObservation = (
       ? parseClickhouseUTCDateTimeFormat(record.end_time)
       : null,
     name: record.name ?? null,
-    metadata: record.metadata,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
     level: record.level as ObservationLevel,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: jsonSchema.nullish().parse(record.input) ?? null,
-    output: jsonSchema.nullish().parse(record.output) ?? null,
+    input: (record.input
+      ? parseJsonPrioritised(record.input)
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? parseJsonPrioritised(record.output)
+      : null) as Prisma.JsonValue | null,
     modelParameters: record.model_parameters
       ? JSON.parse(record.model_parameters)
       : null,
@@ -67,31 +87,41 @@ export const convertObservation = (
     promptId: record.prompt_id ?? null,
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
-    promptTokens: record.usage_details?.input
-      ? Number(record.usage_details?.input)
-      : 0,
-    completionTokens: record.usage_details?.output
-      ? Number(record.usage_details?.output)
-      : 0,
-    totalTokens: record.usage_details?.total
-      ? Number(record.usage_details?.total)
-      : 0,
-    calculatedInputCost: record.cost_details?.input
-      ? new Decimal(record.cost_details.input)
-      : null,
-    calculatedOutputCost: record.cost_details?.output
-      ? new Decimal(record.cost_details.output)
-      : null,
+    promptTokens: reducedUsageDetails.input ?? 0,
+    completionTokens: reducedUsageDetails.output ?? 0,
+    totalTokens: reducedUsageDetails.total ?? 0,
+    calculatedInputCost:
+      reducedCostDetails.input != null
+        ? new Decimal(reducedCostDetails.input)
+        : null,
+    calculatedOutputCost:
+      reducedCostDetails.output != null
+        ? new Decimal(reducedCostDetails.output)
+        : null,
     calculatedTotalCost: record.cost_details?.total
       ? new Decimal(record.cost_details.total)
       : null,
-    inputCost: record.cost_details?.input
-      ? new Decimal(record.cost_details?.input)
-      : null,
-    outputCost: record.cost_details?.output
-      ? new Decimal(record.cost_details?.output)
-      : null,
+    inputCost:
+      reducedProvidedCostDetails.input != null
+        ? new Decimal(reducedProvidedCostDetails.input)
+        : null,
+    outputCost:
+      reducedProvidedCostDetails.output != null
+        ? new Decimal(reducedProvidedCostDetails.output)
+        : null,
     totalCost: record.total_cost ? new Decimal(record.total_cost) : null,
+    usageDetails: Object.fromEntries(
+      Object.entries(record.usage_details ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
+    costDetails: Object.fromEntries(
+      Object.entries(record.cost_details ?? {}).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
     model: record.provided_model_name ?? null,
     internalModelId: record.internal_model_id ?? null,
     unit: "TOKENS", // to be removed.
@@ -102,8 +132,35 @@ export const convertObservation = (
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
     timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
+      ? (parseClickhouseUTCDateTimeFormat(
+          record.completion_start_time,
+        ).getTime() -
+          parseClickhouseUTCDateTimeFormat(record.start_time).getTime()) /
+        1000
       : null,
   };
 };
+
+export const reduceUsageOrCostDetails = (
+  details: Record<string, number> | null | undefined,
+): {
+  input: number | null;
+  output: number | null;
+  total: number | null;
+} => {
+  return {
+    input: Object.entries(details ?? {})
+      .filter(([usageType]) => usageType.startsWith("input"))
+      .reduce(
+        (acc, [_, value]) => (acc ?? 0) + Number(value),
+        null as number | null, // default to null if no input usage is found
+      ),
+    output: Object.entries(details ?? {})
+      .filter(([usageType]) => usageType.startsWith("output"))
+      .reduce(
+        (acc, [_, value]) => (acc ?? 0) + Number(value),
+        null as number | null, // default to null if no output usage is found
+      ),
+    total: Number(details?.total ?? 0),
+  };
+};

packages/shared/src/server/repositories/scores.ts

@@ -6,7 +6,7 @@ import {
   upsertClickhouse,
 } from "./clickhouse";
 import { FilterList } from "../queries/clickhouse-sql/clickhouse-filter";
-import { FilterState } from "../../types";
+import { FilterCondition, FilterState, TimeFilter } from "../../types";
 import {
   createFilterFromFilterState,
   getProjectIdDefaultFilter,
@@ -24,28 +24,8 @@ import {
 } from "./scores_converters";
 import { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./constants";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-
-export type FetchScoresReturnType = {
-  id: string;
-  timestamp: string;
-  project_id: string;
-  trace_id: string;
-  observation_id: string | null;
-  name: string;
-  value: number;
-  source: string;
-  comment: string | null;
-  author_user_id: string | null;
-  config_id: string | null;
-  data_type: string;
-  string_value: string | null;
-  queue_id: string | null;
-  created_at: string;
-  updated_at: string;
-  event_ts: string;
-  is_deleted: number;
-  projectId: string;
-};
+import { ScoreRecordReadType } from "./definitions";
+import { env } from "../../env";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -74,7 +54,7 @@ export const searchExistingAnnotationScore = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -103,7 +83,7 @@ export const getScoreById = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -114,28 +94,57 @@ export const getScoreById = async (
   return rows.map(convertToScore).shift();
 };
 
+export const getScoresByIds = async (
+  projectId: string,
+  scoreId: string[],
+  source?: ScoreSource,
+) => {
+  const query = `
+    SELECT *
+    FROM scores s
+    WHERE s.project_id = {projectId: String}
+    AND s.id IN ({scoreId: Array(String)})
+    ${source ? `AND s.source = {source: String}` : ""}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
+  `;
+
+  const rows = await queryClickhouse<ScoreRecordReadType>({
+    query,
+    params: {
+      projectId,
+      scoreId,
+      ...(source !== undefined ? { source } : {}),
+    },
+  });
+  return rows.map(convertToScore);
+};
+
 /**
  * Accepts a score in a Clickhouse-ready format.
  * id, project_id, name, and timestamp must always be provided.
  */
-export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
+export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   if (!["id", "project_id", "name", "timestamp"].every((key) => key in score)) {
     throw new Error("Identifier fields must be provided to upsert Score.");
   }
   await upsertClickhouse({
     table: "scores",
-    records: [score as FetchScoresReturnType],
+    records: [score as ScoreRecordReadType],
     eventBodyMapper: convertToScore,
   });
 };
 
-export const getScoresForTraces = async (
-  projectId: string,
-  traceIds: string[],
-  timestamp?: Date,
-  limit?: number,
-  offset?: number,
-) => {
+export type GetScoresForTracesProps = {
+  projectId: string;
+  traceIds: string[];
+  timestamp?: Date;
+  limit?: number;
+  offset?: number;
+};
+
+export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
+  const { projectId, traceIds, timestamp, limit, offset } = props;
   const query = `
       select 
         *
@@ -148,7 +157,7 @@ export const getScoresForTraces = async (
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId,
@@ -181,7 +190,7 @@ export const getScoresForObservations = async (
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId: projectId,
@@ -497,6 +506,25 @@ export const deleteScoresByTraceIds = async (
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteScoresByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -510,10 +538,14 @@ export const getNumericScoreHistogram = async (
   );
   const chFilterRes = chFilter.apply();
 
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
   const query = `
     select s.value
     from scores s
+    ${traceFilter ? `LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
     WHERE s.project_id = {projectId: String}
+    ${traceFilter ? `AND t.project_id = {projectId: String}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     ORDER BY s.event_ts DESC
     LIMIT 1 BY s.id, s.project_id
@@ -570,3 +602,129 @@ export const getAggregatedScoresForPrompts = async (
     promptId: row.prompt_id,
   }));
 };
+
+export const getScoreCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM scores
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
+export const getDistinctScoreNames = async (
+  projectId: string,
+  cutoffCreatedAt: Date,
+  filter: FilterState,
+  isTimestampFilter: (filter: FilterCondition) => filter is TimeFilter,
+) => {
+  const scoreTimestampFilter = filter?.find(isTimestampFilter);
+
+  const query = `
+    SELECT DISTINCT
+      name
+    FROM scores s 
+    WHERE s.project_id = {projectId: String}
+    AND s.created_at <= {cutoffCreatedAt: DateTime64(3)}
+    ${scoreTimestampFilter ? `AND s.timestamp >= {filterTimestamp: DateTime64(3)}` : ""}
+  `;
+
+  const rows = await queryClickhouse<{ name: string }>({
+    query,
+    params: {
+      projectId,
+      cutoffCreatedAt: convertDateToClickhouseDateTime(cutoffCreatedAt),
+      ...(scoreTimestampFilter
+        ? {
+            filterTimestamp: convertDateToClickhouseDateTime(
+              scoreTimestampFilter.value,
+            ),
+          }
+        : {}),
+    },
+  });
+
+  return rows.map((row) => row.name);
+};
+
+export const getScoresForPostHog = async (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) => {
+  const query = `
+    SELECT
+      s.id as id,
+      s.timestamp as timestamp,
+      s.name as name,
+      s.value as value,
+      s.comment as comment,
+      t.name as trace_name,
+      t.session_id as trace_session_id,
+      t.user_id as trace_user_id,
+      t.release as trace_release,
+      t.tags as trace_tags,
+      t.metadata['$posthog_session_id'] as posthog_session_id
+    FROM scores s FINAL
+    LEFT JOIN traces t FINAL ON s.trace_id = t.id AND s.project_id = t.project_id
+    WHERE s.project_id = {projectId: String}
+    AND t.project_id = {projectId: String}
+    AND s.timestamp >= {minTimestamp: DateTime64(3)}
+    AND s.timestamp <= {maxTimestamp: DateTime64(3)}
+    AND t.timestamp >= {minTimestamp: DateTime64(3)} - INTERVAL 7 DAY
+    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
+  `;
+
+  const records = await queryClickhouse<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+  });
+
+  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
+  return records.map((record) => ({
+    timestamp: record.timestamp,
+    langfuse_score_name: record.name,
+    langfuse_score_value: record.value,
+    langfuse_score_comment: record.comment,
+    langfuse_trace_name: record.trace_name,
+    langfuse_id: record.id,
+    langfuse_session_id: record.trace_session_id,
+    langfuse_project_id: projectId,
+    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+    langfuse_release: record.trace_release,
+    langfuse_tags: record.trace_tags,
+    langfuse_event_version: "1.0.0",
+    $session_id: record.posthog_session_id ?? null,
+    $set: {
+      langfuse_user_url: record.user_id
+        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+        : null,
+    },
+  }));
+};

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,5 +1,5 @@
-import { ScoreSource, ScoreDataType } from "@prisma/client";
-import { FetchScoresReturnType } from "./scores";
+import { ScoreSource, ScoreDataType, Score } from "@prisma/client";
+import { ScoreRecordReadType } from "./definitions";
 
 export type ScoreAggregation = {
   id: string;
@@ -11,22 +11,22 @@ export type ScoreAggregation = {
   comment: string | null;
 };
 
-export const convertToScore = (row: FetchScoresReturnType) => {
+export const convertToScore = (row: ScoreRecordReadType): Score => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),
     projectId: row.project_id,
     traceId: row.trace_id,
-    observationId: row.observation_id,
+    observationId: row.observation_id ?? null,
     name: row.name,
-    value: row.value,
+    value: row.value ?? null,
     source: row.source as ScoreSource,
-    comment: row.comment,
-    authorUserId: row.author_user_id,
-    configId: row.config_id,
+    comment: row.comment ?? null,
+    authorUserId: row.author_user_id ?? null,
+    configId: row.config_id ?? null,
     dataType: row.data_type as ScoreDataType,
-    stringValue: row.string_value,
-    queueId: row.queue_id,
+    stringValue: row.string_value ?? null,
+    queueId: row.queue_id ?? null,
     createdAt: new Date(row.created_at),
     updatedAt: new Date(row.updated_at),
   };

packages/shared/src/server/repositories/trace-sessions.ts

@@ -0,0 +1,61 @@
+import z from "zod";
+import { prisma } from "../../db";
+import { singleFilter, stringOptionsFilter } from "../../interfaces/filters";
+
+export const getPublicSessionsFilter = async (
+  projectId: string,
+  filter: z.infer<typeof singleFilter>[],
+) => {
+  const sessionsBookmarkedFilter = filter?.find((f) => f.column === "⭐️");
+
+  // we are only fetching bookmarked sessions.
+  // They need to be manipulated in the UI and should not be as many.
+  const filteredSessions = sessionsBookmarkedFilter
+    ? await prisma.traceSession.findMany({
+        where: {
+          projectId: projectId,
+          bookmarked: true,
+        },
+        select: {
+          id: true,
+          createdAt: true,
+          bookmarked: true,
+          public: true,
+        },
+      })
+    : [];
+
+  const additionalBookmarkFilter: z.infer<typeof stringOptionsFilter>[] =
+    sessionsBookmarkedFilter &&
+    typeof sessionsBookmarkedFilter.value === "boolean" &&
+    ((sessionsBookmarkedFilter.value === true &&
+      sessionsBookmarkedFilter.operator === "=") ||
+      (sessionsBookmarkedFilter.value === false &&
+        sessionsBookmarkedFilter.operator === "<>"))
+      ? [
+          {
+            column: "id",
+            type: "stringOptions" as const,
+            operator: "any of" as const,
+            value: filteredSessions.map((s) => s.id),
+          },
+        ]
+      : sessionsBookmarkedFilter &&
+          typeof sessionsBookmarkedFilter.value === "boolean" &&
+          (sessionsBookmarkedFilter.value === false ||
+            (sessionsBookmarkedFilter.value === true &&
+              sessionsBookmarkedFilter.operator === "<>"))
+        ? [
+            {
+              column: "id",
+              type: "stringOptions" as const,
+              operator: "none of" as const,
+              value: filteredSessions.map((s) => s.id),
+            },
+          ]
+        : [];
+
+  return filter
+    ? [...filter.filter((f) => f.column !== "⭐️"), ...additionalBookmarkFilter]
+    : [...additionalBookmarkFilter];
+};

packages/shared/src/server/repositories/traces.ts

@@ -12,18 +12,93 @@ import { FilterState } from "../../types";
 import {
   DateTimeFilter,
   FilterList,
+  StringFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
-import { OrderByState } from "../../interfaces/orderBy";
-import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { UiColumnMapping } from "../../tableDefinitions";
-import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
-import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
-import { FetchTracesTableProps } from "../services/traces-ui-table-service";
+import {
+  OBSERVATIONS_TO_TRACE_INTERVAL,
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+} from "./constants";
+import { env } from "../../env";
+
+export const checkTraceExists = async (
+  projectId: string,
+  traceId: string,
+  timestamp: Date | undefined,
+  filter: FilterState,
+): Promise<boolean> => {
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
+  });
+
+  const timeStampFilter = tracesFilter.find(
+    (f) =>
+      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+    new StringFilter({
+      clickhouseTable: "t",
+      field: "id",
+      operator: "=",
+      value: traceId,
+    }),
+  );
+
+  const observationFilter = tracesFilter.find(
+    (f) => f.clickhouseTable === "observations",
+  );
+  const tracesFilterRes = tracesFilter.apply();
+  const observationFilterRes = observationFilter?.apply();
+
+  const query = `
+    WITH observations_agg AS (
+        SELECT
+          
+            multiIf(
+              arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
+              arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
+              arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
+              'DEBUG'
+            ) AS level,
+            trace_id,
+            project_id
+        FROM observations o FINAL 
+        WHERE o.project_id = {projectId: String}
+        ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        GROUP BY trace_id, project_id
+      )
+    SELECT 
+      t.id as id, 
+      t.project_id as project_id
+    FROM traces t FINAL 
+    ${observationFilterRes ? `INNER JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id` : ""}
+    WHERE ${tracesFilterRes.query}
+    AND t.project_id = {projectId: String}
+    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+    GROUP BY t.id, t.project_id
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      projectId,
+      ...tracesFilterRes.params,
+      ...(observationFilterRes ? observationFilterRes.params : {}),
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
 
 /**
  * Accepts a trace in a Clickhouse-ready format.
@@ -108,6 +183,37 @@ export const hasAnyTrace = async (projectId: string) => {
   return rows.length > 0 && Number(rows[0].count) > 0;
 };
 
+export const getTraceCountsByProjectInCreationInterval = async ({
+  start,
+  end,
+}: {
+  start: Date;
+  end: Date;
+}) => {
+  const query = `
+    SELECT 
+      project_id,
+      count(*) as count
+    FROM traces
+    WHERE created_at >= {start: DateTime64(3)}
+    AND created_at < {end: DateTime64(3)}
+    GROUP BY project_id
+  `;
+
+  const rows = await queryClickhouse<{ project_id: string; count: string }>({
+    query,
+    params: {
+      start: convertDateToClickhouseDateTime(start),
+      end: convertDateToClickhouseDateTime(end),
+    },
+  });
+
+  return rows.map((row) => ({
+    projectId: row.project_id,
+    count: Number(row.count),
+  }));
+};
+
 export const getTraceById = async (
   traceId: string,
   projectId: string,
@@ -274,186 +380,6 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
   return rows;
 };
 
-export type SessionDataReturnType = {
-  session_id: string;
-  max_timestamp: string;
-  min_timestamp: string;
-  trace_ids: string[];
-  user_ids: string[];
-  trace_count: number;
-  trace_tags: string[];
-  total_observations: number;
-  duration: number;
-  session_usage_details: Record<string, number>;
-  session_cost_details: Record<string, number>;
-  session_input_cost: string;
-  session_output_cost: string;
-  session_total_cost: string;
-  session_input_usage: string;
-  session_output_usage: string;
-  session_total_usage: string;
-};
-
-export const getSessionsTableCount = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<{ count: string }>({
-    select: `
-      count(session_id) as count
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows.length > 0 ? Number(rows[0].count) : 0;
-};
-
-export const getSessionsTable = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
-    select: `
-    session_id, 
-    max_timestamp, 
-    min_timestamp, 
-    trace_ids, 
-    user_ids, 
-    trace_count, 
-    trace_tags,
-    total_observations,
-    duration,
-    session_usage_details,
-    session_cost_details,
-    session_input_cost,
-    session_output_cost,
-    session_total_cost,
-    session_input_usage,
-    session_output_usage,
-    session_total_usage
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows;
-};
-
-const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, page } = props;
-
-  const { tracesFilter, scoresFilter, observationsFilter } =
-    getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
-
-  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
-
-  const tracesFilterRes = tracesFilter.apply();
-  const scoresAvgFilterRes = scoresFilter.apply();
-  const observationsStatsRes = observationsFilter.apply();
-
-  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
-    (f) =>
-      f.field === "min_timestamp" &&
-      (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  const singleTraceFilter = traceTimestampFilter
-    ? new FilterList([
-        new DateTimeFilter({
-          clickhouseTable: "traces",
-          field: "timestamp",
-          operator: traceTimestampFilter.operator,
-          value: traceTimestampFilter.value,
-        }),
-      ]).apply()
-    : undefined;
-
-  const query = `
-      WITH observations_agg AS (
-        SELECT o.trace_id,
-              count(*) as obs_count,
-              min(o.start_time) as min_start_time,
-              max(o.end_time) as max_end_time,
-              sumMap(usage_details) as sum_usage_details,
-              sumMap(cost_details) as sum_cost_details,
-              anyLast(project_id) as project_id
-        FROM observations o FINAL
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            -- Aggregate observations data at session level
-            sum(o.obs_count) as total_observations,
-            date_diff('milliseconds', min(min_start_time), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            sumMap(o.sum_cost_details)['input'] as session_input_cost,
-            sumMap(o.sum_cost_details)['output'] as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,
-            sumMap(o.sum_usage_details)['input'] as session_input_usage,
-            sumMap(o.sum_usage_details)['output'] as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage
-        FROM traces t FINAL
-        LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id
-        WHERE t.session_id IS NOT NULL
-            AND t.project_id = {projectId: String}
-            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${select}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
-
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...observationsStatsRes.params,
-      ...scoresAvgFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
-    },
-  });
-
-  return res;
-};
-
 export const getTracesIdentifierForSession = async (
   projectId: string,
   sessionId: string,
@@ -505,6 +431,25 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
       projectId,
       traceIds,
     },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
+export const deleteTracesByProjectId = async (projectId: string) => {
+  const query = `
+    DELETE FROM traces
+    WHERE project_id = {projectId: String};
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
   });
 };
 
@@ -542,68 +487,237 @@ export const getTotalUserCount = async (
   });
 };
 
-export const getUserMetrics = async (projectId: string, userIds: string[]) => {
+export const getUserMetrics = async (
+  projectId: string,
+  userIds: string[],
+  filter: FilterState,
+) => {
   if (userIds.length === 0) {
     return [];
   }
+
+  // filter state contains date range filter for traces so far.
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const timestampFilter = chFilter.find(
+    (f) => f.field === "timestamp" && f.operator === ">=",
+  );
+
+  // this query uses window functions on observations + traces to always get only the first row and thereby remove deduplicates
+  // we filter wherever possible by project id and user id
   const query = `
-    WITH observations_agg AS (
-      SELECT o.trace_id,
-             count(*) as obs_count,
-             sumMap(usage_details) as sum_usage_details,
-             sum(total_cost) as sum_total_cost,
-             anyLast(project_id) as project_id
-      FROM observations o FINAL
-      WHERE o.project_id = {projectId: String}
-      GROUP BY o.trace_id
-    ),
-    user_metric_data AS (
-      SELECT t.user_id,
-             max(t.timestamp) as max_timestamp,
-             min(t.timestamp) as min_timestamp,
-             count(*) as trace_count,
-             sum(o.obs_count) as total_observations,
-             sum(o.sum_total_cost) as session_total_cost,
-             sumMap(o.sum_usage_details)['input'] as session_input_usage,
-             sumMap(o.sum_usage_details)['output'] as session_output_usage,
-             sumMap(o.sum_usage_details)['total'] as session_total_usage
-      FROM traces t FINAL
-      LEFT JOIN observations_agg o
-      ON t.id = o.trace_id 
-      AND t.project_id = o.project_id
-      WHERE t.user_id IS NOT NULL
-      AND t.user_id != ''
-      AND t.user_id IN ({userIds: Array(String)})
-      AND t.project_id = {projectId: String}
-      GROUP BY t.user_id
+      WITH stats as (
+        SELECT
+            t.user_id as user_id,
+            count(distinct o.id) as obs_count,
+            sumMap(usage_details) as sum_usage_details,
+            sum(total_cost) as sum_total_cost,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            count(distinct t.id) as trace_count
+        FROM
+            (
+                SELECT
+                    o.project_id,
+                    o.trace_id,
+                    o.usage_details,
+                    o.total_cost,
+                    id,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY id
+                        ORDER BY
+                            event_ts DESC
+                    ) AS rn
+                FROM
+                    observations o
+                WHERE
+                    o.project_id = {projectId: String }
+                    ${timestampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+                    AND o.trace_id in (
+                        SELECT
+                            distinct id
+                        from
+                            traces
+                        where
+                            user_id IN ({userIds: Array(String) })
+                            AND project_id = {projectId: String }
+                            ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
+                    )
+                    AND o.type = 'GENERATION'
+            ) as o
+            JOIN (
+                SELECT
+                    t.id,
+                    t.user_id,
+                    t.project_id,
+                    t.timestamp,
+                    ROW_NUMBER() OVER (
+                        PARTITION BY id
+                        ORDER BY
+                            event_ts DESC
+                    ) AS rn
+                FROM
+                    traces t
+                WHERE
+                    t.user_id IN ({userIds: Array(String) })
+                    AND t.project_id = {projectId: String }
+                    ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
+            ) as t on t.id = o.trace_id
+            and t.project_id = o.project_id
+        WHERE
+            o.rn = 1
+            and t.rn = 1
+        group by
+            t.user_id
     )
-    SELECT user_id AS userId,
-           min_timestamp as firstTrace,
-           max_timestamp as lastTrace,
-           trace_count as totalTraces,
-           total_observations as totalObservations,
-           session_input_usage as totalPromptTokens,
-           session_output_usage as totalCompletionTokens,
-           session_total_usage as totalTokens,
-           session_total_cost as sumCalculatedTotalCost
-    FROM user_metric_data umd
+    SELECT
+        arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sum_usage_details))) as input_usage,
+        arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sum_usage_details))) as output_usage,
+        sum_usage_details [ 'total' ] as total_usage,
+        obs_count,
+        trace_count,
+        user_id,
+        sum_total_cost,
+        max_timestamp,
+        min_timestamp
+    FROM
+        stats
+
   `;
 
-  return queryClickhouse<{
-    userId: string;
-    firstTrace: Date | null;
-    lastTrace: Date | null;
-    totalPromptTokens: bigint;
-    totalCompletionTokens: bigint;
-    totalTokens: bigint;
-    totalObservations: bigint;
-    totalTraces: bigint;
-    sumCalculatedTotalCost: number;
+  const rows = await queryClickhouse<{
+    user_id: string;
+    max_timestamp: string;
+    min_timestamp: string;
+    input_usage: string;
+    output_usage: string;
+    total_usage: string;
+    obs_count: string;
+    trace_count: string;
+    sum_total_cost: string;
   }>({
     query,
     params: {
       projectId,
       userIds,
+      ...chFilterRes.params,
+      ...(timestampFilter
+        ? {
+            traceTimestamp: convertDateToClickhouseDateTime(
+              (timestampFilter as DateTimeFilter).value,
+            ),
+          }
+        : {}),
     },
   });
+
+  return rows.map((row) => ({
+    userId: row.user_id,
+    maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
+    minTimestamp: parseClickhouseUTCDateTimeFormat(row.min_timestamp),
+    inputUsage: Number(row.input_usage),
+    outputUsage: Number(row.output_usage),
+    totalUsage: Number(row.total_usage),
+    observationCount: Number(row.obs_count),
+    traceCount: Number(row.trace_count),
+    totalCost: Number(row.sum_total_cost),
+  }));
+};
+
+export const getTracesForPostHog = async (
+  projectId: string,
+  minTimestamp: Date,
+  maxTimestamp: Date,
+) => {
+  const query = `
+    WITH observations_agg AS (
+      SELECT o.project_id,
+             o.trace_id,
+             sum(total_cost) as total_cost,
+             count(*) as observation_count,
+             date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds
+      FROM observations o FINAL
+      WHERE o.project_id = {projectId: String}
+      AND o.start_time >= {minTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
+      GROUP BY o.project_id, o.trace_id
+    )
+
+    SELECT 
+      t.id as id,
+      t.timestamp as timestamp,
+      t.name as name,
+      t.session_id as session_id,
+      t.user_id as user_id,
+      t.release as release,
+      t.version as version,
+      t.tags as tags,
+      t.metadata['$posthog_session_id'] as posthog_session_id,
+      o.total_cost as total_cost,
+      o.latency_milliseconds / 1000 as latency,
+      o.observation_count as observation_count
+    FROM traces t FINAL
+    LEFT JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id
+    WHERE t.project_id = {projectId: String}
+    AND t.timestamp >= {minTimestamp: DateTime64(3)}
+    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
+  `;
+
+  const records = await queryClickhouse<Record<string, unknown>>({
+    query,
+    params: {
+      projectId,
+      minTimestamp: convertDateToClickhouseDateTime(minTimestamp),
+      maxTimestamp: convertDateToClickhouseDateTime(maxTimestamp),
+    },
+  });
+
+  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
+  return records.map((record) => ({
+    timestamp: record.timestamp,
+    langfuse_id: record.id,
+    langfuse_trace_name: record.name,
+    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.id as string)}`,
+    langfuse_cost_usd: record.total_cost,
+    langfuse_count_observations: record.observation_count,
+    langfuse_session_id: record.session_id,
+    langfuse_project_id: projectId,
+    langfuse_user_id: record.user_id || "langfuse_unknown_user",
+    langfuse_latency: record.latency,
+    langfuse_release: record.release,
+    langfuse_version: record.version,
+    langfuse_tags: record.tags,
+    langfuse_event_version: "1.0.0",
+    $session_id: record.posthog_session_id ?? null,
+    $set: {
+      langfuse_user_url: record.user_id
+        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+        : null,
+    },
+  }));
+};
+
+export const getTracesByIdsForAnyProject = async (traceIds: string[]) => {
+  const query = `
+      SELECT id, project_id
+      FROM traces
+      WHERE id IN ({traceIds: Array(String)})
+      ORDER BY event_ts DESC
+      LIMIT 1 by id, project_id;`;
+  const records = await queryClickhouse<{
+    id: string;
+    project_id: string;
+  }>({
+    query,
+    params: {
+      traceIds,
+    },
+  });
+
+  return records.map((record) => ({
+    id: record.id,
+    projectId: record.project_id,
+  }));
 };

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,10 +1,8 @@
-import { ObservationLevel, Trace } from "@prisma/client";
+import { Prisma, Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
-import Decimal from "decimal.js";
-import { ScoreAggregate } from "../../features/scores";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import { TracesTableReturnType } from "../services/traces-ui-table-service";
+import { parseJsonPrioritised } from "../../utils/json";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -46,61 +44,22 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: record.input ?? null,
-    output: record.output ?? null,
-    metadata: record.metadata,
+    input: (record.input
+      ? parseJsonPrioritised(record.input)
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? parseJsonPrioritised(record.output)
+      : null) as Prisma.JsonValue | null,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
     externalId: null,
   };
 };
-
-export type TracesAllReturnType = {
-  id: string;
-  timestamp: Date;
-  name: string | null;
-  projectId: string;
-  userId: string | null;
-  release: string | null;
-  version: string | null;
-  public: boolean;
-  bookmarked: boolean;
-  sessionId: string | null;
-  tags: string[];
-};
-
-export const convertToDomain = (row: TracesTableReturnType) => {
-  return {
-    id: row.id,
-    projectId: row.project_id,
-    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
-    tags: row.tags,
-    bookmarked: row.bookmarked,
-    name: row.name ?? null,
-    release: row.release ?? null,
-    version: row.version ?? null,
-    userId: row.user_id ?? null,
-    sessionId: row.session_id ?? null,
-    latencyMilliseconds: Number(row.latency_milliseconds),
-    usageDetails: row.usage_details,
-    costDetails: row.cost_details,
-    level: row.level,
-    observationCount: Number(row.observation_count),
-    scoresAvg: row.scores_avg,
-    public: row.public,
-  };
-};
-
-export type TracesMetricsReturnType = {
-  id: string;
-  promptTokens: bigint;
-  completionTokens: bigint;
-  totalTokens: bigint;
-  latency: number | null;
-  level: ObservationLevel;
-  observationCount: bigint;
-  calculatedTotalCost: Decimal | null;
-  calculatedInputCost: Decimal | null;
-  calculatedOutputCost: Decimal | null;
-  scores: ScoreAggregate;
-};

packages/shared/src/server/services/S3StorageService.ts

@@ -1,172 +0,0 @@
-import type { Readable } from "stream";
-import {
-  GetObjectCommand,
-  ListObjectsV2Command,
-  PutObjectCommand,
-  S3Client,
-} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { logger } from "../logger";
-
-type UploadFile = {
-  fileName: string;
-  fileType: string;
-  data: Readable | string;
-  expiresInSeconds: number;
-};
-
-export class S3StorageService {
-  private client: S3Client;
-  private bucketName: string;
-
-  constructor(params: {
-    accessKeyId: string | undefined;
-    secretAccessKey: string | undefined;
-    bucketName: string;
-    endpoint: string | undefined;
-    region: string | undefined;
-    forcePathStyle: boolean;
-  }) {
-    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
-    const { accessKeyId, secretAccessKey } = params;
-    const credentials =
-      accessKeyId !== undefined && secretAccessKey !== undefined
-        ? {
-            accessKeyId,
-            secretAccessKey,
-          }
-        : undefined;
-
-    this.client = new S3Client({
-      credentials,
-      endpoint: params.endpoint,
-      region: params.region,
-      forcePathStyle: params.forcePathStyle,
-    });
-    this.bucketName = params.bucketName;
-  }
-
-  public async uploadFile({
-    fileName,
-    fileType,
-    data,
-    expiresInSeconds,
-  }: UploadFile): Promise<{ signedUrl: string }> {
-    try {
-      await new Upload({
-        client: this.client,
-        params: {
-          Bucket: this.bucketName,
-          Key: fileName,
-          Body: data,
-          ContentType: fileType,
-        },
-      }).done();
-
-      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
-
-      return { signedUrl };
-    } catch (err) {
-      logger.error(`Failed to upload file to ${fileName}`, err);
-      throw new Error("Failed to upload to S3 or generate signed URL");
-    }
-  }
-
-  public async uploadJson(path: string, body: Record<string, unknown>[]) {
-    const putCommand = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-      Body: JSON.stringify(body),
-      ContentType: "application/json",
-    });
-
-    try {
-      await this.client.send(putCommand);
-    } catch (err) {
-      logger.error(`Failed to upload JSON to S3 ${path}`, err);
-      throw Error("Failed to upload JSON to S3");
-    }
-  }
-
-  public async download(path: string): Promise<string> {
-    const getCommand = new GetObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-    });
-
-    try {
-      const response = await this.client.send(getCommand);
-      return (await response.Body?.transformToString()) ?? "";
-    } catch (err) {
-      logger.error(`Failed to download file from S3 ${path}`, err);
-      throw Error("Failed to download file from S3");
-    }
-  }
-
-  public async listFiles(prefix: string): Promise<string[]> {
-    const listCommand = new ListObjectsV2Command({
-      Bucket: this.bucketName,
-      Prefix: prefix,
-    });
-
-    try {
-      const response = await this.client.send(listCommand);
-      return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
-      );
-    } catch (err) {
-      logger.error(`Failed to list files from S3 ${prefix}`, err);
-      throw Error("Failed to list files from S3");
-    }
-  }
-
-  public async getSignedUrl(
-    fileName: string,
-    ttlSeconds: number,
-    asAttachment: boolean = true,
-  ): Promise<string> {
-    try {
-      return await getSignedUrl(
-        this.client,
-        new GetObjectCommand({
-          Bucket: this.bucketName,
-          Key: fileName,
-          ResponseContentDisposition: asAttachment
-            ? `attachment; filename="${fileName}"`
-            : undefined,
-        }),
-        { expiresIn: ttlSeconds },
-      );
-    } catch (err) {
-      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
-      throw Error("Failed to generate signed URL");
-    }
-  }
-
-  public async getSignedUploadUrl(params: {
-    path: string;
-    ttlSeconds: number;
-    sha256Hash: string;
-    contentType: string;
-    contentLength: number;
-  }): Promise<string> {
-    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
-
-    return await getSignedUrl(
-      this.client,
-      new PutObjectCommand({
-        Bucket: this.bucketName,
-        Key: path,
-        ContentType: contentType,
-        ChecksumSHA256: sha256Hash,
-        ContentLength: contentLength,
-      }),
-      {
-        expiresIn: ttlSeconds,
-        signableHeaders: new Set(["content-type", "content-length"]),
-        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
-      },
-    );
-  }
-}

packages/shared/src/server/services/StorageService.ts

@@ -0,0 +1,443 @@
+import { Readable } from "stream";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from "@azure/storage-blob";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+type UploadFile = {
+  fileName: string;
+  fileType: string;
+  data: Readable | string;
+  expiresInSeconds: number;
+};
+
+export interface StorageService {
+  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>;
+
+  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>;
+
+  download(path: string): Promise<string>;
+
+  listFiles(prefix: string): Promise<{ file: string; createdAt: Date }[]>;
+
+  getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string>;
+
+  getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string>;
+}
+
+export class StorageServiceFactory {
+  public static getInstance(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }): StorageService {
+    if (env.LANGFUSE_USE_AZURE_BLOB === "true") {
+      return new AzureBlobStorageService(params);
+    }
+    return new S3StorageService(params);
+  }
+}
+
+class AzureBlobStorageService implements StorageService {
+  private client: ContainerClient;
+  private container: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    const { accessKeyId, secretAccessKey, endpoint } = params;
+    if (!accessKeyId || !secretAccessKey || !endpoint) {
+      throw new Error(
+        `Endpoint, account and account key must be configured to use Azure Blob Storage`,
+      );
+    }
+
+    const sharedKeyCredential = new StorageSharedKeyCredential(
+      accessKeyId,
+      secretAccessKey,
+    );
+    const blobServiceClient = new BlobServiceClient(
+      endpoint,
+      sharedKeyCredential,
+    );
+    this.container = params.bucketName;
+    this.client = blobServiceClient.getContainerClient(this.container);
+  }
+
+  private async createContainerIfNotExists(): Promise<void> {
+    try {
+      await this.client.createIfNotExists();
+    } catch (err) {
+      logger.error(
+        `Failed to create Azure Blob Storage container ${this.container}`,
+        err,
+      );
+      throw Error("Failed to create Azure Blob Storage container ");
+    }
+  }
+
+  public async uploadFile(params: UploadFile): Promise<{ signedUrl: string }> {
+    const { fileName, data, expiresInSeconds } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+
+      if (typeof data === "string") {
+        await blockBlobClient.upload(data, data.length);
+      } else if (data instanceof Readable) {
+        let offset = 0;
+        const blockIds = [];
+        for await (const chunk of data) {
+          const blockId = Buffer.from(`block-${offset}`).toString("base64");
+          const bufferChunk = Buffer.isBuffer(chunk)
+            ? chunk
+            : Buffer.from(chunk);
+
+          await blockBlobClient.stageBlock(
+            blockId,
+            bufferChunk,
+            bufferChunk.length,
+          );
+          blockIds.push(blockId);
+
+          offset += bufferChunk.length;
+        }
+
+        await blockBlobClient.commitBlockList(blockIds);
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+
+      return {
+        signedUrl: await this.getSignedUrl(fileName, expiresInSeconds, false),
+      };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to upload file to Azure Blob Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    await this.createContainerIfNotExists();
+
+    const blockBlobClient = this.client.getBlockBlobClient(path);
+    const content = JSON.stringify(body);
+    try {
+      await blockBlobClient.upload(content, content.length);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to Azure Blob Storage ${path}`, err);
+      throw Error("Failed to upload JSON to Azure Blob Storage");
+    }
+  }
+
+  private async streamToString(
+    readableStream: NodeJS.ReadableStream,
+  ): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const chunks: string[] = [];
+      readableStream.on("data", (data) => {
+        chunks.push(data.toString());
+      });
+      readableStream.on("end", () => {
+        resolve(chunks.join(""));
+      });
+      readableStream.on("error", reject);
+    });
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blobClient = this.client.getBlobClient(path);
+      const downloadResponse = await blobClient.download();
+      if (!downloadResponse.readableStreamBody) {
+        throw Error("No stream body available");
+      }
+      return this.streamToString(downloadResponse.readableStreamBody);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const result = await this.client.listBlobsFlat({ prefix });
+      const files = [];
+      for await (const blob of result) {
+        if (blob.name.startsWith(prefix)) {
+          files.push({
+            file: blob.name,
+            createdAt: blob?.properties?.createdOn ?? new Date(),
+          });
+        }
+      }
+      return files;
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Azure Blob Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("r"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentDisposition: asAttachment
+          ? `attachment; filename="${fileName}"`
+          : undefined,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL for Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate presigned URL for Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(path);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("w"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentType: contentType,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL for Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate presigned upload URL for Azure Blob Storage",
+      );
+    }
+  }
+}
+
+class S3StorageService implements StorageService {
+  private client: S3Client;
+  private bucketName: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
+    const { accessKeyId, secretAccessKey } = params;
+    const credentials =
+      accessKeyId !== undefined && secretAccessKey !== undefined
+        ? {
+            accessKeyId,
+            secretAccessKey,
+          }
+        : undefined;
+
+    this.client = new S3Client({
+      credentials,
+      endpoint: params.endpoint,
+      region: params.region,
+      forcePathStyle: params.forcePathStyle,
+    });
+    this.bucketName = params.bucketName;
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      await new Upload({
+        client: this.client,
+        params: {
+          Bucket: this.bucketName,
+          Key: fileName,
+          Body: data,
+          ContentType: fileType,
+        },
+      }).done();
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(`Failed to upload file to ${fileName}`, err);
+      throw new Error("Failed to upload to S3 or generate signed URL");
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    const putCommand = new PutObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+      Body: JSON.stringify(body),
+      ContentType: "application/json",
+    });
+
+    try {
+      await this.client.send(putCommand);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to S3 ${path}`, err);
+      throw Error("Failed to upload JSON to S3");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+
+    try {
+      const response = await this.client.send(getCommand);
+      return (await response.Body?.transformToString()) ?? "";
+    } catch (err) {
+      logger.error(`Failed to download file from S3 ${path}`, err);
+      throw Error("Failed to download file from S3");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: prefix,
+    });
+
+    try {
+      const response = await this.client.send(listCommand);
+      return (
+        response.Contents?.flatMap((file) =>
+          file.Key
+            ? [{ file: file.Key, createdAt: file.LastModified ?? new Date() }]
+            : [],
+        ) ?? []
+      );
+    } catch (err) {
+      logger.error(`Failed to list files from S3 ${prefix}`, err);
+      throw Error("Failed to list files from S3");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      return await getSignedUrl(
+        this.client,
+        new GetObjectCommand({
+          Bucket: this.bucketName,
+          Key: fileName,
+          ResponseContentDisposition: asAttachment
+            ? `attachment; filename="${fileName}"`
+            : undefined,
+        }),
+        { expiresIn: ttlSeconds },
+      );
+    } catch (err) {
+      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
+      throw Error("Failed to generate signed URL");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
+
+    return await getSignedUrl(
+      this.client,
+      new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: path,
+        ContentType: contentType,
+        ChecksumSHA256: sha256Hash,
+        ContentLength: contentLength,
+      }),
+      {
+        expiresIn: ttlSeconds,
+        signableHeaders: new Set(["content-type", "content-length"]),
+        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
+      },
+    );
+  }
+}

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -0,0 +1,307 @@
+import { OrderByState } from "../../interfaces/orderBy";
+import { sessionCols } from "../../tableDefinitions/mapSessionTable";
+import { FilterState } from "../../types";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
+import {
+  getProjectIdDefaultFilter,
+  createFilterFromFilterState,
+} from "../queries/clickhouse-sql/factory";
+import {
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+  queryClickhouse,
+} from "../repositories";
+
+export type SessionDataReturnType = {
+  session_id: string;
+  max_timestamp: string;
+  min_timestamp: string;
+  trace_ids: string[];
+  user_ids: string[];
+  trace_count: number;
+  trace_tags: string[];
+};
+
+export type SessionWithMetricsReturnType = SessionDataReturnType & {
+  total_observations: number;
+  duration: number;
+  session_usage_details: Record<string, number>;
+  session_cost_details: Record<string, number>;
+  session_input_cost: string;
+  session_output_cost: string;
+  session_total_cost: string;
+  session_input_usage: string;
+  session_output_usage: string;
+  session_total_usage: string;
+};
+
+export const getSessionsTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<{ count: string }>({
+    select: "count",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.length > 0 ? Number(rows[0].count) : 0;
+};
+
+export const getSessionsTable = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
+    select: "rows",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+  }));
+};
+
+export const getSessionsWithMetrics = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({
+    select: "metrics",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+    total_observations: Number(row.total_observations),
+  }));
+};
+
+export type FetchSessionsTableProps = {
+  select: "count" | "rows" | "metrics";
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
+  const { select, projectId, filter, orderBy, limit, page } = props;
+
+  let sqlSelect: string;
+  switch (select) {
+    case "count":
+      sqlSelect = "count(session_id) as count";
+      break;
+    case "rows":
+      sqlSelect = `
+          session_id, 
+          max_timestamp, 
+          min_timestamp, 
+          trace_ids, 
+          user_ids, 
+          trace_count, 
+          trace_tags`;
+      break;
+    case "metrics":
+      sqlSelect = `
+        session_id, 
+        max_timestamp, 
+        min_timestamp, 
+        trace_ids, 
+        user_ids, 
+        trace_count, 
+        trace_tags,
+        total_observations,
+        duration,
+        session_usage_details,
+        session_cost_details,
+        session_input_cost,
+        session_output_cost,
+        session_total_cost,
+        session_input_usage,
+        session_output_usage,
+        session_total_usage`;
+      break;
+    default:
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "s",
+  });
+
+  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
+
+  const tracesFilterRes = tracesFilter.apply();
+
+  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
+    (f) =>
+      f.field === "min_timestamp" &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const filters = [];
+  if (traceTimestampFilter) {
+    filters.push(
+      new DateTimeFilter({
+        clickhouseTable: "traces",
+        field: "timestamp",
+        operator: traceTimestampFilter.operator,
+        value: traceTimestampFilter.value,
+      }),
+    );
+  }
+
+  const additionalSingleTraceFilter = tracesFilter.find(
+    (f) => f.field === "bookmarked" || f.field === "session_id",
+  );
+
+  if (additionalSingleTraceFilter) {
+    filters.push(additionalSingleTraceFilter);
+  }
+
+  const singleTraceFilter =
+    filters.length > 0 ? new FilterList(filters).apply() : undefined;
+
+  const hasMetricsFilter = tracesFilter.find((f) =>
+    [
+      "session_total_cost",
+      "session_input_cost",
+      "session_output_cost",
+      "duration",
+      "session_total_usage",
+      "session_output_usage",
+      "session_input_usage",
+    ].includes(f.field),
+  );
+
+  const selectMetrics = select === "metrics" || hasMetricsFilter;
+
+  // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
+  const query = `
+    WITH deduplicated_traces AS (
+      SELECT * EXCEPT input, output, metadata
+      FROM traces t
+      WHERE t.session_id IS NOT NULL 
+        AND t.project_id = {projectId: String}
+        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    deduplicated_observations AS (
+        SELECT * 
+        FROM observations o
+        WHERE o.project_id = {projectId: String}
+        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+        AND o.trace_id IN (
+          SELECT id
+          FROM deduplicated_traces
+        )
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    observations_agg AS (
+      SELECT o.trace_id,
+            count(*) as obs_count,
+            min(o.start_time) as min_start_time,
+            max(o.end_time) as max_end_time,
+            sumMap(usage_details) as sum_usage_details,
+            sumMap(cost_details) as sum_cost_details,
+            anyLast(project_id) as project_id
+      FROM deduplicated_observations o
+      WHERE o.project_id = {projectId: String}
+      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+      GROUP BY o.trace_id
+    ),
+    session_data AS (
+        SELECT
+            t.session_id,
+            anyLast(t.project_id) as project_id,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            groupArray(t.id) AS trace_ids,
+            groupUniqArray(t.user_id) AS user_ids,
+            count(*) as trace_count,
+            groupUniqArrayArray(t.tags) as trace_tags
+            -- Aggregate observations data at session level
+            ${
+              selectMetrics
+                ? `
+            ,
+            sum(o.obs_count) as total_observations,
+            date_diff('millisecond', min(min_start_time), max(max_end_time)) as duration,
+            sumMap(o.sum_usage_details) as session_usage_details,
+            sumMap(o.sum_cost_details) as session_cost_details,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+            sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                : ""
+            }
+        FROM deduplicated_traces t
+        ${
+          selectMetrics
+            ? `LEFT JOIN observations_agg o
+        ON t.id = o.trace_id AND t.project_id = o.project_id`
+            : ""
+        }
+        WHERE t.session_id IS NOT NULL
+            AND t.project_id = {projectId: String}
+            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        GROUP BY t.session_id
+    )
+    SELECT ${sqlSelect}
+    FROM session_data s
+    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    `;
+
+  const obsStartTimeValue = traceTimestampFilter
+    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
+    : null;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      projectId,
+      limit: limit,
+      offset: limit && page ? limit * page : 0,
+      ...tracesFilterRes.params,
+      ...singleTraceFilter?.params,
+      ...(obsStartTimeValue
+        ? { observationsStartTime: obsStartTimeValue }
+        : {}),
+    },
+  });
+
+  return res;
+};

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -13,9 +13,18 @@ import {
 } from "../queries/clickhouse-sql/factory";
 import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
-import { convertToDomain } from "../repositories";
-import { queryClickhouse } from "../repositories/clickhouse";
+import {
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+} from "../repositories/clickhouse";
 import { TraceRecordReadType } from "../repositories/definitions";
+import {
+  OBSERVATIONS_TO_TRACE_INTERVAL,
+  SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
+} from "../repositories/constants";
+import Decimal from "decimal.js";
+import { ScoreAggregate } from "../../features/scores";
+import { reduceUsageOrCostDetails } from "../repositories";
 
 export type TracesTableReturnType = Pick<
   TraceRecordReadType,
@@ -30,17 +39,109 @@ export type TracesTableReturnType = Pick<
   | "session_id"
   | "tags"
   | "public"
-> & {
+>;
+
+export type TracesAllUiReturnType = {
+  id: string;
+  timestamp: Date;
+  name: string | null;
+  projectId: string;
+  userId: string | null;
+  release: string | null;
+  version: string | null;
+  public: boolean;
+  bookmarked: boolean;
+  sessionId: string | null;
+  tags: string[];
+};
+
+export type TracesMetricsUiReturnType = {
+  id: string;
+  projectId: string;
+  promptTokens: bigint;
+  completionTokens: bigint;
+  totalTokens: bigint;
+  latency: number | null;
+  level: ObservationLevel;
+  observationCount: bigint;
+  calculatedTotalCost: Decimal | null;
+  calculatedInputCost: Decimal | null;
+  calculatedOutputCost: Decimal | null;
+  scores: ScoreAggregate;
+  usageDetails: Record<string, number>;
+  costDetails: Record<string, number>;
+};
+
+export const convertToUiTableRows = (
+  row: TracesTableReturnType,
+): TracesAllUiReturnType => {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
+    tags: row.tags,
+    bookmarked: row.bookmarked,
+    name: row.name ?? null,
+    release: row.release ?? null,
+    version: row.version ?? null,
+    userId: row.user_id ?? null,
+    sessionId: row.session_id ?? null,
+    public: row.public,
+  };
+};
+
+export const convertToUITableMetrics = (
+  row: TracesTableMetricsClickhouseReturnType,
+): Omit<TracesMetricsUiReturnType, "scores"> => {
+  const usageDetails = reduceUsageOrCostDetails(row.usage_details);
+
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    latency: Number(row.latency),
+    promptTokens: BigInt(usageDetails.input ?? 0),
+    completionTokens: BigInt(usageDetails.output ?? 0),
+    totalTokens: BigInt(usageDetails.total ?? 0),
+    usageDetails: Object.fromEntries(
+      Object.entries(row.usage_details).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
+    costDetails: Object.fromEntries(
+      Object.entries(row.cost_details).map(([key, value]) => [
+        key,
+        Number(value),
+      ]),
+    ),
+    observationCount: BigInt(row.observation_count ?? 0),
+    calculatedTotalCost: row.cost_details?.total
+      ? new Decimal(row.cost_details.total)
+      : null,
+    calculatedInputCost: row.cost_details?.input
+      ? new Decimal(row.cost_details.input)
+      : null,
+    calculatedOutputCost: row.cost_details?.output
+      ? new Decimal(row.cost_details.output)
+      : null,
+    level: row.level,
+  };
+};
+
+export type TracesTableMetricsClickhouseReturnType = {
+  id: string;
+  project_id: string;
+  timestamp: Date;
   level: ObservationLevel;
   observation_count: number | null;
-  latency_milliseconds: string | null;
+  latency: string | null;
   usage_details: Record<string, number>;
   cost_details: Record<string, number>;
   scores_avg: Array<{ name: string; avg_value: number }>;
 };
 
 export type FetchTracesTableProps = {
-  select: string;
+  select: "count" | "rows" | "metrics";
   projectId: string;
   filter: FilterState;
   searchQuery?: string;
@@ -58,7 +159,7 @@ export const getTracesTableCount = async (props: {
   page?: number;
 }) => {
   const countRows = await getTracesTableGeneric<{ count: string }>({
-    select: "count(*) as count",
+    select: "count",
     ...props,
   });
 
@@ -69,6 +170,23 @@ export const getTracesTableCount = async (props: {
   return converted.length > 0 ? converted[0].count : 0;
 };
 
+export const getTracesTableMetrics = async (props: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}): Promise<Array<Omit<TracesMetricsUiReturnType, "scores">>> => {
+  const countRows =
+    await getTracesTableGeneric<TracesTableMetricsClickhouseReturnType>({
+      select: "metrics",
+      ...props,
+    });
+
+  return countRows.map(convertToUITableMetrics);
+};
+
 export const getTracesTable = async (
   projectId: string,
   filter: FilterState,
@@ -78,24 +196,7 @@ export const getTracesTable = async (
   page?: number,
 ) => {
   const rows = await getTracesTableGeneric<TracesTableReturnType>({
-    select: `
-    t.id, 
-    t.project_id as project_id, 
-    t.timestamp, 
-    t.tags, 
-    t.bookmarked, 
-    t.name, 
-    t.release, 
-    t.version, 
-    t.user_id, 
-    t.session_id,
-    os.latency_milliseconds,
-    os.cost_details as cost_details,
-    os.usage_details as usage_details,
-    os.level as level,
-    os.observation_count as observation_count,
-    s.scores_avg as scores_avg,
-    t.public`,
+    select: "rows",
     projectId,
     filter,
     searchQuery,
@@ -104,13 +205,50 @@ export const getTracesTable = async (
     page,
   });
 
-  return rows.map(convertToDomain);
+  return rows.map(convertToUiTableRows);
 };
 
 const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
   const { select, projectId, filter, orderBy, limit, page, searchQuery } =
     props;
 
+  let sqlSelect: string;
+  switch (select) {
+    case "count":
+      sqlSelect = "count(*) as count";
+      break;
+    case "metrics":
+      sqlSelect = `
+        t.id as id,
+        t.project_id as project_id,
+        t.timestamp as timestamp,
+        os.latency_milliseconds / 1000 as latency,
+        os.cost_details as cost_details,
+        os.usage_details as usage_details,
+        os.level as level,
+        os.observation_count as observation_count,
+        s.scores_avg as scores_avg,
+        t.public as public`;
+      break;
+    case "rows":
+      sqlSelect = `
+        t.id as id,
+        t.project_id as project_id,
+        t.timestamp as timestamp,
+        t.tags as tags,
+        t.bookmarked as bookmarked,
+        t.name as name,
+        t.release as release,
+        t.version as version,
+        t.user_id as user_id,
+        t.session_id as session_id,
+        t.public as public`;
+      break;
+    default:
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
 
@@ -156,27 +294,20 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
       f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
   ) as DateTimeFilter | undefined;
 
-  timeStampFilter
-    ? scoresFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "scores",
-          field: "timestamp",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
+  const requiresScoresJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "scores") !== undefined ||
+    tracesTableUiColumnDefinitions.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "scores";
 
-  timeStampFilter
-    ? observationsFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "observations",
-          field: "start_time",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
+  const requiresObservationsJoin =
+    tracesFilter.find((f) => f.clickhouseTable === "observations") !==
+      undefined ||
+    tracesTableUiColumnDefinitions.find(
+      (c) =>
+        c.uiTableName === orderBy?.column || c.uiTableId === orderBy?.column,
+    )?.clickhouseTableName === "observations";
 
   const tracesFilterRes = tracesFilter.apply();
   const scoresFilterRes = scoresFilter.apply();
@@ -184,13 +315,43 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
 
   const search = clickhouseSearchCondition(searchQuery);
 
+  const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
+  const orderByCols = [
+    ...tracesTableUiColumnDefinitions,
+    {
+      clickhouseSelect: "toDate(t.timestamp)",
+      uiTableName: "timestamp_to_date",
+      uiTableId: "timestamp_to_date",
+      clickhouseTableName: "observations",
+    },
+  ];
+  const chOrderBy = orderByToClickhouseSql(
+    [
+      defaultOrder
+        ? {
+            column: "timestamp_to_date",
+            order: orderBy.order,
+          }
+        : null,
+      orderBy ?? null,
+    ],
+    orderByCols,
+  );
+
+  // complex query ahead:
+  // - we only join scores and observations if we really need them to speed up default views
+  // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
+  // - we filter the observations and scores as much as possible before joining them to traces.
+  // - we order by todate(timestamp), timestamp per default and do not use FINAL.
+  //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
+
   const query = `
     WITH observations_stats AS (
       SELECT
         COUNT(*) AS observation_count,
           sumMap(usage_details) as usage_details,
           SUM(total_cost) AS total_cost,
-          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
           multiIf(
             arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
             arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
@@ -200,8 +361,10 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
           sumMap(cost_details) as cost_details,
           trace_id,
           project_id
-      FROM observations FINAL
-      WHERE ${observationFilterRes.query}
+      FROM observations o FINAL 
+      WHERE o.project_id = {projectId: String}
+      ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+      ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
       GROUP BY trace_id, project_id
     ),
     scores_avg AS (
@@ -214,21 +377,28 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
                 trace_id,
                 name,
                 avg(value) avg_value
-        FROM scores final
-        WHERE ${scoresFilterRes.query}
+        FROM scores s FINAL 
+        WHERE project_id = {projectId: String}
+        ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+        ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
         GROUP BY project_id,
                   trace_id,
                   name
       ) tmp
       GROUP BY project_id, trace_id
     )
-    SELECT ${select}
-    FROM traces t final
-    LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
-    LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
-    WHERE ${tracesFilterRes.query}
+    SELECT ${sqlSelect}
+    -- FINAL is used for non default ordering and count.
+    FROM traces t  ${["metrics", "rows"].includes(select) && defaultOrder ? "" : "FINAL"}
+    ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+    ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+    WHERE t.project_id = {projectId: String}
+    ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
     ${search.query}
-    ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
+    ${chOrderBy}
+    -- This is used for metrics and row queries. Count has only one result.
+    -- This is only used for default ordering. Otherwise, we use final.
+    ${["metrics", "rows"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
     ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
   `;
 
@@ -237,6 +407,8 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
     params: {
       limit: limit,
       offset: limit && page ? limit * page : 0,
+      traceTimestamp: timeStampFilter?.value.getTime(),
+      projectId: projectId,
       ...tracesFilterRes.params,
       ...observationFilterRes.params,
       ...scoresFilterRes.params,

packages/shared/src/server/test-utils/clickhouse-helpers.ts

@@ -0,0 +1,32 @@
+import { clickhouseClient } from "../clickhouse/client";
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../repositories/definitions";
+
+export const createTracesCh = async (trace: TraceRecordInsertType[]) => {
+  return await clickhouseClient().insert({
+    table: "traces",
+    format: "JSONEachRow",
+    values: trace,
+  });
+};
+
+export const createObservationsCh = async (
+  observations: ObservationRecordInsertType[],
+) => {
+  return await clickhouseClient().insert({
+    table: "observations",
+    format: "JSONEachRow",
+    values: observations,
+  });
+};
+
+export const createScoresCh = async (scores: ScoreRecordInsertType[]) => {
+  return await clickhouseClient().insert({
+    table: "scores",
+    format: "JSONEachRow",
+    values: scores,
+  });
+};

packages/shared/src/server/test-utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./tracing-factory";
+export * from "./clickhouse-helpers";
+export * from "./org-factory";

packages/shared/src/server/test-utils/org-factory.ts

@@ -0,0 +1,50 @@
+import { v4 } from "uuid";
+import { prisma } from "../../db";
+import { hashSecretKey, getDisplaySecretKey } from "../auth/apiKeys";
+
+export function createBasicAuthHeader(
+  username: string,
+  password: string,
+): string {
+  const base64Credentials = Buffer.from(`${username}:${password}`).toString(
+    "base64",
+  );
+  return `Basic ${base64Credentials}`;
+}
+
+export type CreateOrgProjectAndApiKeyOptions = {
+  projectId?: string;
+};
+export const createOrgProjectAndApiKey = async (
+  props?: CreateOrgProjectAndApiKeyOptions,
+) => {
+  const projectId = props?.projectId ?? v4();
+  const org = await prisma.organization.create({
+    data: {
+      id: v4(),
+      name: v4(),
+    },
+  });
+  await prisma.project.create({
+    data: {
+      id: projectId,
+      name: v4(),
+      orgId: org.id,
+    },
+  });
+  const publicKey = v4();
+  const secretKey = v4();
+
+  const auth = createBasicAuthHeader(publicKey, secretKey);
+  await prisma.apiKey.create({
+    data: {
+      id: v4(),
+      projectId: projectId,
+      publicKey: publicKey,
+      hashedSecretKey: await hashSecretKey(secretKey),
+      displaySecretKey: getDisplaySecretKey(secretKey),
+    },
+  });
+
+  return { projectId, publicKey, secretKey, auth };
+};

packages/shared/src/server/test-utils/tracing-factory.ts

@@ -1,9 +1,9 @@
-import {
-  type ScoreRecordInsertType,
-  type ObservationRecordInsertType,
-  type TraceRecordInsertType,
-} from "@langfuse/shared/src/server";
 import { v4 } from "uuid";
+import {
+  TraceRecordInsertType,
+  ObservationRecordInsertType,
+  ScoreRecordInsertType,
+} from "../repositories/definitions";
 
 export const createTrace = (trace: Partial<TraceRecordInsertType>) => {
   return {

packages/shared/src/server/utils/transforms/transformStreamToCsv.ts

@@ -12,7 +12,7 @@ export function transformStreamToCsv(): Transform {
     transform(
       row: Record<string, any>,
       encoding: BufferEncoding,
-      callback: TransformCallback
+      callback: TransformCallback,
     ): void {
       if (isFirstChunk) {
         // Extract headers from the first object
@@ -23,7 +23,7 @@ export function transformStreamToCsv(): Transform {
 
       // Convert the object to a CSV line and push it
       const csvRow = headers.map((header) => {
-        const field = row[header];
+        const field = row[header] ?? "";
         let str = stringify(field);
 
         // escape and format fields that contain commas

packages/shared/src/tableDefinitions/mapDashboards.ts

@@ -85,4 +85,10 @@ export const dashboardColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "version",
     uiTableName: "Version",
   },
+  {
+    clickhouseTableName: "observations",
+    clickhouseSelect: "provided_model_name",
+    uiTableId: "model",
+    uiTableName: "Model",
+  },
 ];