security: upgrade dompurify v2 (#5570)

security: upgrade dompurify

.env.dev-azure.example

@@ -0,0 +1,92 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 storage
+S3_ENDPOINT=http://localhost:10000/devstoreaccount1
+S3_ACCESS_KEY_ID=devstoreaccount1
+S3_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+S3_BUCKET_NAME=langfuse
+S3_REGION=auto
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=false
+LANGFUSE_RETURN_FROM_CLICKHOUSE=true
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
+
+LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.example

@@ -11,6 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_DISABLED="true"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -42,6 +43,17 @@ S3_REGION=us-east-1
 ## Necessary for minio compatibility
 S3_FORCE_PATH_STYLE=true
 
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
 # S3 Event Bucket Upload
 ## Set to true to test uploading all events to S3
 LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
@@ -66,4 +78,12 @@ REDIS_AUTH="myredissecret"
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=false
+LANGFUSE_RETURN_FROM_CLICKHOUSE=true
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
+
+LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.legacy.example

@@ -0,0 +1,81 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_CLUSTER_ENABLED="false"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# S3 storage
+S3_ENDPOINT=http://localhost:9090
+S3_ACCESS_KEY_ID=minio
+S3_SECRET_ACCESS_KEY=miniosecret
+S3_BUCKET_NAME=langfuse
+S3_REGION=us-east-1
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# # S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
+LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,12 +92,17 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -245,4 +255,4 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion

.github/workflows/codespell.yml

@@ -3,9 +3,14 @@ name: Codespell
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
   pull_request:
-    branches: [main]
+    branches:
+      - "**"
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -66,10 +66,10 @@ jobs:
         run: |
           timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
 
-  tests-web:
+  tests-web-sync:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
         node-version: [20]
@@ -82,7 +82,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -105,7 +105,7 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -131,17 +131,87 @@ jobs:
           LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
           LANGFUSE_INIT_USER_NAME: "Demo User"
           LANGFUSE_INIT_USER_PASSWORD: "password"
+      - name: run test-sync
+        run: pnpm --filter=web run test-sync
+
+  tests-web-async:
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
+    strategy:
+      matrix:
+        node-version: [20]
+        postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
+    steps:
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+          swap-size-gb: 10
+      - uses: actions/checkout@v4
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm install
+      - name: Load default env
+        run: |
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: ${{ matrix.postgres-version }}
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm --filter=shared ch:up
+      - name: Build
+        run: pnpm run build
+      - name: Start Langfuse
+        run: (pnpm run start&)
+        env:
+          LANGFUSE_INIT_ORG_ID: "seed-org-id"
+          LANGFUSE_INIT_ORG_NAME: "Seed Org"
+          LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+          LANGFUSE_INIT_PROJECT_NAME: "Seed Project"
+          LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890"
+          LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890"
+          LANGFUSE_INIT_USER_EMAIL: "demo@langfuse.com"
+          LANGFUSE_INIT_USER_NAME: "Demo User"
+          LANGFUSE_INIT_USER_PASSWORD: "password"
       - name: run tests
         run: pnpm --filter=web run test
 
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -167,17 +237,17 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status
@@ -258,9 +328,15 @@ jobs:
       - name: install dependencies
         run: |
           pnpm install
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
@@ -269,14 +345,29 @@ jobs:
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed:examples
       - name: Build
         run: pnpm run build
       - name: Run server
         run: (pnpm run start&)
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e:server
 
@@ -286,11 +377,12 @@ jobs:
     needs:
       [
         lint,
-        tests-web,
+        tests-web-sync,
         tests-worker,
         e2e-tests,
         test-docker-build,
         e2e-server-tests,
+        tests-web-async,
       ]
     if: always()
     steps:
@@ -349,6 +441,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -356,6 +450,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.gitignore

@@ -37,8 +37,10 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env*
 !.env.dev.example
+!.env.dev-azure.example
 !.env.local.example
 !.env.prod.example
+!.env.dev.legacy.example
 
 # vercel
 .vercel

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.v3preview.yml

@@ -18,9 +18,14 @@ services:
       LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
       LANGFUSE_ASYNC_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_INGESTION_PROCESSING:-true}
       LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: ${LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:-true}
+      LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: ${LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE:-true}
+      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
+      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
+      CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
       CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
       CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
       LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}
@@ -28,6 +33,15 @@ services:
       LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
       LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: ${LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:-http://minio:9000}
       LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_EVENT_UPLOAD_PREFIX: ${LANGFUSE_S3_EVENT_UPLOAD_PREFIX:-events/}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENABLED: ${LANGFUSE_S3_MEDIA_UPLOAD_ENABLED:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: ${LANGFUSE_S3_MEDIA_UPLOAD_BUCKET:-langfuse}
+      LANGFUSE_S3_MEDIA_UPLOAD_REGION: ${LANGFUSE_S3_MEDIA_UPLOAD_REGION:-us-east-1}
+      LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: ${LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID:-minio}
+      LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY: ${LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY:-miniosecret}
+      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
+      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
+      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
       REDIS_HOST: ${REDIS_HOST:-redis}
       REDIS_PORT: ${REDIS_PORT:-6379}
       REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
@@ -98,7 +112,7 @@ services:
     ports:
       - 6379:6379
     healthcheck:
-      test: [ 'CMD', 'redis-cli', 'ping' ]
+      test: ["CMD", "redis-cli", "ping"]
       interval: 3s
       timeout: 10s
       retries: 10
@@ -107,7 +121,7 @@ services:
     image: postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 3s
       timeout: 3s
       retries: 10

ee/package.json

@@ -27,8 +27,9 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "https-proxy-agent": "^7.0.6",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +48,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.2.0"
     }
   }
 }

fern/apis/server/definition/media.yml

@@ -0,0 +1,103 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    get:
+      docs: Get a media record
+      method: GET
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      response: GetMediaResponse
+
+    patch:
+      docs: Patch a media record
+      method: PATCH
+      path: /media/{mediaId}
+      path-parameters:
+        mediaId:
+          type: string
+          docs: The unique langfuse identifier of a media record
+      request: PatchMediaBody
+
+    getUploadUrl:
+      docs: Get a presigned upload URL for a media record
+      method: POST
+      path: /media
+      request: GetMediaUploadUrlRequest
+      response: GetMediaUploadUrlResponse
+
+types:
+  GetMediaResponse:
+    properties:
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+      contentType:
+        type: string
+        docs: The MIME type of the media record
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      url:
+        type: string
+        docs: The download URL of the media record
+      urlExpiry:
+        type: string
+        docs: The expiry date and time of the media record download URL
+
+  PatchMediaBody:
+    properties:
+      uploadedAt:
+        type: datetime
+        docs: The date and time when the media record was uploaded
+      uploadHttpStatus:
+        type: integer
+        docs: The HTTP status code of the upload
+      uploadHttpError:
+        type: optional<string>
+        docs: The HTTP error message of the upload
+      uploadTimeMs:
+        type: optional<integer>
+        docs: The time in milliseconds it took to upload the media record
+
+  GetMediaUploadUrlRequest:
+    properties:
+      traceId:
+        type: string
+        docs: The trace ID associated with the media record
+      observationId:
+        type: optional<string>
+        docs: The observation ID associated with the media record. If the media record is associated directly with a trace, this will be null.
+      contentType: MediaContentType
+      contentLength:
+        type: integer
+        docs: The size of the media record in bytes
+      sha256Hash:
+        type: string
+        docs: The SHA-256 hash of the media record
+      field:
+        type: string
+        docs: The trace / observation field the media record is associated with. This can be one of `input`, `output`, `metadata`
+
+  GetMediaUploadUrlResponse:
+    properties:
+      uploadUrl:
+        type: optional<string>
+        docs: The presigned upload URL. If the asset is already uploaded, this will be null
+      mediaId:
+        type: string
+        docs: The unique langfuse identifier of a media record
+
+  MediaContentType:
+    type: literal<"image/png","image/jpeg","image/jpg","image/webp","audio/mpeg","audio/mp3","audio/wav","text/plain","application/pdf">
+    docs: The MIME type of the media record

fern/apis/server/definition/score.yml

@@ -147,10 +147,29 @@ types:
       tags:
         type: optional<list<string>>
         docs: A list of tags associated with the trace referenced by score
-  GetScoresResponseData:
+
+  GetScoresResponseDataNumeric:
+    extends: commons.NumericScore
     properties:
-      <<: commons.Score
       trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataCategorical:
+    extends: commons.CategoricalScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseDataBoolean:
+    extends: commons.BooleanScore
+    properties:
+      trace: GetScoresResponseTraceData
+
+  GetScoresResponseData:
+    discriminant: dataType
+    union:
+      NUMERIC: GetScoresResponseDataNumeric
+      CATEGORICAL: GetScoresResponseDataCategorical
+      BOOLEAN: GetScoresResponseDataBoolean
+
   GetScoresResponse:
     properties:
       data: list<GetScoresResponseData>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.89.0",
+  "version": "2.95.1",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -83,7 +83,12 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
     }
   },
   "packageManager": "pnpm@9.5.0"

packages/shared/clickhouse/migrations/clustered/0001_traces.down.sql

@@ -0,0 +1 @@
+DROP TABLE traces ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0001_traces.up.sql

@@ -0,0 +1,32 @@
+CREATE TABLE traces ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `name` String,
+    `user_id` Nullable(String),
+    `metadata` Map(LowCardinality(String), String),
+    `release` Nullable(String),
+    `version` Nullable(String),
+    `project_id` String,
+    `public` Bool,
+    `bookmarked` Bool,
+    `tags` Array(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `session_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    updated_at DateTime64(3) DEFAULT now(),
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_metadata_key mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+     project_id,
+     toDate(timestamp)
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/clustered/0002_observations.down.sql

@@ -0,0 +1 @@
+DROP TABLE observations ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0002_observations.up.sql

@@ -0,0 +1,47 @@
+CREATE TABLE observations ON CLUSTER default (
+    `id` String,
+    `trace_id` String,
+    `project_id` String,
+    `type` LowCardinality(String),
+    `parent_observation_id` Nullable(String),
+    `start_time` DateTime64(3),
+    `end_time` Nullable(DateTime64(3)),
+    `name` String,
+    `metadata` Map(LowCardinality(String), String),
+    `level` LowCardinality(String),
+    `status_message` Nullable(String),
+    `version` Nullable(String),
+    `input` Nullable(String) CODEC(ZSTD(3)),
+    `output` Nullable(String) CODEC(ZSTD(3)),
+    `provided_model_name` Nullable(String),
+    `internal_model_id` Nullable(String),
+    `model_parameters` Nullable(String),
+    `provided_usage_details` Map(LowCardinality(String), UInt64),
+    `usage_details` Map(LowCardinality(String), UInt64),
+    `provided_cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `cost_details` Map(LowCardinality(String), Decimal64(12)),
+    `total_cost` Nullable(Decimal64(12)),
+    `completion_start_time` Nullable(DateTime64(3)),
+    `prompt_id` Nullable(String),
+    `prompt_name` Nullable(String),
+    `prompt_version` Nullable(UInt16),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    is_deleted UInt8,
+    INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
+    INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
+PRIMARY KEY (
+    project_id,
+    `type`,
+    toDate(start_time)
+)
+ORDER BY (
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
+

packages/shared/clickhouse/migrations/clustered/0003_scores.down.sql

@@ -0,0 +1 @@
+DROP TABLE scores ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0003_scores.up.sql

@@ -0,0 +1,33 @@
+CREATE TABLE scores ON CLUSTER default (
+    `id` String,
+    `timestamp` DateTime64(3),
+    `project_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+    `name` String,
+    `value` Float64,
+    `source` String,
+    `comment` Nullable(String) CODEC(ZSTD(1)),
+    `author_user_id` Nullable(String),
+    `config_id` Nullable(String),
+    `data_type` String,
+    `string_value` Nullable(String),
+    `queue_id` Nullable(String),
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+    event_ts DateTime64(3),
+    `is_deleted` UInt8,
+    INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
+) ENGINE = ReplicatedReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
+PRIMARY KEY (
+    project_id,
+    toDate(timestamp),
+    name
+)
+ORDER BY (
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default ADD INDEX IF NOT EXISTS idx_project_id project_id TYPE bloom_filter() GRANULARITY 1;

packages/shared/clickhouse/migrations/clustered/0004_drop_observations_index.up.sql

@@ -0,0 +1 @@
+ALTER TABLE observations ON CLUSTER default DROP INDEX IF EXISTS idx_project_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.down.sql

@@ -0,0 +1 @@
+ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/clustered/0005_add_session_id_index.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE traces ON CLUSTER default ADD INDEX IF NOT EXISTS idx_session_id session_id TYPE bloom_filter() GRANULARITY 1;
+ALTER TABLE traces ON CLUSTER default MATERIALIZE INDEX IF EXISTS idx_session_id;

packages/shared/clickhouse/migrations/unclustered/0001_traces.up.sql

@@ -22,11 +22,11 @@ CREATE TABLE traces (
     INDEX idx_res_metadata_value mapValues(metadata) TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
 PRIMARY KEY (
-        project_id,
-        toDate(timestamp)
+     project_id,
+     toDate(timestamp)
 )
 ORDER BY (
-        project_id,
-        toDate(timestamp),
-        id
-    );
+    project_id,
+    toDate(timestamp),
+    id
+);

packages/shared/clickhouse/migrations/unclustered/0002_observations.up.sql

@@ -34,14 +34,14 @@ CREATE TABLE observations (
     INDEX idx_project_id project_id TYPE bloom_filter() GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(start_time)
 PRIMARY KEY (
-        project_id,
-        `type`,
-        toDate(start_time)
-    )
+    project_id,
+    `type`,
+    toDate(start_time)
+)
 ORDER BY (
-        project_id,
-        `type`,
-        toDate(start_time),
-        id
-    );
+    project_id,
+    `type`,
+    toDate(start_time),
+    id
+);
 

packages/shared/clickhouse/migrations/unclustered/0003_scores.up.sql

@@ -21,13 +21,13 @@ CREATE TABLE scores (
     INDEX idx_project_trace_observation (project_id, trace_id, observation_id) TYPE bloom_filter(0.001) GRANULARITY 1
 ) ENGINE = ReplacingMergeTree(event_ts, is_deleted) Partition by toYYYYMM(timestamp)
 PRIMARY KEY (
-        project_id,
-        toDate(timestamp),
-        name
-    )
+    project_id,
+    toDate(timestamp),
+    name
+)
 ORDER BY (
-        project_id,
-        toDate(timestamp),
-        name,
-        id
-    )
+    project_id,
+    toDate(timestamp),
+    name,
+    id
+)

packages/shared/clickhouse/scripts/down.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,10 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" down
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
-# Execute the down command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" down
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
+fi

packages/shared/clickhouse/scripts/seed.ts

@@ -1,21 +1,71 @@
-import { clickhouseClient } from "@langfuse/shared/src/server";
-import { prisma } from "../../src/db";
+import {
+  clickhouseClient,
+  ObservationRecordReadType,
+} from "@langfuse/shared/src/server";
+import { Prisma, prisma } from "../../src/db";
 import { redis } from "@langfuse/shared/src/server";
 import { prepareClickhouse } from "../../scripts/prepareClickhouse";
+import { createDatasets } from "../../prisma/seed";
+import { queryClickhouse } from "../../src/server/repositories/clickhouse";
+import { convertObservation } from "../../src/server/repositories/observations_converters";
 
 async function main() {
   try {
     const projectIds = ["7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"]; // Example project IDs
+    if (
+      await prisma.project.findFirst({
+        where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+      })
+    ) {
+      projectIds.push("239ad00f-562f-411d-af14-831c75ddd875");
+    }
     await prepareClickhouse(projectIds, {
       numberOfDays: 3,
-      totalObservations: 1000,
+      totalObservations: 10000,
     });
 
+    const project1 = await prisma.project.findFirst({
+      where: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
+    });
+
+    const project2 =
+      projectIds.length > 1
+        ? await prisma.project.findFirst({
+            where: { id: "239ad00f-562f-411d-af14-831c75ddd875" },
+          })
+        : await prisma.project.findFirst();
+
+    const query = `
+        SELECT *
+        FROM observations o
+        WHERE o.project_id IN ({projectIds: Array(String)})
+        LIMIT 2000;
+      `;
+
+    const res = await queryClickhouse<ObservationRecordReadType>({
+      query,
+      params: {
+        projectIds,
+      },
+    });
+
+    await createDatasets(
+      project1!,
+      project2!,
+      (await Promise.all(res.map(convertObservation))).map((o) => ({
+        ...o,
+        metadata: {},
+        modelParameters: {},
+        input: {},
+        output: {},
+      })),
+    );
+
     console.log("Clickhouse preparation completed successfully.");
   } catch (error) {
     console.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     console.log("Disconnected from Clickhouse.");

packages/shared/clickhouse/scripts/up.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 
 # Load environment variables
-source ../../.env
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_URL is configured
+if [ -z "${CLICKHOUSE_URL}" ]; then
+  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
+  exit 0
+fi
 
 # Check if golang-migrate is installed
 if ! command -v migrate &> /dev/null
@@ -13,11 +19,22 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
-else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
-fi
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+  fi
 
-# Execute the up command
-migrate -source file://clickhouse/migrations -database "$DATABASE_URL" up
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/clustered -database "$DATABASE_URL" up
+else
+  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+  else
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+  fi
+
+  # Execute the up command
+  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
+fi

packages/shared/package.json

@@ -43,7 +43,6 @@
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
     "db:seed:examples": "dotenv -e ../../.env -- npx prisma db seed -- --environment examples",
     "db:seed:load": "dotenv -e ../../.env -- npx prisma db seed -- --environment load",
-    "ch:status": "dotenv -e ../../.env -- goose -dir './clickhouse/migrations/' status",
     "ch:up": "bash clickhouse/scripts/up.sh",
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
@@ -60,13 +59,14 @@
     "@aws-sdk/client-s3": "^3.675.0",
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.1",
-    "@langchain/aws": "^0.1.0",
-    "@langchain/core": "^0.3.9",
-    "@langchain/openai": "^0.3.0",
+    "@langchain/anthropic": "^0.3.8",
+    "@langchain/aws": "^0.1.2",
+    "@langchain/core": "^0.3.18",
+    "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^5.20.0",
+    "@prisma/client": "^5.22.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
@@ -76,11 +76,13 @@
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.2",
+    "langchain": "^0.3.6",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
@@ -104,7 +106,7 @@
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
     "prettier": "^3.3.3",
-    "prisma": "^5.20.0",
+    "prisma": "^5.22.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
@@ -119,7 +121,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -278,6 +278,8 @@ export type JobExecution = {
     end_time: Timestamp | null;
     error: string | null;
     job_input_trace_id: string | null;
+    job_input_observation_id: string | null;
+    job_input_dataset_item_id: string | null;
     job_output_score_id: string | null;
 };
 export type LlmApiKeys = {
@@ -294,6 +296,20 @@ export type LlmApiKeys = {
     config: unknown | null;
     project_id: string;
 };
+export type Media = {
+    id: string;
+    sha_256_hash: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    uploaded_at: Timestamp | null;
+    upload_http_status: number | null;
+    upload_http_error: string | null;
+    bucket_path: string;
+    bucket_name: string;
+    content_type: string;
+    content_length: string;
+};
 export type MembershipInvitation = {
     id: string;
     email: string;
@@ -354,6 +370,16 @@ export type Observation = {
     completion_start_time: Timestamp | null;
     prompt_id: string | null;
 };
+export type ObservationMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    observation_id: string;
+    field: string;
+};
 export type ObservationView = {
     id: string;
     trace_id: string | null;
@@ -515,6 +541,15 @@ export type Trace = {
     created_at: Generated<Timestamp>;
     updated_at: Generated<Timestamp>;
 };
+export type TraceMedia = {
+    id: string;
+    project_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+    media_id: string;
+    trace_id: string;
+    field: string;
+};
 export type TraceSession = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -579,8 +614,10 @@ export type DB = {
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
     llm_api_keys: LlmApiKeys;
+    media: Media;
     membership_invitations: MembershipInvitation;
     models: Model;
+    observation_media: ObservationMedia;
     observations: Observation;
     observations_view: ObservationView;
     organization_memberships: OrganizationMembership;
@@ -594,6 +631,7 @@ export type DB = {
     scores: Score;
     Session: Session;
     sso_configs: SsoConfig;
+    trace_media: TraceMedia;
     trace_sessions: TraceSession;
     traces: Trace;
     traces_view: TraceView;

packages/shared/prisma/migrations/20241106122605_add_media_tables/migration.sql

@@ -0,0 +1,71 @@
+-- CreateTable
+CREATE TABLE "media" (
+    "id" TEXT NOT NULL,
+    "sha_256_hash" CHAR(44) NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "uploaded_at" TIMESTAMP(3),
+    "upload_http_status" INTEGER,
+    "upload_http_error" TEXT,
+    "bucket_path" TEXT NOT NULL,
+    "bucket_name" TEXT NOT NULL,
+    "content_type" TEXT NOT NULL,
+    "content_length" BIGINT NOT NULL,
+
+    CONSTRAINT "media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "trace_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "trace_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "observation_media" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "media_id" TEXT NOT NULL,
+    "trace_id" TEXT NOT NULL,
+    "observation_id" TEXT NOT NULL,
+    "field" TEXT NOT NULL,
+
+    CONSTRAINT "observation_media_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "media_project_id_sha_256_hash_key" ON "media"("project_id", "sha_256_hash");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "trace_media_project_id_trace_id_media_id_field_key" ON "trace_media"("project_id", "trace_id", "media_id", "field");
+
+-- CreateIndex
+CREATE INDEX "observation_media_project_id_observation_id_idx" ON "observation_media"("project_id", "observation_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "observation_media_project_id_trace_id_observation_id_media__key" ON "observation_media"("project_id", "trace_id", "observation_id", "media_id", "field");
+
+-- AddForeignKey
+ALTER TABLE "media" ADD CONSTRAINT "media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "trace_media" ADD CONSTRAINT "trace_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "observation_media" ADD CONSTRAINT "observation_media_media_id_fkey" FOREIGN KEY ("media_id") REFERENCES "media"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20241114175010_job_executions_add_observation_dataset_item_cols/migration.sql

@@ -0,0 +1,6 @@
+-- DropForeignKey
+ALTER TABLE "job_executions" DROP CONSTRAINT "job_executions_job_input_trace_id_fkey";
+
+-- AlterTable
+ALTER TABLE "job_executions" ADD COLUMN     "job_input_dataset_item_id" TEXT,
+ADD COLUMN     "job_input_observation_id" TEXT;

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/prisma/schema.prisma

@@ -136,6 +136,9 @@ model Project {
   comment             Comment[]
   annotationQueue     AnnotationQueue[]
   annotationQueueItem AnnotationQueueItem[]
+  TraceMedia          TraceMedia[]
+  Media               Media[]
+  ObservationMedia    ObservationMedia[]
 
   @@index([orgId])
   @@map("projects")
@@ -300,8 +303,6 @@ model Trace {
   createdAt  DateTime      @default(now()) @map("created_at")
   updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
 
-  JobExecution JobExecution[]
-
   @@index([projectId, timestamp])
   @@index([sessionId])
   @@index([name])
@@ -895,8 +896,11 @@ model JobExecution {
   endTime   DateTime?          @map("end_time")
   error     String?
 
-  jobInputTraceId String? @map("job_input_trace_id")
-  trace           Trace?  @relation(fields: [jobInputTraceId], references: [id], onDelete: SetNull) // job remains when traces are deleted
+  jobInputTraceId String? @map("job_input_trace_id") // no fk constraint - traces in ClickHouse, deletion handled via project cascade
+
+  jobInputObservationId String? @map("job_input_observation_id") // no fk constraint - observations in ClickHouse, deletion handled via project cascade
+
+  jobInputDatasetItemId String?      @map("job_input_dataset_item_id") // no fk constraint - job execution sensible standalone
 
   jobOutputScoreId String? @map("job_output_score_id")
   score            Score?  @relation(fields: [jobOutputScoreId], references: [id], onDelete: SetNull) // job remains when scores are deleted
@@ -961,3 +965,56 @@ model BatchExport {
   @@index([status])
   @@map("batch_exports")
 }
+
+model Media {
+  id               String             @id @default(cuid())
+  sha256Hash       String             @map("sha_256_hash") @db.Char(44)
+  projectId        String             @map("project_id")
+  project          Project            @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt        DateTime           @default(now()) @map("created_at")
+  updatedAt        DateTime           @default(now()) @updatedAt @map("updated_at")
+  uploadedAt       DateTime?          @map("uploaded_at")
+  uploadHttpStatus Int?               @map("upload_http_status")
+  uploadHttpError  String?            @map("upload_http_error")
+  bucketPath       String             @map("bucket_path")
+  bucketName       String             @map("bucket_name")
+  contentType      String             @map("content_type")
+  contentLength    BigInt             @map("content_length")
+  TraceMedia       TraceMedia[]
+  ObservationMedia ObservationMedia[]
+
+  @@unique([projectId, sha256Hash])
+  @@map("media")
+}
+
+model TraceMedia {
+  id        String   @id @default(cuid())
+  projectId String   @map("project_id")
+  project   Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId   String   @map("media_id")
+  media     Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId   String   @map("trace_id")
+  field     String   @map("field")
+
+  @@unique([projectId, traceId, mediaId, field])
+  @@map("trace_media")
+}
+
+model ObservationMedia {
+  id            String   @id @default(cuid())
+  projectId     String   @map("project_id")
+  project       Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt     DateTime @default(now()) @map("created_at")
+  updatedAt     DateTime @default(now()) @updatedAt @map("updated_at")
+  mediaId       String   @map("media_id")
+  media         Media    @relation(fields: [mediaId], references: [id], onDelete: Cascade)
+  traceId       String   @map("trace_id")
+  observationId String   @map("observation_id")
+  field         String   @map("field")
+
+  @@unique([projectId, traceId, observationId, mediaId, field])
+  @@index([projectId, observationId])
+  @@map("observation_media")
+}

packages/shared/prisma/seed.ts

@@ -387,16 +387,62 @@ async function main() {
       update: {},
     });
 
-    for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
-      const dataset = await prisma.dataset.create({
-        data: {
-          name: `demo-dataset-${datasetNumber}`,
-          description:
-            datasetNumber === 0 ? "Dataset test description" : undefined,
-          projectId: project2.id,
-          metadata: datasetNumber === 0 ? { key: "value" } : undefined,
-        },
-      });
+    await createDatasets(project1, project2, observations);
+  }
+}
+
+main()
+  .then(async () => {
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+  })
+  .catch(async (e) => {
+    logger.error(e);
+    await prisma.$disconnect();
+    redis?.disconnect();
+    logger.info("Disconnected from postgres and redis");
+    process.exit(1);
+  });
+
+export async function createDatasets(
+  project1: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  project2: {
+    id: string;
+    orgId: string;
+    createdAt: Date;
+    updatedAt: Date;
+    name: string;
+  },
+  observations: Prisma.ObservationCreateManyInput[],
+) {
+  for (let datasetNumber = 0; datasetNumber < 2; datasetNumber++) {
+    for (const projectId of [project1.id, project2.id]) {
+      const datasetName = `demo-dataset-${datasetNumber}`;
+
+      // check if ds already exists
+      const dataset =
+        (await prisma.dataset.findFirst({
+          where: {
+            projectId,
+            name: datasetName,
+          },
+        })) ??
+        (await prisma.dataset.create({
+          data: {
+            name: datasetName,
+            description:
+              datasetNumber === 0 ? "Dataset test description" : undefined,
+            projectId,
+            metadata: datasetNumber === 0 ? { key: "value" } : undefined,
+          },
+        }));
 
       const datasetItemIds = [];
       for (let i = 0; i < 18; i++) {
@@ -406,7 +452,7 @@ async function main() {
             : undefined;
         const datasetItem = await prisma.datasetItem.create({
           data: {
-            projectId: project2.id,
+            projectId,
             datasetId: dataset.id,
             sourceTraceId: sourceObservation?.traceId,
             sourceObservationId:
@@ -431,9 +477,16 @@ async function main() {
       }
 
       for (let datasetRunNumber = 0; datasetRunNumber < 5; datasetRunNumber++) {
-        const datasetRun = await prisma.datasetRuns.create({
-          data: {
-            projectId: project2.id,
+        const datasetRun = await prisma.datasetRuns.upsert({
+          where: {
+            datasetId_projectId_name: {
+              datasetId: dataset.id,
+              projectId,
+              name: `demo-dataset-run-${datasetRunNumber}`,
+            },
+          },
+          create: {
+            projectId,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,
@@ -445,11 +498,12 @@ async function main() {
               ["tag1", "tag2"],
             ][datasetRunNumber % 5],
           },
+          update: {},
         });
 
         for (const datasetItemId of datasetItemIds) {
           const relevantObservations = observations.filter(
-            (o) => o.projectId === project2.id,
+            (o) => o.projectId === projectId,
           );
           const observation =
             relevantObservations[
@@ -458,7 +512,7 @@ async function main() {
 
           await prisma.datasetRunItems.create({
             data: {
-              projectId: project2.id,
+              projectId,
               datasetItemId,
               traceId: observation.traceId as string,
               observationId: Math.random() > 0.5 ? observation.id : undefined,
@@ -471,20 +525,6 @@ async function main() {
   }
 }
 
-main()
-  .then(async () => {
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-  })
-  .catch(async (e) => {
-    logger.error(e);
-    await prisma.$disconnect();
-    redis?.disconnect();
-    logger.info("Disconnected from postgres and redis");
-    process.exit(1);
-  });
-
 async function uploadObjects(
   traces: Prisma.TraceCreateManyInput[],
   observations: Prisma.ObservationCreateManyInput[],
@@ -1012,60 +1052,59 @@ async function generatePromptsForProject(projects: Project[]) {
   return promptIds;
 }
 
-async function generatePrompts(project: Project) {
-  const promptIds: string[] = [];
-  const prompts = [
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 1 content",
-      name: "Prompt 1",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 2 content",
-      name: "Prompt 2",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "API",
-      prompt: "Prompt 3 content",
-      name: "Prompt 3 by API",
-      version: 1,
-      labels: ["production", "latest"],
-    },
-    {
-      id: `prompt-${v4()}`,
-      projectId: project.id,
-      createdBy: "user-1",
-      prompt: "Prompt 4 content",
-      name: "Prompt 4",
-      version: 1,
-      labels: ["production", "latest"],
-      tags: ["tag1", "tag2"],
-    },
-  ];
+export const SEED_PROMPTS = [
+  {
+    id: `prompt-123`,
+    createdBy: "user-1",
+    prompt: "Prompt 1 content",
+    name: "Prompt 1",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-456`,
+    createdBy: "user-1",
+    prompt: "Prompt 2 content",
+    name: "Prompt 2",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-789`,
+    createdBy: "API",
+    prompt: "Prompt 3 content",
+    name: "Prompt 3 by API",
+    version: 1,
+    labels: ["production", "latest"],
+  },
+  {
+    id: `prompt-abc`,
+    createdBy: "user-1",
+    prompt: "Prompt 4 content",
+    name: "Prompt 4",
+    version: 1,
+    labels: ["production", "latest"],
+    tags: ["tag1", "tag2"],
+  },
+];
 
-  for (const prompt of prompts) {
+export const PROMPT_IDS: string[] = [];
+
+async function generatePrompts(project: Project) {
+  const promptIds = [];
+  for (const prompt of SEED_PROMPTS) {
     await prisma.prompt.upsert({
       where: {
         projectId_name_version: {
-          projectId: prompt.projectId,
+          projectId: prompt.id + project.id,
           name: prompt.name,
           version: prompt.version,
         },
+        id: prompt.id + project.id,
       },
       create: {
-        id: prompt.id,
-        projectId: prompt.projectId,
+        id: prompt.id + project.id,
+        projectId: project.id,
         createdBy: prompt.createdBy,
         prompt: prompt.prompt,
         name: prompt.name,
@@ -1073,9 +1112,7 @@ async function generatePrompts(project: Project) {
         labels: prompt.labels,
         tags: prompt.tags,
       },
-      update: {
-        id: prompt.id,
-      },
+      update: {},
     });
     promptIds.push(prompt.id);
   }
@@ -1129,6 +1166,7 @@ async function generatePrompts(project: Project) {
           name: version.name,
           version: version.version,
         },
+        id: version.id,
       },
       create: {
         id: version.id,
@@ -1159,6 +1197,7 @@ async function generatePrompts(project: Project) {
           name: promptName,
           version: i,
         },
+        id: promptId,
       },
       create: {
         id: promptId,

packages/shared/scripts/load-seed.ts

@@ -105,10 +105,6 @@ async function main() {
     numberOfDays = 3;
   }
 
-  logger.info(
-    `Preparing Clickhouse for ${numOfProjects} projects and ${numberOfDays} days with max Observations ${totalObservations}.`,
-  );
-
   try {
     const projectIds = [
       "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
@@ -128,7 +124,7 @@ async function main() {
   } catch (error) {
     logger.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     logger.info("Disconnected from Clickhouse.");

packages/shared/scripts/prepareClickhouse.ts

@@ -1,3 +1,4 @@
+import { SEED_PROMPTS } from "../prisma/seed";
 import { prisma } from "../src/db";
 import { clickhouseClient, logger } from "../src/server";
 
@@ -110,9 +111,15 @@ export const prepareClickhouse = async (
       map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
       toDecimal64(randUniform(0, 2000), 12) AS total_cost,
       start_time AS completion_start_time,
-      toString(rand()) AS prompt_id,
-      toString(rand()) AS prompt_name,
-      1000 AS prompt_version,
+      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_id,
+      array(${SEED_PROMPTS.map((p) => `'${p.name}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_name,
+      array(${SEED_PROMPTS.map((p) => `'${p.version}'`).join(
+        ",",
+      )})[(number % ${SEED_PROMPTS.length})+1] AS prompt_version,
       start_time AS created_at,
       start_time AS updated_at,
       start_time AS event_ts,
@@ -120,6 +127,8 @@ export const prepareClickhouse = async (
     FROM numbers(${observationsPerProject});
   `;
 
+    console.log(observationsQuery);
+
     const scoresQuery = `
     INSERT INTO scores
     SELECT toString(floor(randUniform(0, 100))) AS id,
@@ -151,7 +160,7 @@ export const prepareClickhouse = async (
 
     for (const query of queries) {
       logger.info(`Executing query: ${query}`);
-      await clickhouseClient.command({
+      await clickhouseClient().command({
         query,
         clickhouse_settings: {
           wait_end_of_query: 1,
@@ -165,7 +174,7 @@ export const prepareClickhouse = async (
       FROM traces 
       WHERE session_id IS NOT NULL;
     `;
-    const sessionResult = await clickhouseClient.query({
+    const sessionResult = await clickhouseClient().query({
       query: sessionQuery,
       format: "JSONEachRow",
     });
@@ -203,7 +212,7 @@ export const prepareClickhouse = async (
           ORDER BY count() desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });
@@ -234,7 +243,7 @@ export const prepareClickhouse = async (
           ORDER BY event_date desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });

packages/shared/src/env.ts

@@ -30,6 +30,16 @@ const EnvSchema = z.object({
   CLICKHOUSE_URL: z.string().url().optional(),
   CLICKHOUSE_USER: z.string().optional(),
   CLICKHOUSE_PASSWORD: z.string().optional(),
+  LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_ASYNC_INGESTION_PROCESSING: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(15_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -48,6 +58,8 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
 });
 
 export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/features/evals/types.ts

@@ -5,6 +5,7 @@ export const langfuseObjects = [
   "span",
   "generation",
   "event",
+  "dataset_item",
 ] as const;
 
 // variable mapping stored in the db for eval templates
@@ -21,7 +22,7 @@ export const variableMapping = z
     (value) => value.langfuseObject === "trace" || value.objectName !== null,
     {
       message: "objectName is required for langfuseObjects other than trace",
-    }
+    },
   );
 
 export const variableMappingList = z.array(variableMapping);
@@ -44,7 +45,7 @@ const observationCols = [
   { name: "Output", id: "output", internal: 'o."output"' },
 ];
 
-export const availableEvalVariables = [
+export const availableTraceEvalVariables = [
   {
     id: "trace",
     display: "Trace",
@@ -76,6 +77,28 @@ export const availableEvalVariables = [
   },
 ];
 
+export const availableDatasetEvalVariables = [
+  {
+    id: "dataset_item",
+    display: "Dataset item",
+    availableColumns: [
+      {
+        name: "Metadata",
+        id: "metadata",
+        type: "stringObject",
+        internal: 'd."metadata"',
+      },
+      { name: "Input", id: "input", internal: 'd."input"' },
+      {
+        name: "Expected output",
+        id: "expected_output",
+        internal: 'd."expected_output"',
+      },
+    ],
+  },
+  ...availableTraceEvalVariables,
+];
+
 export const OutputSchema = z.object({
   reasoning: z.string(),
   score: z.string(),
@@ -83,6 +106,7 @@ export const OutputSchema = z.object({
 
 export enum EvalTargetObject {
   Trace = "trace",
+  Dataset = "dataset",
 }
 
 export const DEFAULT_TRACE_JOB_DELAY = 10_000;

packages/shared/src/index.ts

@@ -11,6 +11,7 @@ export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";

packages/shared/src/interfaces/filters.ts

@@ -15,6 +15,7 @@ export const filterOperators = {
   ],
   numberObject: ["=", ">", "<", ">=", "<="],
   boolean: ["=", "<>"],
+  null: ["is null", "is not null"],
 } as const;
 
 export const timeFilter = z.object({
@@ -68,6 +69,12 @@ export const booleanFilter = z.object({
   operator: z.enum(filterOperators.boolean),
   value: z.boolean(),
 });
+export const nullFilter = z.object({
+  type: z.literal("null"),
+  column: z.string(),
+  operator: z.enum(filterOperators.null),
+  value: z.literal(""),
+});
 export const singleFilter = z.discriminatedUnion("type", [
   timeFilter,
   stringFilter,
@@ -77,4 +84,5 @@ export const singleFilter = z.discriminatedUnion("type", [
   stringObjectFilter,
   numberObjectFilter,
   booleanFilter,
+  nullFilter,
 ]);

packages/shared/src/server/auth/customSsoProvider.ts

@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
     authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
-    idToken: true,
     profile(profile) {
       return {
         id: profile.sub,

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/clickhouse/client.ts

@@ -1,24 +1,28 @@
 import { createClient } from "@clickhouse/client";
 import { env } from "../../env";
+import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
 export type ClickhouseClientType = ReturnType<typeof createClient>;
 
-export const clickhouseClient = createClient({
-  url: env.CLICKHOUSE_URL,
-  username: env.CLICKHOUSE_USER,
-  password: env.CLICKHOUSE_PASSWORD,
-  database: "default",
-  clickhouse_settings: {
-    async_insert: 1,
-    wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
-  },
-});
+export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
+  createClient({
+    ...opts,
+    url: env.CLICKHOUSE_URL,
+    username: env.CLICKHOUSE_USER,
+    password: env.CLICKHOUSE_PASSWORD,
+    database: "default",
+    clickhouse_settings: {
+      async_insert: 1,
+      wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
+    },
+  });
 
+export const defaultClickhouseClient = clickhouseClient();
 /**
  * Accepts a JavaScript date and returns the DateTime in format YYYY-MM-DD HH:MM:SS
  */
 
 export const convertDateToClickhouseDateTime = (date: Date): string => {
-  // 2024-11-06T20:37:00.000Z -> 2024-11-06 21:37:00
-  return date.toISOString().slice(0, 19).replace("T", " ");
+  // 2024-11-06T20:37:00.123Z -> 2024-11-06 21:37:00.123
+  return date.toISOString().replace("T", " ").replace("Z", "");
 };

packages/shared/src/server/filterToPrisma.ts

@@ -106,10 +106,12 @@ export function tableColumnsToSqlFilter(
           ", ",
         )}] `;
         break;
-
       case "boolean":
         valuePrisma = Prisma.sql`${filter.value}`;
         break;
+      case "null":
+        valuePrisma = Prisma.sql``;
+        break;
     }
     const jsonKeyPrisma =
       filter.type === "stringObject" || filter.type === "numberObject"

packages/shared/src/server/index.ts

@@ -1,10 +1,12 @@
-export * from "./services/S3StorageService";
+export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
+export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -15,13 +17,18 @@ export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
 export * from "./ingestion/modelMatch";
+export * from "./ingestion/processEventBatch";
 export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
+export * from "./redis/CloudUsageMeteringQueue";
+export * from "./redis/getQueue";
+export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
 export * from "./ingestion/legacy/index";
 export * from "./queues";
@@ -32,3 +39,4 @@ export * from "./instrumentation";
 export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
+export * from "./redis/evalExecutionQueue";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -20,6 +20,9 @@ import { jsonSchema } from "../../../utils/zod";
 import { prisma } from "../../../db";
 import { LegacyIngestionAccessScope } from ".";
 import { logger } from "../../logger";
+import { env } from "../../../env";
+import { upsertTrace } from "../../repositories";
+import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
 
 export interface EventProcessor {
   auth(apiScope: LegacyIngestionAccessScope): void;
@@ -131,19 +134,6 @@ export class ObservationProcessor implements EventProcessor {
           })
         : undefined;
 
-    const traceId =
-      !this.event.body.traceId && !existingObservation
-        ? // Create trace if no traceid
-          (
-            await prisma.trace.create({
-              data: {
-                projectId: apiScope.projectId,
-                name: this.event.body.name,
-              },
-            })
-          ).id
-        : this.event.body.traceId;
-
     // Token counts
     const [newInputCount, newOutputCount] =
       "usage" in this.event.body
@@ -230,11 +220,46 @@ export class ObservationProcessor implements EventProcessor {
         return newId;
       })();
 
+    let traceId = this.event.body?.traceId;
+    if (!this.event.body.traceId && !existingObservation) {
+      // Create trace if no traceId
+      traceId = observationId;
+
+      // Insert trace into postgres
+      await prisma.trace.upsert({
+        where: {
+          id: observationId,
+        },
+        create: {
+          projectId: apiScope.projectId,
+          name: this.event.body.name,
+          id: observationId,
+          timestamp: this.event.body.startTime || new Date(),
+        },
+        update: {},
+      });
+
+      if (env.CLICKHOUSE_URL) {
+        // Insert trace into clickhouse if enabled
+        await upsertTrace({
+          id: observationId,
+          project_id: apiScope.projectId,
+          timestamp: convertDateToClickhouseDateTime(
+            this.event.body.startTime
+              ? new Date(this.event.body.startTime)
+              : new Date(),
+          ),
+          created_at: convertDateToClickhouseDateTime(new Date()),
+          updated_at: convertDateToClickhouseDateTime(new Date()),
+        });
+      }
+    }
+
     return {
       id: observationId,
       create: {
         id: observationId,
-        traceId: traceId,
+        traceId,
         type: type,
         name: this.event.body.name,
         startTime: this.event.body.startTime
@@ -600,19 +625,32 @@ export class TraceProcessor implements EventProcessor {
           : undefined;
 
     if (body.sessionId) {
-      await prisma.traceSession.upsert({
-        where: {
-          id_projectId: {
+      try {
+        await prisma.traceSession.upsert({
+          where: {
+            id_projectId: {
+              id: body.sessionId,
+              projectId: apiScope.projectId,
+            },
+          },
+          create: {
             id: body.sessionId,
             projectId: apiScope.projectId,
           },
-        },
-        create: {
-          id: body.sessionId,
-          projectId: apiScope.projectId,
-        },
-        update: {},
-      });
+          update: {},
+        });
+      } catch (e) {
+        if (
+          e instanceof Prisma.PrismaClientKnownRequestError &&
+          e.code === "P2002"
+        ) {
+          logger.warn(
+            `Failed to upsert session. Session ${body.sessionId} in project ${apiScope.projectId} already exists`,
+          );
+        } else {
+          throw e;
+        }
+      }
     }
 
     // Do not use nested upserts or multiple where conditions as this should be a single native database upsert

packages/shared/src/server/ingestion/legacy/index.ts

@@ -3,13 +3,7 @@ import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
 import { getProcessorForEvent } from "./EventProcessor";
-import { TraceUpsertEventType } from "../../queues";
-import {
-  convertTraceUpsertEventsToRedisEvents,
-  TraceUpsertQueue,
-} from "../../redis/traceUpsert";
 import { ApiAccessScope } from "../../auth/types";
-import { redis } from "../../redis/redis";
 import { backOff } from "exponential-backoff";
 import { Model } from "../../..";
 import { logger } from "../../logger";
@@ -163,42 +157,5 @@ export function cleanEvent(obj: unknown): unknown {
   }
 }
 
-export const isNotNullOrUndefined = <T>(
-  val?: T | null,
-): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
-
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
-
-export const addTracesToTraceUpsertQueue = async (
-  batchResults: BatchResult[],
-  projectId: string,
-): Promise<void> => {
-  const traceEvents: TraceUpsertEventType[] = batchResults
-    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
-    .map((result) =>
-      result.result &&
-      typeof result.result === "object" &&
-      "id" in result.result
-        ? // ingestion API only gets traces for one projectId
-          { traceId: result.result.id as string, projectId }
-        : null,
-    )
-    .filter(isNotNullOrUndefined);
-
-  try {
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
-
-      const queue = TraceUpsertQueue.getInstance();
-      if (!queue) {
-        logger.error("TraceUpsertQueue not initialized");
-        return;
-      }
-
-      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    }
-  } catch (error) {
-    logger.error("Error sending events to worker", error);
-  }
-};

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -0,0 +1,396 @@
+import { randomUUID } from "crypto";
+import { z } from "zod";
+
+import { type Model } from "../../db";
+import { env } from "../../env";
+import {
+  InvalidRequestError,
+  LangfuseNotFoundError,
+  UnauthorizedError,
+} from "../../errors";
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
+import {
+  getCurrentSpan,
+  instrumentAsync,
+  instrumentSync,
+  recordIncrement,
+  traceException,
+} from "../instrumentation";
+import { logger } from "../logger";
+import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { IngestionQueue } from "../redis/ingestionQueue";
+import { LegacyIngestionQueue } from "../redis/legacyIngestion";
+import { redis } from "../redis/redis";
+import { handleBatch } from "./legacy";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
+import { getProcessorForEvent } from "./legacy/EventProcessor";
+import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+
+export type TokenCountDelegate = (p: {
+  model: Model;
+  text: unknown;
+}) => number | undefined;
+
+let s3StorageServiceClient: StorageService;
+
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
+  }
+  return s3StorageServiceClient;
+};
+
+export const processEventBatch = async (
+  input: unknown[],
+  authCheck: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): Promise<{
+  successes: { id: string; status: number }[];
+  errors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[];
+}> => {
+  // add context of api call to the span
+  const currentSpan = getCurrentSpan();
+  recordIncrement("langfuse.ingestion.event", input.length);
+  currentSpan?.setAttribute("event_count", input.length);
+
+  /**************
+   * VALIDATION *
+   **************/
+  const validationErrors: { id: string; error: unknown }[] = [];
+  const authenticationErrors: { id: string; error: unknown }[] = [];
+
+  const batch: z.infer<typeof ingestionEvent>[] = input
+    .flatMap((event) => {
+      const parsed = instrumentSync(
+        { name: "ingestion-zod-parse-individual-event" },
+        (span) => {
+          const parsedBody = ingestionEvent.safeParse(event);
+          if (parsedBody.data?.id !== undefined) {
+            span.setAttribute("object.id", parsedBody.data.id);
+          }
+          return parsedBody;
+        },
+      );
+      if (!parsed.success) {
+        validationErrors.push({
+          id:
+            typeof event === "object" && event && "id" in event
+              ? typeof event.id === "string"
+                ? event.id
+                : "unknown"
+              : "unknown",
+          error: new InvalidRequestError(parsed.error.message),
+        });
+        return [];
+      }
+      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+        authenticationErrors.push({
+          id: parsed.data.id,
+          error: new UnauthorizedError("Access Scope Denied"),
+        });
+        return [];
+      }
+      return [parsed.data];
+    })
+    .flatMap((event) => {
+      if (event.type === eventTypes.SDK_LOG) {
+        // Log SDK_LOG events, but remove them from further processing
+        logger.info("SDK Log Event", { event });
+        return [];
+      }
+      return [event];
+    });
+
+  const sortedBatch = sortBatch(batch);
+
+  // We group events by eventBodyId which allows us to store and process them
+  // as one which reduces infra interactions per event. Only used in the S3 case.
+  const sortedBatchByEventBodyId = sortedBatch.reduce(
+    (
+      acc: Record<
+        string,
+        {
+          data: IngestionEventType[];
+          key: string;
+          eventBodyId: string;
+          type: (typeof eventTypes)[keyof typeof eventTypes];
+        }
+      >,
+      event,
+    ) => {
+      if (!event.body?.id) {
+        return acc;
+      }
+      const key = `${getClickhouseEntityType(event.type)}-${event.body.id}`;
+      if (!acc[key]) {
+        acc[key] = {
+          data: [],
+          key: event.id,
+          type: event.type,
+          eventBodyId: event.body.id,
+        };
+      }
+      acc[key].data.push(event);
+      return acc;
+    },
+    {},
+  );
+
+  /********************
+   * ASYNC PROCESSING *
+   ********************/
+  let s3UploadErrored = false;
+  if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
+    await instrumentAsync({ name: "s3-upload-events" }, async () => {
+      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
+        throw new Error("S3 event store is enabled but no bucket is set");
+      }
+      const s3Client = getS3StorageServiceClient(
+        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+      );
+      // S3 Event Upload is currently blocking, but non-failing.
+      // If a promise rejects, we log it below, but do not throw an error.
+      // In this case, we upload the full batch into the Redis queue.
+      const results = await Promise.allSettled(
+        Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+          // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+          // That way we batch updates from the same invocation into a single file and reduce
+          // write operations on S3.
+          const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+          return s3Client.uploadJson(
+            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+            data,
+          );
+        }),
+      );
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          s3UploadErrored = true;
+          logger.error("Failed to upload event to S3", {
+            error: result.reason,
+          });
+        }
+      });
+    });
+  }
+
+  // Send each event individually to IngestionQueue for new processing
+  if (
+    env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" &&
+    env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" &&
+    env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING === "true" &&
+    redis &&
+    !s3UploadErrored
+  ) {
+    const queue = IngestionQueue.getInstance();
+    const results = await Promise.allSettled(
+      Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+        queue
+          ? queue.add(
+              QueueJobs.IngestionJob,
+              {
+                id: randomUUID(),
+                timestamp: new Date(),
+                name: QueueJobs.IngestionJob as const,
+                payload: {
+                  data: {
+                    type: sortedBatchByEventBodyId[id].type,
+                    eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                  },
+                  authCheck,
+                },
+              },
+              {
+                delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+              },
+            )
+          : Promise.reject("Failed to instantiate queue"),
+      ),
+    );
+    results.forEach((result) => {
+      if (result.status === "rejected") {
+        logger.error("Failed to add event to IngestionQueue", {
+          error: result.reason,
+        });
+      }
+    });
+  }
+
+  // As part of the legacy processing we sent the entire batch to the worker.
+  if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
+    const queue = LegacyIngestionQueue.getInstance();
+
+    if (queue) {
+      let addToQueueFailed = false;
+
+      const queuePayload: LegacyIngestionEventType =
+        env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" && !s3UploadErrored
+          ? {
+              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
+                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+                return {
+                  type,
+                  eventBodyId,
+                  eventId: key,
+                };
+              }),
+              authCheck,
+              useS3EventStore: true,
+            }
+          : { data: sortedBatch, authCheck, useS3EventStore: false };
+
+      try {
+        await queue.add(QueueJobs.LegacyIngestionJob, {
+          payload: queuePayload,
+          id: randomUUID(),
+          timestamp: new Date(),
+          name: QueueJobs.LegacyIngestionJob as const,
+        });
+      } catch (e: unknown) {
+        logger.warn(
+          "Failed to add batch to queue, falling back to sync processing",
+          e,
+        );
+        addToQueueFailed = true;
+      }
+
+      if (!addToQueueFailed) {
+        return aggregateBatchResult(
+          // we are not sending additional server errors to the client in case of early return
+          [...validationErrors, ...authenticationErrors],
+          sortedBatch.map((event) => ({ id: event.id, result: event })),
+        );
+      }
+    } else {
+      logger.error(
+        "Ingestion queue not initialized, falling back to sync processing",
+      );
+    }
+  }
+
+  /*******************
+   * SYNC PROCESSING *
+   *******************/
+  const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
+
+  //  in case we did not return early, we return the result here
+  return aggregateBatchResult(
+    [...validationErrors, ...authenticationErrors, ...result.errors],
+    result.results,
+  );
+};
+
+const isAuthorized = (
+  event: IngestionEventType,
+  authScope: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): boolean => {
+  try {
+    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+    return true;
+  } catch (error) {
+    return false;
+  }
+};
+
+/**
+ * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
+ */
+const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+  const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
+    eventTypes.GENERATION_UPDATE,
+    eventTypes.SPAN_UPDATE,
+    eventTypes.OBSERVATION_UPDATE, // legacy event type
+  ];
+  const updates = batch
+    .filter((event) => updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+  const others = batch
+    .filter((event) => !updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+
+  // Return the array with non-update events first, followed by update events
+  return [...others, ...updates];
+};
+
+export const aggregateBatchResult = (
+  errors: Array<{ id: string; error: unknown }>,
+  results: Array<{ id: string; result: unknown }>,
+) => {
+  const returnedErrors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[] = [];
+
+  const successes: {
+    id: string;
+    status: number;
+  }[] = [];
+
+  errors.forEach((error) => {
+    if (error.error instanceof InvalidRequestError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 400,
+        message: "Invalid request data",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof UnauthorizedError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 401,
+        message: "Authentication error",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof LangfuseNotFoundError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 404,
+        message: "Resource not found",
+        error: error.error.message,
+      });
+    } else {
+      returnedErrors.push({
+        id: error.id,
+        status: 500,
+        error: "Internal Server Error",
+      });
+    }
+  });
+
+  if (returnedErrors.length > 0) {
+    traceException(errors);
+    logger.error("Error processing events", returnedErrors);
+  }
+
+  results.forEach((result) => {
+    successes.push({
+      id: result.id,
+      status: 201,
+    });
+  });
+
+  return { successes, errors: returnedErrors };
+};

packages/shared/src/server/instrumentation/index.ts

@@ -1,10 +1,10 @@
-import * as opentelemetry from "@opentelemetry/api";
-import * as dd from "dd-trace";
-import { env } from "../../env";
 import {
   CloudWatchClient,
   PutMetricDataCommand,
 } from "@aws-sdk/client-cloudwatch";
+import * as opentelemetry from "@opentelemetry/api";
+import * as dd from "dd-trace";
+import { env } from "../../env";
 import { logger } from "../logger";
 
 // type CallbackFn<T> = () => T;

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,5 +1,7 @@
 import type { ZodSchema } from "zod";
 
+import { CallbackHandler } from "langfuse-langchain";
+
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
@@ -17,11 +19,27 @@ import {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import {
+  processEventBatch,
+  type TokenCountDelegate,
+} from "../ingestion/processEventBatch";
+import { logger } from "../logger";
 import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
 
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
+type ProcessTracedEvents = () => Promise<void>;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};
+
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
@@ -31,6 +49,7 @@ type LLMCompletionParams = {
   apiKey: string;
   maxRetries?: number;
   config?: Record<string, string> | null;
+  traceParams?: TraceParams;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -40,25 +59,34 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
-  }
-): Promise<IterableReadableStream<Uint8Array>>;
+  },
+): Promise<{
+  completion: IterableReadableStream<Uint8Array>;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
-  }
-): Promise<string>;
+  },
+): Promise<{ completion: string; processTracedEvents: ProcessTracedEvents }>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
     structuredOutputSchema: ZodSchema;
-  }
-): Promise<unknown>;
+  },
+): Promise<{
+  completion: unknown;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
-  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
+  params: FetchLLMCompletionParams,
+): Promise<{
+  completion: string | IterableReadableStream<Uint8Array> | unknown;
+  processTracedEvents: ProcessTracedEvents;
+}> {
   // the apiKey must never be printed to the console
   const {
     messages,
@@ -69,8 +97,39 @@ export async function fetchLLMCompletion(
     baseURL,
     maxRetries,
     config,
+    traceParams,
   } = params;
 
+  let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
+  let processTracedEvents: ProcessTracedEvents = () => Promise.resolve();
+
+  if (traceParams) {
+    const handler = new CallbackHandler({
+      _projectId: traceParams.projectId,
+      _isLocalEventExportEnabled: true,
+      tags: traceParams.tags,
+    });
+
+    finalCallbacks.push(handler);
+
+    processTracedEvents = async () => {
+      try {
+        const events = await handler.langfuse._exportLocalEvents(
+          traceParams.projectId,
+        );
+        await processEventBatch(
+          JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
+          traceParams.authCheck,
+          traceParams.tokenCountDelegate,
+        );
+      } catch (e) {
+        logger.error("Failed to process traced events", { error: e });
+      }
+    };
+  }
+
+  finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
+
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -89,7 +148,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       clientOptions: { maxRetries },
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
@@ -99,7 +158,8 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      streamUsage: false, // https://github.com/langchain-ai/langchainjs/issues/6533
+      callbacks: finalCallbacks,
       maxRetries,
       configuration: {
         baseURL,
@@ -114,7 +174,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
@@ -128,7 +188,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
     });
   } else {
@@ -137,10 +197,19 @@ export async function fetchLLMCompletion(
     throw new Error("This model provider is not supported.");
   }
 
+  const runConfig = {
+    callbacks: finalCallbacks,
+    runId: traceParams?.traceId,
+    runName: traceParams?.traceName,
+  };
+
   if (params.structuredOutputSchema) {
-    return await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-      .withStructuredOutput(params.structuredOutputSchema)
-      .invoke(finalMessages);
+    return {
+      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+        .withStructuredOutput(params.structuredOutputSchema)
+        .invoke(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
   /*
@@ -156,27 +225,41 @@ export async function fetchLLMCompletion(
   Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
   */
   if (modelParams.model.startsWith("o1-")) {
-    return await new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      temperature: 1,
-      maxTokens: undefined,
-      topP: undefined,
-      callbacks,
-      maxRetries,
-      configuration: {
-        baseURL,
-      },
-    })
-      .pipe(new StringOutputParser())
-      .invoke(
-        finalMessages.filter((message) => message._getType() !== "system")
-      );
+    return {
+      completion: await new ChatOpenAI({
+        openAIApiKey: apiKey,
+        modelName: modelParams.model,
+        temperature: 1,
+        maxTokens: undefined,
+        topP: undefined,
+        callbacks,
+        maxRetries,
+        configuration: {
+          baseURL,
+        },
+      })
+        .pipe(new StringOutputParser())
+        .invoke(
+          finalMessages.filter((message) => message._getType() !== "system"),
+          runConfig,
+        ),
+      processTracedEvents,
+    };
   }
 
   if (streaming) {
-    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
+    return {
+      completion: await chatModel
+        .pipe(new BytesOutputParser())
+        .stream(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
-  return await chatModel.pipe(new StringOutputParser()).invoke(finalMessages);
+  return {
+    completion: await chatModel
+      .pipe(new StringOutputParser())
+      .invoke(finalMessages, runConfig),
+    processTracedEvents,
+  };
 }

packages/shared/src/server/llm/types.ts

@@ -26,6 +26,20 @@ export enum ChatMessageRole {
 
 export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
 
+const ChatMessageSchema = z.object({
+  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+
+export const PromptContentSchema = z.union([
+  ChatMessageListSchema,
+  TextPromptSchema,
+]);
+export type PromptContent = z.infer<typeof PromptContentSchema>;
+
 export type ModelParams = {
   provider: string;
   adapter: LLMAdapter;
@@ -49,6 +63,17 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
+// Experiment config
+export const ExperimentMetadataSchema = z
+  .object({
+    prompt_id: z.string(),
+    provider: z.string(),
+    model: z.string(),
+    model_params: ZodModelConfig,
+  })
+  .strict();
+export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
+
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -1,20 +1,13 @@
 import { filterOperators } from "../../../interfaces/filters";
+import { clickhouseCompliantRandomCharacters } from "../../repositories";
 
-function randomCharacters() {
-  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-  let result = "";
-  const randomArray = new Uint8Array(5);
-  crypto.getRandomValues(randomArray);
-  randomArray.forEach((number) => {
-    result += chars[number % chars.length];
-  });
-  return result;
-}
-
+export type ClickhouseOperator =
+  | (typeof filterOperators)[keyof typeof filterOperators][number]
+  | "!=";
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
-  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  operator: ClickhouseOperator;
   field: string;
 }
 type ClickhouseFilter = {
@@ -44,7 +37,7 @@ export class StringFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varName = `stringFilter${randomCharacters()}`;
+    const varName = `stringFilter${clickhouseCompliantRandomCharacters()}`;
 
     const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     let query: string;
@@ -79,29 +72,33 @@ export class NumberFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
   public value: number;
-  public operator: (typeof filterOperators)["number"][number];
+  public operator: (typeof filterOperators)["number"][number] | "!=";
+  public clickhouseTypeOverwrite?: string;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["number"][number];
+    operator: (typeof filterOperators)["number"][number] | "!=";
     value: number;
     tablePrefix?: string;
+    clickhouseTypeOverwrite?: string;
   }) {
     this.clickhouseTable = opts.clickhouseTable;
     this.field = opts.field;
     this.value = opts.value;
     this.operator = opts.operator;
     this.tablePrefix = opts.tablePrefix;
+    this.clickhouseTypeOverwrite = opts.clickhouseTypeOverwrite;
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `numberFilter${uid}`;
+    const type = this.clickhouseTypeOverwrite ?? "Decimal64(12)";
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Decimal}`,
-      params: { [varName]: this.value },
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: ${type}}`,
+      params: { [varName]: this.value.toString() },
     };
   }
 }
@@ -128,7 +125,7 @@ export class DateTimeFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `dateTimeFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: DateTime64(3)}`,
@@ -159,7 +156,7 @@ export class StringOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `stringOptionsFilter${uid}`;
     return {
       query:
@@ -198,8 +195,8 @@ export class StringObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `stringObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `stringObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `stringObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `stringObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
 
     //  const query: `${column}['{varKeyName: String}'] ${this.operator} {${varValueName}: String}`,
@@ -254,7 +251,7 @@ export class ArrayOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `arrayOptionsFilter${uid}`;
     let query: string;
 
@@ -266,7 +263,7 @@ export class ArrayOptionsFilter implements Filter {
         query = `hasAny({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`;
         break;
       case "all of":
-        query = `arrayAll(x -> has({${varName}: Array(String)}, x), ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`;
+        query = `hasAll(${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}, {${varName}: Array(String)}) = True`;
         break;
       default:
         throw new Error(`Unsupported operator: ${this.operator}`);
@@ -279,18 +276,44 @@ export class ArrayOptionsFilter implements Filter {
   }
 }
 
-export class NumberObjectFilter implements Filter {
+export class NullFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
-  public key: string;
-  public value: number;
-  public operator: (typeof filterOperators)["numberObject"][number];
+  public operator: (typeof filterOperators)["null"][number];
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["numberObject"][number];
+    operator: (typeof filterOperators)["null"][number];
+    tablePrefix?: string;
+  }) {
+    this.clickhouseTable = opts.clickhouseTable;
+    this.field = opts.field;
+    this.operator = opts.operator;
+    this.tablePrefix = opts.tablePrefix;
+  }
+
+  apply(): ClickhouseFilter {
+    return {
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator}`,
+      params: {},
+    };
+  }
+}
+
+export class NumberObjectFilter implements Filter {
+  public clickhouseTable: string;
+  public field: string;
+  public key: string;
+  public value: number;
+  public operator: (typeof filterOperators)["numberObject"][number] | "!=";
+  protected tablePrefix?: string;
+
+  constructor(opts: {
+    clickhouseTable: string;
+    field: string;
+    operator: (typeof filterOperators)["numberObject"][number] | "!=";
     key: string;
     value: number;
     tablePrefix?: string;
@@ -304,11 +327,11 @@ export class NumberObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `numberObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `numberObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `numberObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `numberObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     return {
-      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal})), ${column})) = 0`,
+      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal64(12)})), ${column})) = 0`,
       params: { [varKeyName]: this.key, [varValueName]: this.value },
     };
   }
@@ -336,7 +359,7 @@ export class BooleanFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `booleanFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Boolean}`,
@@ -348,7 +371,7 @@ export class BooleanFilter implements Filter {
 export class FilterList {
   private filters: Filter[];
 
-  constructor(filters: Filter[]) {
+  constructor(filters: Filter[] = []) {
     this.filters = filters;
   }
 
@@ -360,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  length() {
+    return this.filters.length;
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -14,6 +14,7 @@ import {
   BooleanFilter,
   NumberObjectFilter,
   StringObjectFilter,
+  NullFilter,
 } from "./clickhouse-filter";
 
 export class QueryBuilderError extends Error {
@@ -66,6 +67,7 @@ export const createFilterFromFilterState = (
           operator: frontEndFilter.operator,
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
+          clickhouseTypeOverwrite: column.clickhouseTypeOverwrite,
         });
       case "arrayOptions":
         return new ArrayOptionsFilter({
@@ -101,9 +103,16 @@ export const createFilterFromFilterState = (
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
         });
-
+      case "null":
+        return new NullFilter({
+          clickhouseTable: column.clickhouseTableName,
+          field: column.clickhouseSelect,
+          operator: frontEndFilter.operator,
+          tablePrefix: column.queryPrefix,
+        });
       default:
-        logger.error(`Invalid filter type: ${JSON.stringify(frontEndFilter)}`);
+        const exhaustiveCheck: never = frontEndFilter;
+        logger.error(`Invalid filter type: ${JSON.stringify(exhaustiveCheck)}`);
         throw new QueryBuilderError(`Invalid filter type`);
     }
   });

packages/shared/src/server/queries/createTracesQuery.ts

@@ -11,13 +11,13 @@ export function parseTraceAllFilters(input: TableFilters) {
   const filterCondition = tableColumnsToSqlFilterAndPrefix(
     input.filter ?? [],
     tracesTableCols,
-    "traces"
+    "traces",
   );
   const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
 
   // to improve query performance, add timeseries filter to observation queries as well
   const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime"
+    (f) => f.column === "Timestamp" && f.type === "datetime",
   );
 
   const observationTimeseriesFilter =
@@ -25,7 +25,7 @@ export function parseTraceAllFilters(input: TableFilters) {
       ? datetimeFilterToPrismaSql(
           "start_time",
           timeseriesFilter.operator,
-          timeseriesFilter.value
+          timeseriesFilter.value,
         )
       : Prisma.empty;
 
@@ -130,6 +130,6 @@ export function createTracesQuery({
     ${filterCondition}
   ${orderByCondition}
   ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
+  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
 `;
 }

packages/shared/src/server/queries/index.ts

@@ -7,3 +7,16 @@ export {
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
+export {
+  FilterList,
+  StringFilter,
+  DateTimeFilter,
+  StringOptionsFilter,
+  NumberFilter,
+  ArrayOptionsFilter,
+  BooleanFilter,
+  NumberObjectFilter,
+  StringObjectFilter,
+  NullFilter,
+  type ClickhouseOperator,
+} from "./clickhouse-sql/clickhouse-filter";

packages/shared/src/server/queues.ts

@@ -7,6 +7,7 @@ export enum EventName {
   EvaluationExecution = "EvaluationExecution",
   LegacyIngestion = "LegacyIngestion",
   CloudUsageMetering = "CloudUsageMetering",
+  ExperimentCreate = "ExperimentCreate",
 }
 
 export const LegacyIngestionEventFull = z.object({
@@ -66,17 +67,36 @@ export const TraceUpsertEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
 });
+export const DatasetRunItemUpsertEventSchema = z.object({
+  projectId: z.string(),
+  datasetItemId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().optional(),
+});
 export const EvalExecutionEvent = z.object({
   projectId: z.string(),
   jobExecutionId: z.string(),
   delay: z.number().nullish(),
 });
 
+export const ExperimentCreateEventSchema = z.object({
+  projectId: z.string(),
+  datasetId: z.string(),
+  runId: z.string(),
+  description: z.string().optional(),
+});
+
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
+export type DatasetRunItemUpsertEventType = z.infer<
+  typeof DatasetRunItemUpsertEventSchema
+>;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
 export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
+export type ExperimentCreateEventType = z.infer<
+  typeof ExperimentCreateEventSchema
+>;
 
 export const EventBodySchema = z.union([
   z.object({
@@ -91,26 +111,34 @@ export const EventBodySchema = z.union([
     name: z.literal(EventName.BatchExport),
     payload: BatchExportJobSchema,
   }),
+  z.object({
+    name: z.literal(EventName.ExperimentCreate),
+    payload: ExperimentCreateEventSchema,
+  }),
 ]);
 export type EventBodyType = z.infer<typeof EventBodySchema>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   EvaluationExecution = "evaluation-execution-queue", // Worker executes Evals
+  DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
   LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  ExperimentCreate = "experiment-create-queue",
 }
 
 export enum QueueJobs {
   TraceUpsert = "trace-upsert",
+  DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
   EnqueueBatchExportJobs = "enqueue-batch-export-jobs",
   LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  ExperimentCreateJob = "experiment-create-job",
 }
 
 export type TQueueJobTypes = {
@@ -120,6 +148,12 @@ export type TQueueJobTypes = {
     payload: TraceUpsertEventType;
     name: QueueJobs.TraceUpsert;
   };
+  [QueueName.DatasetRunItemUpsert]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetRunItemUpsertEventType;
+    name: QueueJobs.DatasetRunItemUpsert;
+  };
   [QueueName.EvaluationExecution]: {
     timestamp: Date;
     id: string;
@@ -144,4 +178,10 @@ export type TQueueJobTypes = {
     payload: IngestionEventQueueType;
     name: QueueJobs.IngestionJob;
   };
+  [QueueName.ExperimentCreate]: {
+    timestamp: Date;
+    id: string;
+    payload: ExperimentCreateEventType;
+    name: QueueJobs.ExperimentCreateJob;
+  };
 };

packages/shared/src/server/redis/CloudUsageMeteringQueue.ts

@@ -0,0 +1,61 @@
+import { Queue } from "bullmq";
+import { env } from "../..";
+import { logger } from "@azure/storage-blob";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class CloudUsageMeteringQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudUsageMeteringQueue.instance) {
+      return CloudUsageMeteringQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudUsageMeteringQueue.instance = newRedis
+      ? new Queue(QueueName.CloudUsageMeteringQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudUsageMeteringQueue.instance?.on("error", (err) => {
+      logger.error("CloudUsageMeteringQueue error", err);
+    });
+
+    if (CloudUsageMeteringQueue.instance) {
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {
+          repeat: { pattern: "5 * * * *" },
+        },
+      );
+
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {},
+      );
+    }
+
+    return CloudUsageMeteringQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -0,0 +1,47 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DatasetRunItemUpsertQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetRunItemUpsert]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetRunItemUpsert]
+  > | null {
+    if (DatasetRunItemUpsertQueue.instance)
+      return DatasetRunItemUpsertQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetRunItemUpsertQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetRunItemUpsert]>(
+          QueueName.DatasetRunItemUpsert,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 5,
+              delay: 30_000, // 30 seconds
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetRunItemUpsertQueue.instance?.on("error", (err) => {
+      logger.error("DatasetRunItemUpsertQueue error", err);
+    });
+
+    return DatasetRunItemUpsertQueue.instance;
+  }
+}

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class EvalExecutionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null {
+    if (EvalExecutionQueue.instance) return EvalExecutionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    EvalExecutionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.EvaluationExecution]>(
+          QueueName.EvaluationExecution,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 10,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    EvalExecutionQueue.instance?.on("error", (err) => {
+      logger.error("EvalExecutionQueue error", err);
+    });
+
+    return EvalExecutionQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class ExperimentCreateQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null {
+    if (ExperimentCreateQueue.instance) return ExperimentCreateQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ExperimentCreateQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ExperimentCreate]>(
+          QueueName.ExperimentCreate,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ExperimentCreateQueue.instance?.on("error", (err) => {
+      logger.error("ExperimentCreateQueue error", err);
+    });
+
+    return ExperimentCreateQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -0,0 +1,34 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { BatchExportQueue } from "./batchExport";
+import { CloudUsageMeteringQueue } from "./CloudUsageMeteringQueue";
+import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
+import { EvalExecutionQueue } from "./evalExecutionQueue";
+import { ExperimentCreateQueue } from "./experimentCreateQueue";
+import { IngestionQueue } from "./ingestionQueue";
+import { LegacyIngestionQueue } from "./legacyIngestion";
+import { TraceUpsertQueue } from "./traceUpsert";
+
+export function getQueue(queueName: QueueName): Queue | null {
+  switch (queueName) {
+    case QueueName.LegacyIngestionQueue:
+      return LegacyIngestionQueue.getInstance();
+    case QueueName.BatchExport:
+      return BatchExportQueue.getInstance();
+    case QueueName.CloudUsageMeteringQueue:
+      return CloudUsageMeteringQueue.getInstance();
+    case QueueName.DatasetRunItemUpsert:
+      return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.EvaluationExecution:
+      return EvalExecutionQueue.getInstance();
+    case QueueName.ExperimentCreate:
+      return ExperimentCreateQueue.getInstance();
+    case QueueName.TraceUpsert:
+      return TraceUpsertQueue.getInstance();
+    case QueueName.IngestionQueue:
+      return IngestionQueue.getInstance();
+    default:
+      const exhaustiveCheckDefault: never = queueName;
+      throw new Error(`Queue ${queueName} not found`);
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -32,6 +32,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
+              delay: 10_000, // 10 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,
@@ -48,43 +49,3 @@ export class TraceUpsertQueue {
     return TraceUpsertQueue.instance;
   }
 }
-
-export function convertTraceUpsertEventsToRedisEvents(
-  events: TraceUpsertEventType[],
-) {
-  const uniqueTracesPerProject = events.reduce((acc, event) => {
-    if (!acc.get(event.projectId)) {
-      acc.set(event.projectId, new Set());
-    }
-    acc.get(event.projectId)?.add(event.traceId);
-    return acc;
-  }, new Map<string, Set<string>>());
-
-  return [...uniqueTracesPerProject.entries()]
-    .map((tracesPerProject) => {
-      const [projectId, traceIds] = tracesPerProject;
-
-      return [...traceIds].map((traceId) => ({
-        name: QueueJobs.TraceUpsert,
-        data: {
-          payload: {
-            projectId,
-            traceId,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.TraceUpsert as const,
-        },
-        opts: {
-          removeOnFail: 1_000,
-          removeOnComplete: true,
-          attempts: 5,
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-        },
-      }));
-    })
-    .flat();
-}

packages/shared/src/server/repositories/clickhouse.ts

@@ -5,15 +5,19 @@ import {
 } from "../clickhouse/client";
 import { logger } from "../logger";
 import { instrumentAsync } from "../instrumentation";
-import { S3StorageService } from "../services/S3StorageService";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
+import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -28,7 +32,7 @@ const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces"; // TODO: Modify eventType logic to support more tables going forward
+  table: "scores" | "traces" | "observations";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>;
 }): Promise<void> {
@@ -36,10 +40,6 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // drop trailing s and pretend it's always a create.
-    // Only applicable to scores and traces.
-    const eventType = `${opts.table.slice(0, -1)}-create`;
-
     // If event upload is enabled, we store all rows in S3 to have a backup
     if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
       if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
@@ -50,6 +50,13 @@ export async function upsertClickhouse<
       );
       await Promise.all(
         opts.records.map((record) => {
+          // drop trailing s and pretend it's always a create.
+          // Only applicable to scores and traces.
+          let eventType = `${opts.table.slice(0, -1)}-create`;
+          if (opts.table === "observations") {
+            // @ts-ignore - If it's an observation we now that `type` is a string
+            eventType = `${record["type"].toLowerCase()}-create`;
+          }
           s3Client.uploadJson(
             `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
             [
@@ -65,7 +72,7 @@ export async function upsertClickhouse<
       );
     }
 
-    const res = await clickhouseClient.insert({
+    const res = await clickhouseClient().insert({
       table: opts.table,
       values: opts.records.map((record) => ({
         ...record,
@@ -103,12 +110,13 @@ export async function upsertClickhouse<
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
 }): Promise<T[]> {
   return await instrumentAsync({ name: "clickhouse-query" }, async (span) => {
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.text", opts.query);
 
-    const res = await clickhouseClient.query({
+    const res = await clickhouseClient(opts.clickhouseConfigs).query({
       query: opts.query,
       format: "JSONEachRow",
       query_params: opts.params,
@@ -145,12 +153,12 @@ export async function queryClickhouse<T>(opts: {
 export async function commandClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
 }): Promise<void> {
   return await instrumentAsync({ name: "clickhouse-command" }, async (span) => {
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.text", opts.query);
-
-    const res = await clickhouseClient.command({
+    const res = await clickhouseClient(opts.clickhouseConfigs).command({
       query: opts.query,
       query_params: opts.params,
     });
@@ -184,3 +192,14 @@ export async function commandClickhouse<T>(opts: {
 export function parseClickhouseUTCDateTimeFormat(dateStr: string): Date {
   return new Date(`${dateStr.replace(" ", "T")}Z`);
 }
+
+export function clickhouseCompliantRandomCharacters() {
+  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+  let result = "";
+  const randomArray = new Uint8Array(5);
+  crypto.getRandomValues(randomArray);
+  randomArray.forEach((number) => {
+    result += chars[number % chars.length];
+  });
+  return result;
+}

packages/shared/src/server/repositories/constants.ts

@@ -1,3 +1,5 @@
+// Rule of thumb: If you join observations from left, use observations to trace and vice versa
+
 // t.timestamp > observation.start_time - 2 days
 export const OBSERVATIONS_TO_TRACE_INTERVAL = "INTERVAL 2 DAY";
 // observation.start_time > t.timestamp - 1 hour

packages/shared/src/server/repositories/dashboards.ts

@@ -8,6 +8,7 @@ import {
 import { dashboardColumnDefinitions } from "../../tableDefinitions/mapDashboards";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import {
+  OBSERVATIONS_TO_TRACE_INTERVAL,
   SCORE_TO_TRACE_OBSERVATIONS_INTERVAL,
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
@@ -55,6 +56,7 @@ export const getObservationsCostGroupedByName = async (
   const appliedFilter = chFilter.apply();
 
   const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  // TODO: Validate whether we can filter traces on timestamp here.
 
   const query = `
     SELECT 
@@ -99,6 +101,7 @@ export const getScoreAggregate = async (
   const chFilterApplied = chFilter.apply();
 
   const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  // TODO: Validate whether we can filter traces on timestamp here.
 
   const query = `
     SELECT 
@@ -111,7 +114,7 @@ export const getScoreAggregate = async (
      ${hasTraceFilter ? "JOIN traces t FINAL ON t.id = s.trace_id AND t.project_id = s.project_id" : ""}
     WHERE s.project_id = {projectId: String}
     AND ${chFilterApplied.query}
-    ${timeFilter && hasTraceFilter ? `AND t.timestamp >= {tracesTimestamp: DateTime} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+    ${timeFilter && hasTraceFilter ? `AND t.timestamp >= {tracesTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
     GROUP BY s.name, s.source, s.data_type
     ORDER BY count(*) DESC
     `;
@@ -184,6 +187,16 @@ export const getObservationUsageByTime = async (
 
   const appliedFilter = chFilter.apply();
 
+  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  const timeFilter = tracesFilter
+    ? (chFilter.find(
+        (f) =>
+          f.clickhouseTable === "observations" &&
+          f.field.includes("start_time") &&
+          (f.operator === ">=" || f.operator === ">"),
+      ) as DateTimeFilter | undefined)
+    : undefined;
+
   const query = `
     SELECT 
       ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
@@ -191,9 +204,10 @@ export const getObservationUsageByTime = async (
       sumMap(cost_details)['total'] as sum_cost_details,
       provided_model_name
     FROM observations o FINAL
-    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
+    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY start_time, provided_model_name
     ${orderByTimeSeries(groupBy, "start_time")}
     `;
@@ -208,6 +222,9 @@ export const getObservationUsageByTime = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...(timeFilter
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
     },
   });
 
@@ -229,13 +246,24 @@ export const getDistinctModels = async (
 
   const appliedFilter = chFilter.apply();
 
+  const tracesFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  const timeFilter = tracesFilter
+    ? (chFilter.find(
+        (f) =>
+          f.clickhouseTable === "observations" &&
+          f.field.includes("start_time") &&
+          (f.operator === ">=" || f.operator === ">"),
+      ) as DateTimeFilter | undefined)
+    : undefined;
+
+  // No need for final as duplicates are caught by distinct anyway.
   const query = `
-    SELECT 
-      distinct(provided_model_name) as model
-    FROM observations o FINAL
-    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
+    SELECT distinct(provided_model_name) as model
+    FROM observations o
+    ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
+    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     `;
 
   const result = await queryClickhouse<{ model: string }>({
@@ -243,6 +271,9 @@ export const getDistinctModels = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...(timeFilter
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
     },
   });
 
@@ -261,6 +292,7 @@ export const getScoresAggregateOverTime = async (
   const appliedFilter = chFilter.apply();
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+  // TODO: Validate whether we can filter traces on timestamp here.
 
   const query = `
   SELECT 
@@ -318,7 +350,7 @@ export const getModelUsageByUser = async (
   const timeFilter = chFilter.find(
     (f) =>
       f.clickhouseTable === "observations" &&
-      f.field === "start_time" &&
+      f.field.includes("start_time") &&
       (f.operator === ">=" || f.operator === ">"),
   ) as DateTimeFilter | undefined;
 
@@ -328,11 +360,12 @@ export const getModelUsageByUser = async (
       sumMap(cost_details)['total'] as sum_cost_details,
       user_id
     FROM observations o FINAL
-      JOIN traces t FINAL ON o.trace_id = t.id AND o.project_id = t.project_id
+    JOIN traces t FINAL
+    ON o.trace_id = t.id AND o.project_id = t.project_id
     WHERE project_id = {projectId: String}
     AND t.user_id IS NOT NULL
     AND ${appliedFilter.query}
-    ${timeFilter ? `AND t.timestamp >= {tractTimestamp: DateTime} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY user_id
     ORDER BY sum_cost_details DESC
     `;
@@ -346,7 +379,9 @@ export const getModelUsageByUser = async (
     params: {
       projectId,
       ...appliedFilter.params,
-      ...(timeFilter ? { tractTimestamp: timeFilter.value } : {}),
+      ...(timeFilter
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
+        : {}),
     },
   });
 
@@ -367,11 +402,12 @@ export const getObservationLatencies = async (
 
   const appliedFilter = chFilter.apply();
 
+  // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
     SELECT
-      quantilesExactLow(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
       name
-    FROM observations o FINAL
+    FROM observations o
     ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
@@ -410,14 +446,15 @@ export const getTracesLatencies = async (
       (f.operator === ">=" || f.operator === ">"),
   ) as DateTimeFilter | undefined;
 
+  // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
     WITH trace_latencies as (
       select o.trace_id,
              t.name,
              o.project_id,
              date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
-      FROM traces t FINAL 
-      JOIN observations o FINAL
+      FROM traces t 
+      JOIN observations o
       ON o.trace_id = t.id AND o.project_id = t.project_id
       WHERE project_id = {projectId: String}
       AND ${appliedFilter.query}
@@ -426,7 +463,7 @@ export const getTracesLatencies = async (
     )
 
     SELECT
-      quantilesExactLow(0.5, 0.9, 0.95, 0.99)(duration) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(duration) as quantiles,
       name
     FROM trace_latencies
     GROUP BY name
@@ -466,12 +503,13 @@ export const getModelLatenciesOverTime = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
   SELECT 
     ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
     provided_model_name,
-    quantilesExactLow(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
-  FROM observations o FINAL
+    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+  FROM observations o
   ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
   AND ${appliedFilter.query}
@@ -496,6 +534,88 @@ export const getModelLatenciesOverTime = async (
   }));
 };
 
+export const getNumericScoreTimeSeries = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+    SELECT
+    ${selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp")},
+    s.name as score_name,
+    AVG(s.value) as avg_value
+    FROM scores s final
+    ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
+    WHERE s.project_id = {projectId: String}
+    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
+    GROUP BY score_name, score_timestamp
+    ${orderByTimeSeries(groupBy, "score_timestamp")}
+  `;
+
+  return queryClickhouse<{
+    score_timestamp: Date;
+    score_name: string;
+    avg_value: number;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...(chFilterRes ? chFilterRes.params : {}),
+    },
+  });
+};
+
+export const getCategoricalScoreTimeSeries = async (
+  projectId: string,
+  filter: FilterState,
+  groupBy: DateTrunc | undefined,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
+
+  const query = `
+    SELECT
+    ${groupBy ? selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp") + ", " : ""}
+    s.name as score_name,
+    s.data_type as score_data_type,
+    s.source as score_source,
+    s.string_value as score_value,
+    count(s.string_value) as count
+    FROM scores s final
+    ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
+    WHERE s.project_id = {projectId: String}
+    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
+    GROUP BY score_name, score_data_type, score_source, score_value ${groupBy ? ", score_timestamp" : ""}
+    ${groupBy ? orderByTimeSeries(groupBy, "score_timestamp") : ""}
+  `;
+
+  return queryClickhouse<{
+    score_timestamp?: Date;
+    score_name: string;
+    score_data_type: string;
+    score_source: string;
+    score_value: string;
+    count: number;
+  }>({
+    query,
+    params: {
+      projectId,
+      ...(chFilterRes ? chFilterRes.params : {}),
+    },
+  });
+};
+
 const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
   let interval;
   switch (dateTrunc) {

packages/shared/src/server/repositories/definitions.ts

@@ -129,8 +129,8 @@ export const scoreRecordBaseSchema = z.object({
   project_id: z.string(),
   trace_id: z.string(),
   observation_id: z.string().nullish(),
-  name: z.string().nullish(),
-  value: z.union([z.number(), z.string()]).nullish(),
+  name: z.string(),
+  value: z.number().nullish(),
   source: z.string(),
   comment: z.string().nullish(),
   author_user_id: z.string().nullish(),

packages/shared/src/server/repositories/index.ts

@@ -5,3 +5,5 @@ export * from "./types";
 export * from "./dashboards";
 export * from "./traces_converters";
 export * from "./scores_converters";
+export * from "./observations_converters";
+export * from "./clickhouse";

packages/shared/src/server/repositories/observations.ts

@@ -1,5 +1,10 @@
-import { commandClickhouse, queryClickhouse } from "./clickhouse";
-import { Observation, ObservationLevel } from "@prisma/client";
+import {
+  commandClickhouse,
+  parseClickhouseUTCDateTimeFormat,
+  queryClickhouse,
+  upsertClickhouse,
+} from "./clickhouse";
+import { ObservationLevel } from "@prisma/client";
 import { logger } from "../logger";
 import { InternalServerError, LangfuseNotFoundError } from "../../errors";
 import { prisma } from "../../db";
@@ -10,7 +15,10 @@ import {
   FilterList,
   StringFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
-import { FullObservations } from "../queries/createGenerationsQuery";
+import {
+  FullObservation,
+  FullObservations,
+} from "../queries/createGenerationsQuery";
 import { createFilterFromFilterState } from "../queries/clickhouse-sql/factory";
 import {
   observationsTableTraceUiColumnDefinitions,
@@ -26,11 +34,208 @@ import {
   convertObservation,
 } from "./observations_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
-import { OBSERVATIONS_TO_TRACE_INTERVAL } from "./constants";
+import {
+  OBSERVATIONS_TO_TRACE_INTERVAL,
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+} from "./constants";
+
+export const checkObservationExists = async (
+  projectId: string,
+  id: string,
+  startTime: Date | undefined,
+): Promise<boolean> => {
+  const query = `
+    SELECT id, project_id
+    FROM observations o
+    WHERE project_id = {projectId: String}
+    AND id = {id: String}
+    ${startTime ? `AND start_time >= {startTime: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    ORDER BY event_ts DESC
+    LIMIT 1 BY id, project_id
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
+
+/**
+ * Accepts a trace in a Clickhouse-ready format.
+ * id, project_id, and timestamp must always be provided.
+ */
+export const upsertObservation = async (
+  observation: Partial<ObservationRecordReadType>,
+) => {
+  if (
+    !["id", "project_id", "start_time", "type"].every(
+      (key) => key in observation,
+    )
+  ) {
+    throw new Error(
+      "Identifier fields must be provided to upsert Observation.",
+    );
+  }
+  await upsertClickhouse({
+    table: "observations",
+    records: [observation as ObservationRecordReadType],
+    eventBodyMapper: convertObservation,
+  });
+};
 
 export const getObservationsViewForTrace = async (
   traceId: string,
   projectId: string,
+  timestamp?: Date,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations 
+  WHERE trace_id = {traceId: String}
+  AND project_id = {projectId: String}
+   ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  ORDER BY event_ts DESC
+  LIMIT 1 BY id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: {
+      traceId,
+      projectId,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return records.map(convertObservationToView);
+};
+
+export const getObservationForTraceIdByName = async (
+  traceId: string,
+  projectId: string,
+  name: string,
+  timestamp?: Date,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations 
+  WHERE trace_id = {traceId: String}
+  AND project_id = {projectId: String}
+  AND name = {name: String}
+   ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  ORDER BY event_ts DESC
+  LIMIT 1 BY id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: {
+      traceId,
+      projectId,
+      name,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return records.map(convertObservationToView);
+};
+
+export const getObservationById = async (
+  id: string,
+  projectId: string,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const records = await getObservationByIdInternal(
+    id,
+    projectId,
+    fetchWithInputOutput,
+  );
+  const mapped = records.map(convertObservation);
+
+  if (mapped.length === 0) {
+    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
+  }
+
+  if (mapped.length > 1) {
+    logger.error(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+    throw new InternalServerError(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+  }
+  return mapped.shift();
+};
+
+export const getObservationsById = async (
+  ids: string[],
+  projectId: string,
   fetchWithInputOutput: boolean = false,
 ) => {
   const query = `
@@ -63,18 +268,46 @@ export const getObservationsViewForTrace = async (
     created_at,
     updated_at,
     event_ts
-  FROM observations FINAL WHERE trace_id = {traceId: String} AND project_id = {projectId: String}`;
+  FROM observations
+  WHERE id IN ({ids: Array(String)})
+  AND project_id = {projectId: String}
+  ORDER BY event_ts desc
+  LIMIT 1 by id, project_id`;
   const records = await queryClickhouse<ObservationRecordReadType>({
     query,
-    params: { traceId, projectId },
+    params: { ids, projectId },
   });
-
-  return await Promise.all(
-    records.map(async (o) => await convertObservationToView(o)),
-  );
+  return records.map(convertObservation);
 };
 
-export const getObservationById = async (
+export const getObservationViewById = async (
+  id: string,
+  projectId: string,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const records = await getObservationByIdInternal(
+    id,
+    projectId,
+    fetchWithInputOutput,
+  );
+  const mapped = records.map(convertObservationToView);
+
+  if (mapped.length === 0) {
+    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
+  }
+
+  if (mapped.length > 1) {
+    logger.error(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+    throw new InternalServerError(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+  }
+  return mapped.shift();
+};
+
+const getObservationByIdInternal = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
@@ -109,29 +342,15 @@ export const getObservationById = async (
     created_at,
     updated_at,
     event_ts
-  FROM observations WHERE id = {id: String} AND project_id = {projectId: String} ORDER BY event_ts desc LIMIT 1 by id, project_id`;
-  const records = await queryClickhouse<ObservationRecordReadType>({
+  FROM observations
+  WHERE id = {id: String}
+  AND project_id = {projectId: String}
+  ORDER BY event_ts desc
+  LIMIT 1 by id, project_id`;
+  return await queryClickhouse<ObservationRecordReadType>({
     query,
     params: { id, projectId },
   });
-
-  const mapped = await Promise.all(
-    records.map(async (r) => await convertObservation(r)),
-  );
-
-  if (mapped.length === 0) {
-    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
-  }
-
-  if (mapped.length > 1) {
-    logger.error(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-    throw new InternalServerError(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-  }
-  return mapped.shift() as Observation;
 };
 
 export type ObservationTableQuery = {
@@ -173,8 +392,70 @@ export const getObservationsTableCount = async (opts: ObservationTableQuery) =>
     select: "count(*) as count",
   });
 
+export type ObservationsTableRow = Omit<
+  FullObservation,
+  "modelId" | "inputPrice" | "outputPrice" | "totalPrice"
+>;
+
 export const getObservationsTable = async (
   opts: ObservationTableQuery,
+): Promise<Array<ObservationsTableRow>> => {
+  const observationRecords = await getObservationsTableInternal<
+    Omit<
+      ObservationsTableQueryResult,
+      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+    >
+  >({
+    ...opts,
+    select: `
+        o.id as id,
+        o.name as name,
+        o."model_parameters" as model_parameters,
+        o.start_time as "start_time",
+        o.end_time as "end_time",
+        o.trace_id as "trace_id",
+        o.completion_start_time as "completion_start_time",
+        o.provided_usage_details as "provided_usage_details",
+        o.usage_details as "usage_details",
+        o.provided_cost_details as "provided_cost_details",
+        o.cost_details as "cost_details",
+        o.level as level,
+        o.status_message as "status_message",
+        o.version as version,
+        o.parent_observation_id as "parent_observation_id",
+        o.created_at as "created_at",
+        o.updated_at as "updated_at",
+        o.provided_model_name as "provided_model_name",
+        o.total_cost as "total_cost",
+        internal_model_id as "internal_model_id",
+        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
+  });
+
+  const traces = await getTracesByIds(
+    observationRecords
+      .map((o) => o.trace_id)
+      .filter((o): o is string => Boolean(o)),
+    opts.projectId,
+  );
+
+  return observationRecords.map((o) => {
+    const trace = traces.find((t) => t.id === o.trace_id);
+    return {
+      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      latency: o.latency ? Number(o.latency) / 1000 : null,
+      timeToFirstToken: o.time_to_first_token
+        ? Number(o.time_to_first_token) / 1000
+        : null,
+      traceName: trace?.name ?? null,
+      traceTags: trace?.tags ?? [],
+      userId: trace?.userId ?? null,
+    };
+  });
+};
+
+export const getObservationsTableWithModelData = async (
+  opts: ObservationTableQuery,
 ): Promise<FullObservations> => {
   const observationRecords = await getObservationsTableInternal<
     Omit<
@@ -204,7 +485,6 @@ export const getObservationsTable = async (
         o.provided_model_name as "provided_model_name",
         o.total_cost as "total_cost",
         internal_model_id as "internal_model_id",
-        provided_model_name as "provided_model_name",
         if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
         if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`,
   });
@@ -239,25 +519,27 @@ export const getObservationsTable = async (
     ),
   ]);
 
-  return await Promise.all(
-    observationRecords.map(async (o) => {
-      const model = models.find((p) => p.id === o.internal_model_id);
-      const trace = traces.find((t) => t.id === o.trace_id);
-      return {
-        ...(await convertObservationToView(
-          { ...o, type: "GENERATION" },
-          model,
-        )),
-        latency: o.latency ? Number(o.latency) / 1000 : null,
-        timeToFirstToken: o.time_to_first_token
-          ? Number(o.time_to_first_token) / 1000
-          : null,
-        traceName: trace?.name ?? null,
-        traceTags: trace?.tags ?? [],
-        userId: trace?.userId ?? null,
-      };
-    }),
-  );
+  return observationRecords.map((o) => {
+    const trace = traces.find((t) => t.id === o.trace_id);
+    const model = models.find((m) => m.id === o.internal_model_id);
+    return {
+      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      latency: o.latency ? Number(o.latency) / 1000 : null,
+      timeToFirstToken: o.time_to_first_token
+        ? Number(o.time_to_first_token) / 1000
+        : null,
+      traceName: trace?.name ?? null,
+      traceTags: trace?.tags ?? [],
+      userId: trace?.userId ?? null,
+      modelId: model?.id ?? null,
+      inputPrice:
+        model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
+      outputPrice:
+        model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
+      totalPrice:
+        model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
+    };
+  });
 };
 
 const getObservationsTableInternal = async <T>(
@@ -298,6 +580,15 @@ const getObservationsTableInternal = async <T>(
         .includes(f.column),
   );
 
+  const orderByTraces = opts.orderBy
+    ? observationsTableTraceUiColumnDefinitions
+        .map((c) => c.uiTableId)
+        .includes(opts.orderBy.column) ||
+      observationsTableTraceUiColumnDefinitions
+        .map((c) => c.uiTableName)
+        .includes(opts.orderBy.column)
+    : undefined;
+
   timeFilter
     ? scoresFilter.push(
         new DateTimeFilter({
@@ -359,7 +650,7 @@ const getObservationsTableInternal = async <T>(
       observation_id
   )`;
 
-  if (traceTableFilter.length > 0) {
+  if (traceTableFilter.length > 0 || orderByTraces) {
     // joins with traces are very expensive. We need to filter by time as well.
     // We assume that a trace has to have been within the last 2 days to be relevant.
 
@@ -372,7 +663,7 @@ const getObservationsTableInternal = async <T>(
         LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id
       WHERE ${appliedObservationsFilter.query}
         AND o.type = 'GENERATION'
-        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${timeFilter ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
       ${orderByToClickhouseSql(orderBy ?? null, observationsTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
@@ -441,17 +732,17 @@ export const getObservationsGroupedByModel = async (
 
   const appliedObservationsFilter = observationsFilter.apply();
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-
-    SELECT
-      o.provided_model_name as name
-    FROM observations o FINAL
+    SELECT o.provided_model_name as name
+    FROM observations o
     WHERE ${appliedObservationsFilter.query}
     AND o.type = 'GENERATION'
     GROUP BY o.provided_model_name
     ORDER BY count() DESC
     LIMIT 1000;
-    `;
+  `;
 
   const res = await queryClickhouse<{ name: string }>({
     query,
@@ -485,17 +776,17 @@ export const getObservationsGroupedByName = async (
 
   const appliedObservationsFilter = observationsFilter.apply();
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-
-    SELECT
-      o.name as name
-    FROM observations o FINAL
+    SELECT o.name as name
+    FROM observations o
     WHERE ${appliedObservationsFilter.query}
     AND o.type = 'GENERATION'
     GROUP BY o.name
     ORDER BY count() DESC
     LIMIT 1000;
-    `;
+  `;
 
   const res = await queryClickhouse<{ name: string }>({
     query,
@@ -529,10 +820,11 @@ export const getObservationsGroupedByPromptName = async (
 
   const appliedObservationsFilter = observationsFilter.apply();
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-    SELECT
-      o.prompt_id as id
-    FROM observations o FINAL
+    SELECT o.prompt_id as id
+    FROM observations o
     WHERE ${appliedObservationsFilter.query}
     AND o.type = 'GENERATION'
     AND o.prompt_id IS NOT NULL
@@ -575,13 +867,20 @@ export const getCostForTraces = async (
   projectId: string,
   traceIds: string[],
 ) => {
+  // Wrapping the query in a CTE allows us to skip FINAL which allows Clickhouse to use skip indexes.
   const query = `
-    SELECT
-      sum(o.total_cost) as total_cost
-    FROM observations o FINAL
-    WHERE o.project_id = {projectId: String}
-    AND o.trace_id IN ({traceIds: Array(String)});
-    `;
+    WITH selected_observations AS (
+      SELECT o.total_cost as total_cost
+      FROM observations o
+      WHERE o.project_id = {projectId: String}
+      AND o.trace_id IN ({traceIds: Array(String)})
+      ORDER BY o.event_ts DESC
+      LIMIT 1 BY o.id, o.project_id
+    )
+
+    SELECT sum(total_cost) as total_cost
+    FROM selected_observations
+ `;
 
   const res = await queryClickhouse<{ total_cost: string }>({
     query,
@@ -610,3 +909,163 @@ export const deleteObservationsByTraceIds = async (
     },
   });
 };
+
+export const getObservationsWithPromptName = async (
+  projectId: string,
+  promptNames: string[],
+) => {
+  const query = `
+  SELECT count(*) as count, prompt_name
+  FROM observations FINAL
+  WHERE project_id = {projectId: String}
+  AND prompt_name IN ({promptNames: Array(String)})
+  AND prompt_name IS NOT NULL
+  GROUP BY prompt_name
+`;
+  const rows = await queryClickhouse<{ count: string; prompt_name: string }>({
+    query: query,
+    params: {
+      projectId,
+      promptNames,
+    },
+  });
+
+  return rows.map((r) => ({
+    count: Number(r.count),
+    promptName: r.prompt_name,
+  }));
+};
+
+export const getObservationMetricsForPrompts = async (
+  projectId: string,
+  promptIds: string[],
+) => {
+  const query = `
+      WITH latencies AS
+          (
+              SELECT
+                  prompt_id,
+                  prompt_version,
+                  start_time,
+                  end_time,
+                  usage_details,
+                  cost_details,
+                  dateDiff('milliseconds', start_time, end_time) AS latency_ms
+              FROM observations
+              FINAL
+              WHERE (type = 'GENERATION') 
+              AND (prompt_name IS NOT NULL) 
+              AND project_id={projectId: String} 
+              AND prompt_id IN ({promptIds: Array(String)})
+          )
+      SELECT
+          count(*) AS count,
+          prompt_id,
+          prompt_version,
+          min(start_time) AS first_observation,
+          max(start_time) AS last_observation,
+          medianExact(usage_details['input']) AS median_input_usage,
+          medianExact(usage_details['output']) AS median_output_usage,
+          medianExact(cost_details['total']) AS median_total_cost,
+          medianExact(latency_ms) AS median_latency_ms
+      FROM latencies
+      GROUP BY
+          prompt_id,
+          prompt_version
+      ORDER BY prompt_version DESC
+`;
+  const rows = await queryClickhouse<{
+    count: string;
+    prompt_id: string;
+    prompt_version: number;
+    first_observation: string;
+    last_observation: string;
+    median_input_usage: string;
+    median_output_usage: string;
+    median_total_cost: string;
+    median_latency_ms: string;
+  }>({
+    query: query,
+    params: {
+      projectId,
+      promptIds,
+    },
+  });
+
+  return rows.map((r) => ({
+    count: Number(r.count),
+    promptId: r.prompt_id,
+    promptVersion: r.prompt_version,
+    firstObservation: parseClickhouseUTCDateTimeFormat(r.first_observation),
+    lastObservation: parseClickhouseUTCDateTimeFormat(r.last_observation),
+    medianInputUsage: Number(r.median_input_usage),
+    medianOutputUsage: Number(r.median_output_usage),
+    medianTotalCost: Number(r.median_total_cost),
+    medianLatencyMs: Number(r.median_latency_ms),
+  }));
+};
+
+export const getLatencyAndTotalCostForObservations = async (
+  projectId: string,
+  observationIds: string[],
+) => {
+  const query = `
+    SELECT
+        id,
+        cost_details['total'] AS total_cost,
+        dateDiff('milliseconds', start_time, end_time) AS latency_ms
+    FROM observations FINAL 
+    WHERE project_id = {projectId: String} 
+    AND id IN ({observationIds: Array(String)})
+`;
+  const rows = await queryClickhouse<{
+    id: string;
+    total_cost: string;
+    latency_ms: string;
+  }>({
+    query: query,
+    params: {
+      projectId,
+      observationIds,
+    },
+  });
+
+  return rows.map((r) => ({
+    id: r.id,
+    totalCost: Number(r.total_cost),
+    latency: Number(r.latency_ms) / 1000,
+  }));
+};
+
+export const getLatencyAndTotalCostForObservationsByTraces = async (
+  projectId: string,
+  traceIds: string[],
+) => {
+  const query = `
+    SELECT
+        trace_id,
+        sumMap(cost_details)['total'] AS total_cost,
+        dateDiff('milliseconds', min(start_time), max(end_time)) AS latency_ms
+    FROM observations FINAL
+    WHERE project_id = {projectId: String} 
+    AND trace_id IN ({traceIds: Array(String)})
+    GROUP BY trace_id
+`;
+  const rows = await queryClickhouse<{
+    trace_id: string;
+    total_cost: string;
+    latency_ms: string;
+  }>({
+    query: query,
+    params: {
+      projectId,
+      traceIds,
+    },
+  });
+
+  return rows.map((r) => ({
+    traceId: r.trace_id,
+    totalCost: Number(r.total_cost),
+    latency: Number(r.latency_ms) / 1000,
+  }));
+};

packages/shared/src/server/repositories/observations_converters.ts

@@ -1,51 +1,26 @@
 import {
   Observation,
   ObservationView,
-  Model,
-  Price,
   ObservationType,
   ObservationLevel,
+  Prisma,
 } from "@prisma/client";
 import Decimal from "decimal.js";
-import { prisma } from "../../db";
-import { jsonSchema } from "../../utils/zod";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
+import { parseJsonPrioritised } from "../../utils/json";
+import { jsonSchema } from "../../utils/zod";
 
-export const convertObservation = async (
+export const convertObservationToView = (
   record: ObservationRecordReadType,
-): Promise<Observation> => {
-  const model = record.internal_model_id
-    ? await prisma.model.findFirst({
-        where: {
-          id: record.internal_model_id,
-        },
-        include: {
-          Price: true,
-        },
-      })
-    : undefined;
-  return convertObservationAndModel(record, model ?? undefined);
-};
-
-export const convertObservationToView = async (
-  record: ObservationRecordReadType,
-  providedModel?: Model & { Price: Price[] },
-): Promise<ObservationView> => {
-  const model =
-    providedModel ??
-    (record.internal_model_id
-      ? await prisma.model.findFirst({
-          where: {
-            id: record.internal_model_id,
-          },
-          include: {
-            Price: true,
-          },
-        })
-      : undefined);
+): Omit<ObservationView, "inputPrice" | "outputPrice" | "totalPrice"> => {
+  // these cost are not used from the view. They are in the select statement but not in the
+  // Prisma file. We will not clean this up but keep it as it is for now.
+  // eslint-disable-next-line no-unused-vars
+  const { inputCost, outputCost, totalCost, internalModelId, ...rest } =
+    convertObservation(record ?? undefined);
   return {
-    ...convertObservationAndModel(record, model ?? undefined),
+    ...rest,
     latency: record.end_time
       ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
@@ -54,22 +29,20 @@ export const convertObservationToView = async (
       ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
         parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
       : null,
-    inputPrice:
-      model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
-    outputPrice:
-      model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
-    totalPrice:
-      model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
     promptName: record.prompt_name ?? null,
     promptVersion: record.prompt_version ?? null,
     modelId: record.internal_model_id ?? null,
   };
 };
 
-export const convertObservationAndModel = (
+export const convertObservation = (
   record: ObservationRecordReadType,
-  model?: Model & { Price: Price[] },
-): Observation => {
+): Omit<Observation, "internalModel"> & {
+  promptName: string | null;
+  promptVersion: number | null;
+  latency: number | null;
+  timeToFirstToken: number | null;
+} => {
   return {
     id: record.id,
     traceId: record.trace_id ?? null,
@@ -85,8 +58,12 @@ export const convertObservationAndModel = (
     level: record.level as ObservationLevel,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: jsonSchema.nullish().parse(record.input) ?? null,
-    output: jsonSchema.nullish().parse(record.output) ?? null,
+    input: (record.input
+      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      : null) as Prisma.JsonValue | null,
     modelParameters: record.model_parameters
       ? JSON.parse(record.model_parameters)
       : null,
@@ -123,7 +100,16 @@ export const convertObservationAndModel = (
     totalCost: record.total_cost ? new Decimal(record.total_cost) : null,
     model: record.provided_model_name ?? null,
     internalModelId: record.internal_model_id ?? null,
-    internalModel: model?.modelName ?? null, // to be removed
     unit: "TOKENS", // to be removed.
+    promptName: record.prompt_name ?? null,
+    promptVersion: record.prompt_version ?? null,
+    latency: record.end_time
+      ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
+      : null,
+    timeToFirstToken: record.completion_start_time
+      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
+      : null,
   };
 };

packages/shared/src/server/repositories/scores.ts

@@ -12,31 +12,19 @@ import {
   getProjectIdDefaultFilter,
 } from "../queries/clickhouse-sql/factory";
 import { OrderByState } from "../../interfaces/orderBy";
-import { scoresTableUiColumnDefinitions } from "../../tableDefinitions";
+import {
+  dashboardColumnDefinitions,
+  scoresTableUiColumnDefinitions,
+} from "../../tableDefinitions";
 import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
-import { convertToScore } from "./scores_converters";
-
-export type FetchScoresReturnType = {
-  id: string;
-  timestamp: string;
-  project_id: string;
-  trace_id: string;
-  observation_id: string | null;
-  name: string;
-  value: number;
-  source: string;
-  comment: string | null;
-  author_user_id: string | null;
-  config_id: string | null;
-  data_type: string;
-  string_value: string | null;
-  queue_id: string | null;
-  created_at: string;
-  updated_at: string;
-  event_ts: string;
-  is_deleted: number;
-  projectId: string;
-};
+import {
+  convertScoreAggregation,
+  convertToScore,
+  ScoreAggregation,
+} from "./scores_converters";
+import { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./constants";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { ScoreRecordReadType } from "./definitions";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -50,7 +38,7 @@ export const searchExistingAnnotationScore = async (
   }
   const query = `
     SELECT *
-    FROM scores s FINAL
+    FROM scores s
     WHERE s.project_id = {projectId: String}
     AND s.source = 'ANNOTATION'
     AND s.trace_id = {traceId: String}
@@ -61,10 +49,11 @@ export const searchExistingAnnotationScore = async (
       ${configId ? `OR s.config_id = {configId: String}` : ""}
     )
     ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -80,24 +69,25 @@ export const searchExistingAnnotationScore = async (
 export const getScoreById = async (
   projectId: string,
   scoreId: string,
-  source: ScoreSource,
+  source?: ScoreSource,
 ) => {
   const query = `
     SELECT *
-    FROM scores s FINAL
+    FROM scores s
     WHERE s.project_id = {projectId: String}
     AND s.id = {scoreId: String}
-    AND s.source = {source: String}
+    ${source ? `AND s.source = {source: String}` : ""}
     ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
       scoreId,
-      source,
+      ...(source !== undefined ? { source } : {}),
     },
   });
   return rows.map(convertToScore).shift();
@@ -107,13 +97,13 @@ export const getScoreById = async (
  * Accepts a score in a Clickhouse-ready format.
  * id, project_id, name, and timestamp must always be provided.
  */
-export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
+export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   if (!["id", "project_id", "name", "timestamp"].every((key) => key in score)) {
     throw new Error("Identifier fields must be provided to upsert Score.");
   }
   await upsertClickhouse({
     table: "scores",
-    records: [score as FetchScoresReturnType],
+    records: [score as ScoreRecordReadType],
     eventBodyMapper: convertToScore,
   });
 };
@@ -121,25 +111,32 @@ export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
 export const getScoresForTraces = async (
   projectId: string,
   traceIds: string[],
+  timestamp?: Date,
   limit?: number,
   offset?: number,
 ) => {
   const query = `
       select 
         *
-      from scores s final
+      from scores s
       WHERE s.project_id = {projectId: String}
-      AND s.trace_id IN ({traceIds: Array(String)})
+      AND s.trace_id IN ({traceIds: Array(String)}) 
+      ${timestamp ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+      ORDER BY s.event_ts DESC
+      LIMIT 1 BY s.id, s.project_id
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
-      projectId: projectId,
-      traceIds: traceIds,
-      limit: limit,
-      offset: offset,
+      projectId,
+      traceIds,
+      limit,
+      offset,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
     },
   });
 
@@ -155,13 +152,15 @@ export const getScoresForObservations = async (
   const query = `
       select 
         *
-      from scores s final
+      from scores s
       WHERE s.project_id = {projectId: String}
       AND s.observation_id IN ({observationIds: Array(String)})
+      ORDER BY s.event_ts DESC
+      LIMIT 1 BY s.id, s.project_id
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId: projectId,
@@ -175,12 +174,14 @@ export const getScoresForObservations = async (
 };
 
 export const getScoresGroupedByNameSourceType = async (projectId: string) => {
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
       select 
         name,
         source,
         data_type
-      from scores s final
+      from scores s
       WHERE s.project_id = {projectId: String}
       GROUP BY name, source, data_type
       ORDER BY count() desc
@@ -224,10 +225,12 @@ export const getScoresGroupedByName = async (
     ? new FilterList(chFilter).apply()
     : undefined;
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
       select 
         name as name
-      from scores s final
+      from scores s
       WHERE s.project_id = {projectId: String}
       AND has(['NUMERIC', 'BOOLEAN'], s.data_type)
       ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
@@ -373,16 +376,18 @@ export const getScoresUiGeneric = async <T>(props: {
 
   const scoresFilterRes = scoresFilter.apply();
 
+  // TODO: Can we realistically apply a traces time filter here? Is an order by event_ts a risk?
   const query = `
       SELECT 
           ${select}
       FROM scores s final
-      LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id
+      LEFT JOIN traces t
+      ON s.trace_id = t.id 
+      AND t.project_id = s.project_id
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
-
     `;
 
   const rows = await queryClickhouse<T>({
@@ -410,11 +415,13 @@ export const getScoreNames = async (
   );
   const timestampFilterRes = chFilter.apply();
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
       select 
         name,
         count(*) as count
-      from scores s final
+      from scores s
       WHERE s.project_id = {projectId: String}
       ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
       GROUP BY name
@@ -471,3 +478,74 @@ export const deleteScoresByTraceIds = async (
     },
   });
 };
+
+export const getNumericScoreHistogram = async (
+  projectId: string,
+  filter: FilterState,
+  limit: number,
+) => {
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, dashboardColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const query = `
+    select s.value
+    from scores s
+    WHERE s.project_id = {projectId: String}
+    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
+    ORDER BY s.event_ts DESC
+    LIMIT 1 BY s.id, s.project_id
+    ${limit !== undefined ? `limit {limit: Int32}` : ""}
+  `;
+
+  return queryClickhouse<{ value: number }>({
+    query,
+    params: {
+      projectId,
+      limit,
+      ...(chFilterRes ? chFilterRes.params : {}),
+    },
+  });
+};
+
+export const getAggregatedScoresForPrompts = async (
+  projectId: string,
+  promptIds: string[],
+  fetchScoreRelation: "observation" | "trace",
+) => {
+  const query = `
+    SELECT 
+      prompt_id,
+      s.id,
+      s.name,
+      s.string_value,
+      s.value,
+      s.source,
+      s.data_type,
+      s.comment
+    FROM scores s FINAL LEFT JOIN observations o FINAL 
+      ON o.trace_id = s.trace_id 
+      AND o.project_id = s.project_id 
+      ${fetchScoreRelation === "observation" ? "AND o.id = s.observation_id" : ""}
+    WHERE o.project_id = {projectId: String}
+    AND s.project_id = {projectId: String}
+    AND o.prompt_id IN ({promptIds: Array(String)})
+    AND o.type = 'GENERATION'
+    AND s.name IS NOT NULL
+    ${fetchScoreRelation === "trace" ? "AND s.observation_id IS NULL" : ""}
+  `;
+
+  const rows = await queryClickhouse<ScoreAggregation & { prompt_id: string }>({
+    query,
+    params: {
+      projectId,
+      promptIds,
+    },
+  });
+
+  return rows.map((row) => ({
+    ...convertScoreAggregation(row),
+    promptId: row.prompt_id,
+  }));
+};

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,23 +1,45 @@
 import { ScoreSource, ScoreDataType } from "@prisma/client";
-import { FetchScoresReturnType } from "./scores";
+import { ScoreRecordReadType } from "./definitions";
 
-export const convertToScore = (row: FetchScoresReturnType) => {
+export type ScoreAggregation = {
+  id: string;
+  name: string;
+  string_value: string | null;
+  value: string;
+  source: string;
+  data_type: string;
+  comment: string | null;
+};
+
+export const convertToScore = (row: ScoreRecordReadType) => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),
     projectId: row.project_id,
     traceId: row.trace_id,
-    observationId: row.observation_id,
+    observationId: row.observation_id ?? null,
     name: row.name,
-    value: row.value,
+    value: row.value ?? null,
     source: row.source as ScoreSource,
-    comment: row.comment,
-    authorUserId: row.author_user_id,
-    configId: row.config_id,
+    comment: row.comment ?? null,
+    authorUserId: row.author_user_id ?? null,
+    configId: row.config_id ?? null,
     dataType: row.data_type as ScoreDataType,
-    stringValue: row.string_value,
-    queueId: row.queue_id,
+    stringValue: row.string_value ?? null,
+    queueId: row.queue_id ?? null,
     createdAt: new Date(row.created_at),
     updatedAt: new Date(row.updated_at),
   };
 };
+
+export const convertScoreAggregation = (row: ScoreAggregation) => {
+  return {
+    id: row.id,
+    name: row.name,
+    stringValue: row.string_value,
+    value: Number(row.value),
+    source: row.source as ScoreSource,
+    dataType: row.data_type as ScoreDataType,
+    comment: row.comment,
+  };
+};

packages/shared/src/server/repositories/traces.ts

@@ -8,14 +8,11 @@ import {
   createFilterFromFilterState,
   getProjectIdDefaultFilter,
 } from "../queries/clickhouse-sql/factory";
-import { ObservationLevel, Trace } from "@prisma/client";
 import { FilterState } from "../../types";
-import { logger } from "../logger";
 import {
   DateTimeFilter,
   FilterList,
   StringFilter,
-  StringOptionsFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
@@ -27,240 +24,48 @@ import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
+import { FetchTracesTableProps } from "../services/traces-ui-table-service";
 
-export type TracesTableReturnType = Pick<
-  TraceRecordReadType,
-  | "project_id"
-  | "id"
-  | "name"
-  | "timestamp"
-  | "bookmarked"
-  | "release"
-  | "version"
-  | "user_id"
-  | "session_id"
-  | "tags"
-  | "metadata"
-  | "public"
-> & {
-  level: ObservationLevel;
-  observation_count: number | null;
-  latency_milliseconds: string | null;
-  usage_details: Record<string, number>;
-  cost_details: Record<string, number>;
-  scores_avg: Array<{ name: string; avg_value: number }>;
-};
-
-export const getTracesTableCount = async (props: {
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  offset?: number;
-}) => {
-  const countRows = await getTracesTableGeneric<{ count: string }>({
-    select: "count(*) as count",
-    ...props,
-  });
-
-  const converted = countRows.map((row) => ({
-    count: Number(row.count),
-  }));
-
-  return converted.length > 0 ? converted[0].count : 0;
-};
-
-export const getTracesTable = async (
+export const checkTraceExists = async (
   projectId: string,
+  traceId: string,
+  timestamp: Date | undefined,
   filter: FilterState,
-  searchQuery?: string,
-  orderBy?: OrderByState,
-  limit?: number,
-  offset?: number,
-) => {
-  const rows = await getTracesTableGeneric<TracesTableReturnType>({
-    select: `
-    t.id, 
-    t.project_id, 
-    t.timestamp, 
-    t.tags, 
-    t.bookmarked, 
-    t.name, 
-    t.release, 
-    t.version, 
-    t.user_id, 
-    t.session_id,
-    os.latency_milliseconds,
-    os.cost_details as cost_details,
-    os.usage_details as usage_details,
-    os.level as level,
-    os.observation_count as observation_count,
-    s.scores_avg as scores_avg,
-    t.metadata,
-    t.public`,
-    projectId,
-    filter,
-    searchQuery,
-    orderBy,
-    limit,
-    offset,
+): Promise<boolean> => {
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
   });
 
-  return rows;
-};
-
-type FetchTracesTableProps = {
-  select: string;
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  offset?: number;
-};
-
-const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, offset, searchQuery } =
-    props;
-
-  const { tracesFilter, scoresFilter, observationsFilter } =
-    getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
-
   tracesFilter.push(
     ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+    new StringFilter({
+      clickhouseTable: "t",
+      field: "id",
+      operator: "=",
+      value: traceId,
+    }),
   );
 
-  const traceIdFilter = tracesFilter.find(
-    (f) => f.clickhouseTable === "traces" && f.field === "id",
-  ) as StringFilter | StringOptionsFilter | undefined;
-
-  traceIdFilter
-    ? scoresFilter.push(
-        new StringOptionsFilter({
-          clickhouseTable: "scores",
-          field: "trace_id",
-          operator: "any of",
-          values:
-            traceIdFilter instanceof StringFilter
-              ? [traceIdFilter.value]
-              : traceIdFilter.values,
-        }),
-      )
-    : null;
-  traceIdFilter
-    ? observationsFilter.push(
-        new StringOptionsFilter({
-          clickhouseTable: "observations",
-          field: "trace_id",
-          operator: "any of",
-          values:
-            traceIdFilter instanceof StringFilter
-              ? [traceIdFilter.value]
-              : traceIdFilter.values,
-        }),
-      )
-    : null;
-
-  // for query optimisation, we have to add the timeseries filter to observations + scores as well
-  // stats show, that 98% of all observations have their start_time larger than trace.timestamp - 5 min
-  const timeStampFilter = tracesFilter.find(
-    (f) =>
-      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  timeStampFilter
-    ? scoresFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "scores",
-          field: "timestamp",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
-
-  timeStampFilter
-    ? observationsFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "observations",
-          field: "start_time",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
-
   const tracesFilterRes = tracesFilter.apply();
-  const scoresFilterRes = scoresFilter.apply();
-  const observationFilterRes = observationsFilter.apply();
-
-  const search = clickhouseSearchCondition(searchQuery);
 
   const query = `
-  WITH observations_stats AS (
-  SELECT
-    COUNT(*) AS observation_count,
-      sumMap(usage_details) as usage_details,
-      SUM(total_cost) AS total_cost,
-      date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
-      multiIf(
-        arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
-        arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
-        arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
-        'DEBUG'
-      ) AS level,
-      sumMap(cost_details) as cost_details,
-      trace_id,
-      project_id
-    FROM
-        observations final
-    WHERE ${observationFilterRes.query}
-    
-    group by trace_id, project_id
-),
+    SELECT id, project_id
+    FROM traces t FINAL
+    WHERE ${tracesFilterRes.query}
+    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  `;
 
-         scores_avg AS (SELECT project_id,
-                                trace_id,
-                                groupArray(tuple(name, avg_value)) AS "scores_avg"
-                          FROM (
-                                  SELECT project_id,
-                                          trace_id,
-                                          name,
-                                          avg(value) avg_value
-                                  FROM scores final
-                                  WHERE ${scoresFilterRes.query}
-                                  GROUP BY project_id,
-                                            trace_id,
-                                            name
-                                  ) tmp
-                          GROUP BY project_id,
-                                  trace_id)
-      select 
-       ${select}
-      from traces t final
-              left join observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
-              left join scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
-
-      WHERE ${tracesFilterRes.query}
-      ${search.query}
-      ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
-      ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
-
-  const res = await queryClickhouse<T>({
-    query: query,
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
     params: {
-      limit: limit,
-      offset: offset,
       ...tracesFilterRes.params,
-      ...observationFilterRes.params,
-      ...scoresFilterRes.params,
-      ...search.params,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
     },
   });
 
-  return res;
+  return rows.length > 0;
 };
 
 /**
@@ -278,18 +83,6 @@ export const upsertTrace = async (trace: Partial<TraceRecordReadType>) => {
   });
 };
 
-export const getTraceById = async (
-  traceId: string,
-  projectId: string,
-  timestamp?: Date,
-): Promise<Trace | undefined> => {
-  try {
-    return getTraceByIdOrThrow(traceId, projectId, timestamp);
-  } catch (e) {
-    return undefined;
-  }
-};
-
 export const getTracesByIds = async (
   traceIds: string[],
   projectId: string,
@@ -299,9 +92,10 @@ export const getTracesByIds = async (
       SELECT * 
       FROM traces
       WHERE id IN ({traceIds: Array(String)})
-        AND project_id = {projectId: String}
-        ${timestamp ? `AND timestamp >= {timestamp: DateTime}` : ""} 
-      ORDER BY event_ts DESC LIMIT 1 by id, project_id;`;
+      AND project_id = {projectId: String}
+      ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)}` : ""} 
+      ORDER BY event_ts DESC
+      LIMIT 1 by id, project_id;`;
   const records = await queryClickhouse<TraceRecordReadType>({
     query,
     params: {
@@ -314,35 +108,78 @@ export const getTracesByIds = async (
   return records.map(convertClickhouseToDomain);
 };
 
-export const getTraceByIdOrThrow = async (
+export const getTracesBySessionId = async (
+  projectId: string,
+  sessionIds: string[],
+  timestamp?: Date,
+) => {
+  const query = `
+      SELECT * 
+      FROM traces
+      WHERE session_id IN ({sessionIds: Array(String)})
+      AND project_id = {projectId: String}
+      ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)}` : ""} 
+      ORDER BY event_ts DESC
+      LIMIT 1 by id, project_id;`;
+  const records = await queryClickhouse<TraceRecordReadType>({
+    query,
+    params: {
+      sessionIds,
+      projectId,
+      timestamp: timestamp ? convertDateToClickhouseDateTime(timestamp) : null,
+    },
+  });
+
+  return records.map(convertClickhouseToDomain);
+};
+
+export const hasAnyTrace = async (projectId: string) => {
+  const query = `
+    SELECT count(*) as count
+    FROM traces
+    WHERE project_id = {projectId: String}
+    LIMIT 1
+  `;
+
+  const rows = await queryClickhouse<{ count: string }>({
+    query,
+    params: {
+      projectId,
+    },
+  });
+
+  return rows.length > 0 && Number(rows[0].count) > 0;
+};
+
+export const getTraceById = async (
   traceId: string,
   projectId: string,
   timestamp?: Date,
 ) => {
-  const query = `SELECT * 
-      FROM traces
-      WHERE id = {traceId: String} 
-        AND project_id = {projectId: String}
-        ${timestamp ? `AND timestamp = {timestamp: DateTime64(3)}` : ""} 
-      ORDER BY event_ts DESC LIMIT 1 by id, project_id`;
+  const query = `
+    SELECT * 
+    FROM traces
+    WHERE id = {traceId: String} 
+    AND project_id = {projectId: String}
+    ${timestamp ? `AND toDate(timestamp) = toDate({timestamp: DateTime64(3)})` : ""} 
+    ORDER BY event_ts DESC 
+    LIMIT 1
+  `;
 
   const records = await queryClickhouse<TraceRecordReadType>({
     query,
     params: {
       traceId,
       projectId,
-      timestamp: timestamp ? timestamp.getTime() : null,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
     },
   });
 
   const res = records.map(convertClickhouseToDomain);
 
-  if (res.length === 0) {
-    const errorMessage = `Trace not found for traceId: ${traceId}, projectId: ${projectId}`;
-    logger.error(errorMessage);
-    throw new Error(errorMessage);
-  }
-  return res[0] as Trace;
+  return res.shift();
 };
 
 export const getTracesGroupedByName = async (
@@ -358,11 +195,13 @@ export const getTracesGroupedByName = async (
     ? new FilterList(chFilter).apply()
     : undefined;
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
       select 
         name as name,
         count(*) as count
-      from traces t final
+      from traces t
       WHERE t.project_id = {projectId: String}
       AND t.name IS NOT NULL
       ${timestampFilterRes?.query ? `AND ${timestampFilterRes.query}` : ""}
@@ -388,27 +227,41 @@ export const getTracesGroupedByName = async (
 export const getTracesGroupedByUsers = async (
   projectId: string,
   filter: FilterState,
+  searchQuery?: string,
+  limit?: number,
+  offset?: number,
   columns?: UiColumnMapping[],
 ) => {
-  const chFilter = createFilterFromFilterState(
-    filter,
-    columns ?? tracesTableUiColumnDefinitions,
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
+  });
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      columns ?? tracesTableUiColumnDefinitions,
+    ),
   );
 
-  const filterRes = new FilterList(chFilter).apply();
+  const tracesFilterRes = tracesFilter.apply();
+  const search = clickhouseSearchCondition(searchQuery);
 
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
       select 
         user_id as user,
         count(*) as count
-      from traces t final
+      from traces t
       WHERE t.project_id = {projectId: String}
       AND t.user_id IS NOT NULL
-      ${filterRes?.query ? `AND ${filterRes.query}` : ""}
+      AND t.user_id != ''
+      ${tracesFilterRes?.query ? `AND ${tracesFilterRes.query}` : ""}
+      ${search.query}
       GROUP BY user
       ORDER BY count desc
-      LIMIT 1000;
-    `;
+      ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+  `;
 
   const rows = await queryClickhouse<{
     user: string;
@@ -416,8 +269,11 @@ export const getTracesGroupedByUsers = async (
   }>({
     query: query,
     params: {
-      projectId: projectId,
-      ...(filterRes ? filterRes.params : {}),
+      limit,
+      offset,
+      projectId,
+      ...(tracesFilterRes ? tracesFilterRes.params : {}),
+      ...(searchQuery ? search.params : {}),
     },
   });
 
@@ -427,12 +283,11 @@ export const getTracesGroupedByUsers = async (
 export type GroupedTracesQueryProp = {
   projectId: string;
   filter: FilterState;
-  sessionIdNullFilter?: boolean;
   columns?: UiColumnMapping[];
 };
 
 export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
-  const { projectId, filter, sessionIdNullFilter, columns } = props;
+  const { projectId, filter, columns } = props;
 
   const chFilter = createFilterFromFilterState(
     filter,
@@ -442,14 +297,12 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
   const filterRes = new FilterList(chFilter).apply();
 
   const query = `
-      select 
-        distinct(arrayJoin(tags)) as value
-      from traces t final
-      WHERE t.project_id = {projectId: String}
-      ${sessionIdNullFilter ? "AND t.session_id IS NOT NULL" : ""}
-      ${filterRes?.query ? `AND ${filterRes.query}` : ""}
-      LIMIT 1000;
-    `;
+    select distinct(arrayJoin(tags)) as value
+    from traces t
+    WHERE t.project_id = {projectId: String}
+    ${filterRes?.query ? `AND ${filterRes.query}` : ""}
+    LIMIT 1000;
+  `;
 
   const rows = await queryClickhouse<{
     value: string;
@@ -464,45 +317,6 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
   return rows;
 };
 
-export const getTracesGroupedByUserIds = async (
-  props: GroupedTracesQueryProp,
-) => {
-  const {
-    projectId,
-    filter,
-    sessionIdNullFilter: sessionIdNotNullFilter,
-    columns,
-  } = props;
-
-  const chFilter = createFilterFromFilterState(
-    filter,
-    columns ?? tracesTableUiColumnDefinitions,
-  );
-
-  const appliedFilter = new FilterList(chFilter).apply();
-
-  const query = `
-      select distinct user_id as user_id
-      from traces t final
-      WHERE t.project_id = {projectId: String}
-      ${sessionIdNotNullFilter ? "AND t.session_id IS NOT NULL" : ""}
-      ${appliedFilter?.query ? `AND ${appliedFilter.query}` : ""}
-      LIMIT 1000;
-    `;
-
-  const rows = await queryClickhouse<{
-    user_id: string;
-  }>({
-    query: query,
-    params: {
-      projectId: projectId,
-      ...(appliedFilter ? appliedFilter.params : {}),
-    },
-  });
-
-  return rows;
-};
-
 export type SessionDataReturnType = {
   session_id: string;
   max_timestamp: string;
@@ -528,7 +342,7 @@ export const getSessionsTableCount = async (props: {
   filter: FilterState;
   orderBy?: OrderByState;
   limit?: number;
-  offset?: number;
+  page?: number;
 }) => {
   const rows = await getSessionsTableGeneric<{ count: string }>({
     select: `
@@ -538,7 +352,7 @@ export const getSessionsTableCount = async (props: {
     filter: props.filter,
     orderBy: props.orderBy,
     limit: props.limit,
-    offset: props.offset,
+    page: props.page,
   });
 
   return rows.length > 0 ? Number(rows[0].count) : 0;
@@ -549,7 +363,7 @@ export const getSessionsTable = async (props: {
   filter: FilterState;
   orderBy?: OrderByState;
   limit?: number;
-  offset?: number;
+  page?: number;
 }) => {
   const rows = await getSessionsTableGeneric<SessionDataReturnType>({
     select: `
@@ -575,14 +389,14 @@ export const getSessionsTable = async (props: {
     filter: props.filter,
     orderBy: props.orderBy,
     limit: props.limit,
-    offset: props.offset,
+    page: props.page,
   });
 
   return rows;
 };
 
 const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, offset } = props;
+  const { select, projectId, filter, orderBy, limit, page } = props;
 
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
@@ -621,7 +435,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
               anyLast(project_id) as project_id
         FROM observations o FINAL
         WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
         GROUP BY o.trace_id
     ),
     session_data AS (
@@ -646,7 +460,8 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
             sumMap(o.sum_usage_details)['output'] as session_output_usage,
             sumMap(o.sum_usage_details)['total'] as session_total_usage
         FROM traces t FINAL
-        LEFT JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id
+        LEFT JOIN observations_agg o
+        ON t.id = o.trace_id AND t.project_id = o.project_id
         WHERE t.session_id IS NOT NULL
             AND t.project_id = {projectId: String}
             ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
@@ -656,7 +471,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
     FROM session_data s
     WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
     ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
     `;
 
   const obsStartTimeValue = traceTimestampFilter
@@ -668,7 +483,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
     params: {
       projectId,
       limit: limit,
-      offset: offset,
+      offset: limit && page ? limit * page : 0,
       ...tracesFilterRes.params,
       ...observationsStatsRes.params,
       ...scoresAvgFilterRes.params,
@@ -682,7 +497,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
   return res;
 };
 
-export const getTracesForSession = async (
+export const getTracesIdentifierForSession = async (
   projectId: string,
   sessionId: string,
 ) => {
@@ -693,12 +508,11 @@ export const getTracesForSession = async (
       name,
       timestamp,
       project_id
-      FROM traces
-      WHERE (project_id = {projectId: String}) AND (session_id = {sessionId: String})
-      ORDER BY timestamp ASC
-      LIMIT 1 BY
-          id,
-          project_id;
+    FROM traces
+    WHERE (project_id = {projectId: String})
+    AND (session_id = {sessionId: String})
+    ORDER BY timestamp ASC
+    LIMIT 1 BY id, project_id;
   `;
 
   const rows = await queryClickhouse<{
@@ -736,3 +550,103 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
     },
   });
 };
+
+export const getTotalUserCount = async (
+  projectId: string,
+  filter: FilterState,
+  searchQuery?: string,
+): Promise<{ totalCount: bigint }[]> => {
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
+  });
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+
+  const tracesFilterRes = tracesFilter.apply();
+  const search = clickhouseSearchCondition(searchQuery);
+
+  const query = `
+    SELECT COUNT(DISTINCT t.user_id) AS totalCount
+    FROM traces t
+    WHERE ${tracesFilterRes.query}
+    ${search.query}
+    AND t.user_id IS NOT NULL
+    AND t.user_id != ''
+  `;
+
+  return queryClickhouse({
+    query,
+    params: {
+      ...tracesFilterRes.params,
+      ...search.params,
+    },
+  });
+};
+
+export const getUserMetrics = async (projectId: string, userIds: string[]) => {
+  if (userIds.length === 0) {
+    return [];
+  }
+  const query = `
+    WITH observations_agg AS (
+      SELECT o.trace_id,
+             count(*) as obs_count,
+             sumMap(usage_details) as sum_usage_details,
+             sum(total_cost) as sum_total_cost,
+             anyLast(project_id) as project_id
+      FROM observations o FINAL
+      WHERE o.project_id = {projectId: String}
+      GROUP BY o.trace_id
+    ),
+    user_metric_data AS (
+      SELECT t.user_id,
+             max(t.timestamp) as max_timestamp,
+             min(t.timestamp) as min_timestamp,
+             count(*) as trace_count,
+             sum(o.obs_count) as total_observations,
+             sum(o.sum_total_cost) as session_total_cost,
+             sumMap(o.sum_usage_details)['input'] as session_input_usage,
+             sumMap(o.sum_usage_details)['output'] as session_output_usage,
+             sumMap(o.sum_usage_details)['total'] as session_total_usage
+      FROM traces t FINAL
+      LEFT JOIN observations_agg o
+      ON t.id = o.trace_id 
+      AND t.project_id = o.project_id
+      WHERE t.user_id IS NOT NULL
+      AND t.user_id != ''
+      AND t.user_id IN ({userIds: Array(String)})
+      AND t.project_id = {projectId: String}
+      GROUP BY t.user_id
+    )
+    SELECT user_id AS userId,
+           min_timestamp as firstTrace,
+           max_timestamp as lastTrace,
+           trace_count as totalTraces,
+           total_observations as totalObservations,
+           session_input_usage as totalPromptTokens,
+           session_output_usage as totalCompletionTokens,
+           session_total_usage as totalTokens,
+           session_total_cost as sumCalculatedTotalCost
+    FROM user_metric_data umd
+  `;
+
+  return queryClickhouse<{
+    userId: string;
+    firstTrace: Date | null;
+    lastTrace: Date | null;
+    totalPromptTokens: bigint;
+    totalCompletionTokens: bigint;
+    totalTokens: bigint;
+    totalObservations: bigint;
+    totalTraces: bigint;
+    sumCalculatedTotalCost: number;
+  }>({
+    query,
+    params: {
+      projectId,
+      userIds,
+    },
+  });
+};

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,10 +1,10 @@
 import { ObservationLevel, Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
-import { TracesTableReturnType } from "./traces";
 import Decimal from "decimal.js";
 import { ScoreAggregate } from "../../features/scores";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { TracesTableReturnType } from "../services/traces-ui-table-service";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -69,20 +69,24 @@ export type TracesAllReturnType = {
   tags: string[];
 };
 
-export const convertToReturnType = (
-  row: TracesTableReturnType,
-): TracesAllReturnType => {
+export const convertToDomain = (row: TracesTableReturnType) => {
   return {
     id: row.id,
-    name: row.name ?? null,
+    projectId: row.project_id,
     timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
     tags: row.tags,
     bookmarked: row.bookmarked,
+    name: row.name ?? null,
     release: row.release ?? null,
     version: row.version ?? null,
-    projectId: row.project_id,
     userId: row.user_id ?? null,
     sessionId: row.session_id ?? null,
+    latency: Number(row.latency),
+    usageDetails: row.usage_details,
+    costDetails: row.cost_details,
+    level: row.level,
+    observationCount: Number(row.observation_count),
+    scoresAvg: row.scores_avg,
     public: row.public,
   };
 };
@@ -100,29 +104,3 @@ export type TracesMetricsReturnType = {
   calculatedOutputCost: Decimal | null;
   scores: ScoreAggregate;
 };
-
-export const convertMetricsReturnType = (
-  row: TracesTableReturnType & { scores: ScoreAggregate },
-): TracesMetricsReturnType => {
-  return {
-    id: row.id,
-    promptTokens: BigInt(row.usage_details?.input ?? 0),
-    completionTokens: BigInt(row.usage_details?.output ?? 0),
-    totalTokens: BigInt(row.usage_details?.total ?? 0),
-    latency: row.latency_milliseconds
-      ? Number(row.latency_milliseconds) / 1000
-      : null,
-    level: row.level,
-    observationCount: BigInt(row.observation_count ?? 0),
-    calculatedTotalCost: row.cost_details?.total
-      ? new Decimal(row.cost_details.total)
-      : null,
-    calculatedInputCost: row.cost_details?.input
-      ? new Decimal(row.cost_details.input)
-      : null,
-    calculatedOutputCost: row.cost_details?.output
-      ? new Decimal(row.cost_details.output)
-      : null,
-    scores: row.scores,
-  };
-};

packages/shared/src/server/services/S3StorageService.ts

@@ -1,143 +0,0 @@
-import type { Readable } from "stream";
-import {
-  GetObjectCommand,
-  ListObjectsV2Command,
-  PutObjectCommand,
-  S3Client,
-} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { logger } from "../logger";
-
-type UploadFile = {
-  fileName: string;
-  fileType: string;
-  data: Readable | string;
-  expiresInSeconds: number;
-};
-
-export class S3StorageService {
-  private client: S3Client;
-  private bucketName: string;
-
-  constructor(params: {
-    accessKeyId: string | undefined;
-    secretAccessKey: string | undefined;
-    bucketName: string;
-    endpoint: string | undefined;
-    region: string | undefined;
-    forcePathStyle: boolean;
-  }) {
-    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
-    const { accessKeyId, secretAccessKey } = params;
-    const credentials =
-      accessKeyId !== undefined && secretAccessKey !== undefined
-        ? {
-            accessKeyId,
-            secretAccessKey,
-          }
-        : undefined;
-
-    this.client = new S3Client({
-      credentials,
-      endpoint: params.endpoint,
-      region: params.region,
-      forcePathStyle: params.forcePathStyle,
-    });
-    this.bucketName = params.bucketName;
-  }
-
-  public async uploadFile({
-    fileName,
-    fileType,
-    data,
-    expiresInSeconds,
-  }: UploadFile): Promise<{ signedUrl: string }> {
-    try {
-      await new Upload({
-        client: this.client,
-        params: {
-          Bucket: this.bucketName,
-          Key: fileName,
-          Body: data,
-          ContentType: fileType,
-        },
-      }).done();
-
-      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
-
-      return { signedUrl };
-    } catch (err) {
-      logger.error(`Failed to upload file to ${fileName}`, err);
-      throw new Error("Failed to upload to S3 or generate signed URL");
-    }
-  }
-
-  public async uploadJson(path: string, body: Record<string, unknown>[]) {
-    const putCommand = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-      Body: JSON.stringify(body),
-      ContentType: "application/json",
-    });
-
-    try {
-      await this.client.send(putCommand);
-    } catch (err) {
-      logger.error(`Failed to upload JSON to S3 ${path}`, err);
-      throw Error("Failed to upload JSON to S3");
-    }
-  }
-
-  public async download(path: string): Promise<string> {
-    const getCommand = new GetObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-    });
-
-    try {
-      const response = await this.client.send(getCommand);
-      return (await response.Body?.transformToString()) ?? "";
-    } catch (err) {
-      logger.error(`Failed to download file from S3 ${path}`, err);
-      throw Error("Failed to download file from S3");
-    }
-  }
-
-  public async listFiles(prefix: string): Promise<string[]> {
-    const listCommand = new ListObjectsV2Command({
-      Bucket: this.bucketName,
-      Prefix: prefix,
-    });
-
-    try {
-      const response = await this.client.send(listCommand);
-      return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
-      );
-    } catch (err) {
-      logger.error(`Failed to list files from S3 ${prefix}`, err);
-      throw Error("Failed to list files from S3");
-    }
-  }
-
-  private async getSignedUrl(
-    fileName: string,
-    ttlSeconds: number,
-  ): Promise<string> {
-    try {
-      return await getSignedUrl(
-        this.client,
-        new GetObjectCommand({
-          Bucket: this.bucketName,
-          Key: fileName,
-          ResponseContentDisposition: `attachment; filename="${fileName}"`,
-        }),
-        { expiresIn: ttlSeconds },
-      );
-    } catch (err) {
-      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
-      throw Error("Failed to generate signed URL");
-    }
-  }
-}

packages/shared/src/server/services/StorageService.ts

@@ -0,0 +1,432 @@
+import { Readable } from "stream";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from "@azure/storage-blob";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+type UploadFile = {
+  fileName: string;
+  fileType: string;
+  data: Readable | string;
+  expiresInSeconds: number;
+};
+
+export interface StorageService {
+  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>;
+
+  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>;
+
+  download(path: string): Promise<string>;
+
+  listFiles(prefix: string): Promise<string[]>;
+
+  getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string>;
+
+  getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string>;
+}
+
+export class StorageServiceFactory {
+  public static getInstance(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }): StorageService {
+    if (env.LANGFUSE_USE_AZURE_BLOB === "true") {
+      return new AzureBlobStorageService(params);
+    }
+    return new S3StorageService(params);
+  }
+}
+
+class AzureBlobStorageService implements StorageService {
+  private client: ContainerClient;
+  private container: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    const { accessKeyId, secretAccessKey, endpoint } = params;
+    if (!accessKeyId || !secretAccessKey || !endpoint) {
+      throw new Error(
+        `Endpoint, account and account key must be configured to use Azure Blob Storage`,
+      );
+    }
+
+    const sharedKeyCredential = new StorageSharedKeyCredential(
+      accessKeyId,
+      secretAccessKey,
+    );
+    const blobServiceClient = new BlobServiceClient(
+      endpoint,
+      sharedKeyCredential,
+    );
+    this.container = params.bucketName;
+    this.client = blobServiceClient.getContainerClient(this.container);
+  }
+
+  private async createContainerIfNotExists(): Promise<void> {
+    try {
+      await this.client.createIfNotExists();
+    } catch (err) {
+      logger.error(
+        `Failed to create Azure Blob Storage container ${this.container}`,
+        err,
+      );
+      throw Error("Failed to create Azure Blob Storage container ");
+    }
+  }
+
+  public async uploadFile(params: UploadFile): Promise<{ signedUrl: string }> {
+    const { fileName, data, expiresInSeconds } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+
+      if (typeof data === "string") {
+        await blockBlobClient.upload(data, data.length);
+      } else if (data instanceof Readable) {
+        let offset = 0;
+        const blockIds = [];
+        for await (const chunk of data) {
+          const blockId = Buffer.from(`block-${offset}`).toString("base64");
+          const bufferChunk = Buffer.isBuffer(chunk)
+            ? chunk
+            : Buffer.from(chunk);
+
+          await blockBlobClient.stageBlock(
+            blockId,
+            bufferChunk,
+            bufferChunk.length,
+          );
+          blockIds.push(blockId);
+
+          offset += bufferChunk.length;
+        }
+
+        await blockBlobClient.commitBlockList(blockIds);
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+
+      return {
+        signedUrl: await this.getSignedUrl(fileName, expiresInSeconds, false),
+      };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to upload file to Azure Blob Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    await this.createContainerIfNotExists();
+
+    const blockBlobClient = this.client.getBlockBlobClient(path);
+    const content = JSON.stringify(body);
+    try {
+      await blockBlobClient.upload(content, content.length);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to Azure Blob Storage ${path}`, err);
+      throw Error("Failed to upload JSON to Azure Blob Storage");
+    }
+  }
+
+  private async streamToString(
+    readableStream: NodeJS.ReadableStream,
+  ): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const chunks: string[] = [];
+      readableStream.on("data", (data) => {
+        chunks.push(data.toString());
+      });
+      readableStream.on("end", () => {
+        resolve(chunks.join(""));
+      });
+      readableStream.on("error", reject);
+    });
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blobClient = this.client.getBlobClient(path);
+      const downloadResponse = await blobClient.download();
+      if (!downloadResponse.readableStreamBody) {
+        throw Error("No stream body available");
+      }
+      return this.streamToString(downloadResponse.readableStreamBody);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const result = await this.client.listBlobsFlat({ prefix });
+      const files = [];
+      for await (const blob of result) {
+        if (blob.name.startsWith(prefix)) {
+          files.push(blob.name);
+        }
+      }
+      return files;
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Azure Blob Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("r"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentDisposition: asAttachment
+          ? `attachment; filename="${fileName}"`
+          : undefined,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL for Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate presigned URL for Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(path);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("w"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentType: contentType,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL for Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate presigned upload URL for Azure Blob Storage",
+      );
+    }
+  }
+}
+
+class S3StorageService implements StorageService {
+  private client: S3Client;
+  private bucketName: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
+    const { accessKeyId, secretAccessKey } = params;
+    const credentials =
+      accessKeyId !== undefined && secretAccessKey !== undefined
+        ? {
+            accessKeyId,
+            secretAccessKey,
+          }
+        : undefined;
+
+    this.client = new S3Client({
+      credentials,
+      endpoint: params.endpoint,
+      region: params.region,
+      forcePathStyle: params.forcePathStyle,
+    });
+    this.bucketName = params.bucketName;
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      await new Upload({
+        client: this.client,
+        params: {
+          Bucket: this.bucketName,
+          Key: fileName,
+          Body: data,
+          ContentType: fileType,
+        },
+      }).done();
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(`Failed to upload file to ${fileName}`, err);
+      throw new Error("Failed to upload to S3 or generate signed URL");
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    const putCommand = new PutObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+      Body: JSON.stringify(body),
+      ContentType: "application/json",
+    });
+
+    try {
+      await this.client.send(putCommand);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to S3 ${path}`, err);
+      throw Error("Failed to upload JSON to S3");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+
+    try {
+      const response = await this.client.send(getCommand);
+      return (await response.Body?.transformToString()) ?? "";
+    } catch (err) {
+      logger.error(`Failed to download file from S3 ${path}`, err);
+      throw Error("Failed to download file from S3");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: prefix,
+    });
+
+    try {
+      const response = await this.client.send(listCommand);
+      return (
+        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
+      );
+    } catch (err) {
+      logger.error(`Failed to list files from S3 ${prefix}`, err);
+      throw Error("Failed to list files from S3");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      return await getSignedUrl(
+        this.client,
+        new GetObjectCommand({
+          Bucket: this.bucketName,
+          Key: fileName,
+          ResponseContentDisposition: asAttachment
+            ? `attachment; filename="${fileName}"`
+            : undefined,
+        }),
+        { expiresIn: ttlSeconds },
+      );
+    } catch (err) {
+      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
+      throw Error("Failed to generate signed URL");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
+
+    return await getSignedUrl(
+      this.client,
+      new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: path,
+        ContentType: contentType,
+        ChecksumSHA256: sha256Hash,
+        ContentLength: contentLength,
+      }),
+      {
+        expiresIn: ttlSeconds,
+        signableHeaders: new Set(["content-type", "content-length"]),
+        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
+      },
+    );
+  }
+}

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -0,0 +1,248 @@
+import { ObservationLevel } from "@prisma/client";
+import { OrderByState } from "../../interfaces/orderBy";
+import { tracesTableUiColumnDefinitions } from "../../tableDefinitions";
+import { FilterState } from "../../types";
+import {
+  StringFilter,
+  StringOptionsFilter,
+  DateTimeFilter,
+} from "../queries/clickhouse-sql/clickhouse-filter";
+import {
+  getProjectIdDefaultFilter,
+  createFilterFromFilterState,
+} from "../queries/clickhouse-sql/factory";
+import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
+import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
+import { convertToDomain } from "../repositories";
+import { queryClickhouse } from "../repositories/clickhouse";
+import { TraceRecordReadType } from "../repositories/definitions";
+
+export type TracesTableReturnType = Pick<
+  TraceRecordReadType,
+  | "project_id"
+  | "id"
+  | "name"
+  | "timestamp"
+  | "bookmarked"
+  | "release"
+  | "version"
+  | "user_id"
+  | "session_id"
+  | "tags"
+  | "public"
+> & {
+  level: ObservationLevel;
+  observation_count: number | null;
+  latency: string | null;
+  usage_details: Record<string, number>;
+  cost_details: Record<string, number>;
+  scores_avg: Array<{ name: string; avg_value: number }>;
+};
+
+export type FetchTracesTableProps = {
+  select: string;
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+export const getTracesTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const countRows = await getTracesTableGeneric<{ count: string }>({
+    select: "count(*) as count",
+    ...props,
+  });
+
+  const converted = countRows.map((row) => ({
+    count: Number(row.count),
+  }));
+
+  return converted.length > 0 ? converted[0].count : 0;
+};
+
+export const getTracesTable = async (
+  projectId: string,
+  filter: FilterState,
+  searchQuery?: string,
+  orderBy?: OrderByState,
+  limit?: number,
+  page?: number,
+) => {
+  const rows = await getTracesTableGeneric<TracesTableReturnType>({
+    select: `
+    t.id, 
+    t.project_id as project_id, 
+    t.timestamp, 
+    t.tags, 
+    t.bookmarked, 
+    t.name, 
+    t.release, 
+    t.version, 
+    t.user_id, 
+    t.session_id,
+    os.latency_milliseconds / 1000 as latency,
+    os.cost_details as cost_details,
+    os.usage_details as usage_details,
+    os.level as level,
+    os.observation_count as observation_count,
+    s.scores_avg as scores_avg,
+    t.public`,
+    projectId,
+    filter,
+    searchQuery,
+    orderBy,
+    limit,
+    page,
+  });
+
+  return rows.map(convertToDomain);
+};
+
+const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
+  const { select, projectId, filter, orderBy, limit, page, searchQuery } =
+    props;
+
+  const { tracesFilter, scoresFilter, observationsFilter } =
+    getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+
+  const traceIdFilter = tracesFilter.find(
+    (f) => f.clickhouseTable === "traces" && f.field === "id",
+  ) as StringFilter | StringOptionsFilter | undefined;
+
+  traceIdFilter
+    ? scoresFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "scores",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+  traceIdFilter
+    ? observationsFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "observations",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+
+  // for query optimisation, we have to add the timeseries filter to observations + scores as well
+  // stats show, that 98% of all observations have their start_time larger than trace.timestamp - 5 min
+  const timeStampFilter = tracesFilter.find(
+    (f) =>
+      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  timeStampFilter
+    ? scoresFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "scores",
+          field: "timestamp",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
+  timeStampFilter
+    ? observationsFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "observations",
+          field: "start_time",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
+  const tracesFilterRes = tracesFilter.apply();
+  const scoresFilterRes = scoresFilter.apply();
+  const observationFilterRes = observationsFilter.apply();
+
+  const search = clickhouseSearchCondition(searchQuery);
+
+  const query = `
+    WITH observations_stats AS (
+      SELECT
+        COUNT(*) AS observation_count,
+          sumMap(usage_details) as usage_details,
+          SUM(total_cost) AS total_cost,
+          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          multiIf(
+            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
+            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
+            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
+            'DEBUG'
+          ) AS level,
+          sumMap(cost_details) as cost_details,
+          trace_id,
+          project_id
+      FROM observations FINAL
+      WHERE ${observationFilterRes.query}
+      GROUP BY trace_id, project_id
+    ),
+    scores_avg AS (
+      SELECT
+        project_id,
+        trace_id,
+        groupArray(tuple(name, avg_value)) AS "scores_avg"
+      FROM (
+        SELECT project_id,
+                trace_id,
+                name,
+                avg(value) avg_value
+        FROM scores final
+        WHERE ${scoresFilterRes.query}
+        GROUP BY project_id,
+                  trace_id,
+                  name
+      ) tmp
+      GROUP BY project_id, trace_id
+    )
+    SELECT ${select}
+    FROM traces t final
+    LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
+    LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
+    WHERE ${tracesFilterRes.query}
+    ${search.query}
+    ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+  `;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      limit: limit,
+      offset: limit && page ? limit * page : 0,
+      ...tracesFilterRes.params,
+      ...observationFilterRes.params,
+      ...scoresFilterRes.params,
+      ...search.params,
+    },
+  });
+
+  return res;
+};

packages/shared/src/tableDefinitions/mapDashboards.ts

@@ -19,12 +19,24 @@ export const dashboardColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "traces",
     clickhouseSelect: 't."timestamp"',
   },
+  {
+    clickhouseTableName: "scores",
+    clickhouseSelect: "name",
+    uiTableId: "scoreName",
+    uiTableName: "Score Name",
+  },
   {
     clickhouseTableName: "scores",
     clickhouseSelect: "timestamp",
     uiTableId: "scoreTimestamp",
     uiTableName: "Score Timestamp",
   },
+  {
+    clickhouseTableName: "scores",
+    clickhouseSelect: "source",
+    uiTableId: "scoreSource",
+    uiTableName: "Score Source",
+  },
   {
     clickhouseTableName: "scores",
     clickhouseSelect: "data_type",

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -19,7 +19,7 @@ export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [
   {
     uiTableName: "Trace Name",
     uiTableId: "traceName",
-    clickhouseTableName: "observations",
+    clickhouseTableName: "traces",
     clickhouseSelect: 't."name"',
   },
 ];
@@ -69,6 +69,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Latency (s)",
@@ -76,6 +78,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(end_time), NULL, date_diff('seconds', start_time, end_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Tokens per second",

packages/shared/src/tableDefinitions/mapScoresTable.ts

@@ -69,13 +69,13 @@ export const scoresTableUiColumnDefinitions: UiColumnMapping[] = [
   },
   {
     uiTableName: "Trace Name",
-    uiTableId: "trace_name",
+    uiTableId: "traceName",
     clickhouseTableName: "traces",
     clickhouseSelect: "t.name",
   },
   {
     uiTableName: "User ID",
-    uiTableId: "user_id",
+    uiTableId: "userId",
     clickhouseTableName: "traces",
     clickhouseSelect: "t.user_id",
   },

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -119,7 +119,7 @@ export const sessionCols: UiColumnMapping[] = [
   },
   {
     uiTableName: "ID",
-    uiTableId: "sessionId",
+    uiTableId: "id",
     clickhouseTableName: "traces",
     clickhouseSelect: "session_id",
   },

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -91,7 +91,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
   },
   {
     uiTableName: "Usage",
-    uiTableId: "totalTokens",
+    uiTableId: "usage",
     clickhouseTableName: "traces",
     clickhouseSelect:
       "if(mapExists((k, v) -> (k = 'total'), usage_details), usage_details['total'], NULL)",
@@ -106,27 +106,26 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableName: "Latency (s)",
     uiTableId: "latency",
     clickhouseTableName: "traces",
-    clickhouseSelect: "latency",
+    clickhouseSelect: "latency_milliseconds / 1000",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Input Cost ($)",
     uiTableId: "inputCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'input'), cost_details), usage_details['input'], NULL)",
+    clickhouseSelect: "cost_details['input']",
   },
   {
     uiTableName: "Output Cost ($)",
     uiTableId: "outputCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'output'), cost_details), usage_details['output'], NULL)",
+    clickhouseSelect: "cost_details['output']",
   },
   {
     uiTableName: "Total Cost ($)",
     uiTableId: "totalCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'total'), cost_details), usage_details['total'], NULL)",
+    clickhouseSelect: "cost_details['total']",
   },
 ];

packages/shared/src/tableDefinitions/tracesTable.ts

@@ -138,17 +138,44 @@ export const tracesTableCols: ColumnDefinition[] = [
   },
 ];
 
-export const evalTableCols: ColumnDefinition[] = tracesOnlyCols;
+export const datasetCol: ColumnDefinition = {
+  name: "Dataset",
+  id: "datasetId",
+  type: "stringOptions",
+  internal: 'di."dataset_id"',
+  options: [], // to be filled in at runtime
+};
 
+// Used only for dataset evaluator, not on dataset table
+export const datasetOnlyCols: ColumnDefinition[] = [datasetCol];
+
+export const evalTraceTableCols: ColumnDefinition[] = tracesOnlyCols;
+export const evalDatasetFormFilterCols: ColumnDefinition[] = datasetOnlyCols;
 export type TraceOptions = {
   scores_avg: Array<string>;
   name: Array<OptionsDefinition>;
   tags: Array<OptionsDefinition>;
 };
+export type DatasetOptions = {
+  datasetId: Array<OptionsDefinition>;
+};
+
+// Used only for dataset evaluator, not on dataset table
+export function datasetFormFilterColsWithOptions(
+  options?: DatasetOptions,
+  cols: ColumnDefinition[] = evalDatasetFormFilterCols,
+): ColumnDefinition[] {
+  return cols.map((col) => {
+    if (col.id === "datasetId") {
+      return { ...col, options: options?.datasetId ?? [] };
+    }
+    return col;
+  });
+}
 
 export function tracesTableColsWithOptions(
   options?: TraceOptions,
-  cols: ColumnDefinition[] = tracesTableCols
+  cols: ColumnDefinition[] = tracesTableCols,
 ): ColumnDefinition[] {
   return cols.map((col) => {
     if (col.id === "scores_avg") {

packages/shared/src/tableDefinitions/types.ts

@@ -3,12 +3,14 @@ export type UiColumnMapping = {
   uiTableId: string;
   clickhouseTableName: string;
   clickhouseSelect: string;
+  clickhouseTypeOverwrite?: string;
   queryPrefix?: string;
 };
 
 export type OptionsDefinition = {
   value: string;
   count?: number;
+  displayValue?: string; // FIX: Temporary workaround: Used to display a different value than the actual value since multiSelect doesn't support key-value pairs
 };
 
 export type ColumnDefinition =
@@ -55,6 +57,7 @@ export const tableNames = [
   "sessions",
   "prompts",
   "users",
+  "dataset_items",
 ] as const;
 
 export type TableNames = (typeof tableNames)[number];

packages/shared/src/types.ts

@@ -23,6 +23,7 @@ export type WipFilterState = WipFilterCondition[];
 export type FilterOption = {
   value: string;
   count?: number;
+  displayValue?: string; // FIX: Temporary workaround: Used to display a different value than the actual value since multiSelect doesn't support key-value pairs
 };
 
 export type TableName =

packages/shared/src/utils/stringChecks.ts

@@ -0,0 +1,23 @@
+export function getIsCharOrUnderscore(value: string): boolean {
+  const charOrUnderscore = /^[A-Za-z_]+$/;
+
+  return charOrUnderscore.test(value);
+}
+
+export function extractVariables(mustacheString: string): string[] {
+  const mustacheRegex = /\{\{(.*?)\}\}/g;
+  const uniqueVariables = new Set<string>();
+
+  for (const match of mustacheString.matchAll(mustacheRegex)) {
+    uniqueVariables.add(match[1]);
+  }
+
+  for (const variable of uniqueVariables) {
+    // if validated fails, remove from set
+    if (!getIsCharOrUnderscore(variable)) {
+      uniqueVariables.delete(variable);
+    }
+  }
+
+  return Array.from(uniqueVariables);
+}

patches/next-auth@4.24.11.patch

@@ -0,0 +1,26 @@
+diff --git a/core/lib/oauth/client.js b/core/lib/oauth/client.js
+index 52c51eb6ff422dc0899ccec31baf3fa39e42eeae..bc50c35bb617d0e86b68ca42f64d44b475ba4abb 100644
+--- a/core/lib/oauth/client.js
++++ b/core/lib/oauth/client.js
+@@ -1,5 +1,7 @@
+ "use strict";
+ 
++var HttpsProxyAgent = require('https-proxy-agent').HttpsProxyAgent;
++
+ Object.defineProperty(exports, "__esModule", {
+   value: true
+ });
+@@ -7,7 +9,12 @@ exports.openidClient = openidClient;
+ var _openidClient = require("openid-client");
+ async function openidClient(options) {
+   const provider = options.provider;
+-  if (provider.httpOptions) _openidClient.custom.setHttpOptionsDefaults(provider.httpOptions);
++  let httpOptions = {};
++  if (provider.httpOptions) httpOptions = { ...provider.httpOptions };
++  if (process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY) {
++    httpOptions.agent = new HttpsProxyAgent(process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY);
++  }
++  _openidClient.custom.setHttpOptionsDefaults(httpOptions);
+   let issuer;
+   if (provider.wellKnown) {
+     issuer = await _openidClient.Issuer.discover(provider.wellKnown);

web/Dockerfile

@@ -101,12 +101,16 @@ ENV DOCKER_BUILD 0
 # Set NEXT_MANUAL_SIG_HANDLE for runtime
 ENV NEXT_MANUAL_SIG_HANDLE true
 
-RUN apk add --no-cache dumb-init
+RUN apk add --no-cache dumb-init tzdata
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
-RUN npm install -g --no-package-lock --no-save prisma@5.20.0
+RUN npm install -g --no-package-lock --no-save prisma@5.22.0
+
+RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
+    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
+    mv migrate /usr/bin/migrate
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
 COPY --from=builder --chown=nextjs:nodejs /app/web/package.json .
@@ -118,6 +122,7 @@ COPY --from=builder --chown=nextjs:nodejs /app/web/.next/static ./web/.next/stat
 COPY --from=builder --chown=nextjs:nodejs /app/web/public ./web/public
 
 COPY --from=builder --chown=nextjs:nodejs /app/packages/shared/prisma ./packages/shared/prisma
+COPY --from=builder --chown=nextjs:nodejs /app/packages/shared/clickhouse ./packages/shared/clickhouse
 
 COPY --chown=nextjs:nodejs ./web/entrypoint.sh ./web/entrypoint.sh
 COPY --chown=nextjs:nodejs ./packages/shared/scripts/cleanup.sql ./packages/shared/scripts/cleanup.sql

web/entrypoint.sh

@@ -12,14 +12,19 @@ if [ -z "$DATABASE_URL" ]; then
         echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
         exit 1
     fi
+    if [ -n "$DATABASE_ARGS" ]; then
+        # Append ARGS to DATABASE_URL
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
+        export DATABASE_URL
+    fi
 fi
 
 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
-    export DIRECT_URL=$DATABASE_URL
+    export DIRECT_URL="${DATABASE_URL}"
 fi
 
-# Always execute the scripts, except when disabled.
+# Always execute the postgres migration, except when disabled.
 if [ "$LANGFUSE_AUTO_POSTGRES_MIGRATION_DISABLED" != "true" ]; then
     prisma db execute --url "$DIRECT_URL" --file "./packages/shared/scripts/cleanup.sql"
 
@@ -35,5 +40,21 @@ if [ $status -ne 0 ]; then
     exit $status
 fi
 
+# Execute the Clickhouse migration, except when disabled.
+if [ "$LANGFUSE_AUTO_CLICKHOUSE_MIGRATION_DISABLED" != "true" ]; then
+    # Apply Clickhouse migrations
+    cd ./packages/shared
+    sh ./clickhouse/scripts/up.sh
+    status=$?
+    cd ../../
+fi
+
+# If migration fails (returns non-zero exit status), exit script with that status
+if [ $status -ne 0 ]; then
+    echo "Applying clickhouse migrations failed. This is mostly caused by the database being unavailable."
+    echo "Exiting..."
+    exit $status
+fi
+
 # Run the command passed to the docker image on start
 exec "$@"

web/jest.config.mjs

@@ -13,8 +13,27 @@ const clientTestConfig = {
 };
 
 const serverTestConfig = {
-  displayName: "server",
+  displayName: "sync-server",
   testMatch: ["/**/*.servertest.[jt]s?(x)"],
+  testPathIgnorePatterns: ["async", "__e2e__"],
+  testEnvironment: "jest-environment-node",
+  setupFilesAfterEnv: ["<rootDir>/src/__tests__/after-teardown.ts"],
+  globalTeardown: "<rootDir>/src/__tests__/teardown.ts",
+};
+
+const asyncServerTestConfig = {
+  displayName: "async-server",
+  testPathIgnorePatterns: ["__e2e__"],
+  testMatch: ["/**/async/*.servertest.[jt]s?(x)"],
+  testEnvironment: "jest-environment-node",
+  setupFilesAfterEnv: ["<rootDir>/src/__tests__/after-teardown.ts"],
+  globalTeardown: "<rootDir>/src/__tests__/teardown.ts",
+};
+
+const endToEndServerTestConfig = {
+  displayName: "e2e-server",
+  testMatch: ["/**/*.servertest.[jt]s?(x)"],
+  testPathIgnorePatterns: ["__tests__"],
   testEnvironment: "jest-environment-node",
   setupFilesAfterEnv: ["<rootDir>/src/__tests__/after-teardown.ts"],
   globalTeardown: "<rootDir>/src/__tests__/teardown.ts",
@@ -34,6 +53,18 @@ const config = {
         `/web/node_modules/(?!(${esModules.join("|")})/)`,
       ],
     },
+    {
+      ...(await createJestConfig(asyncServerTestConfig)()),
+      transformIgnorePatterns: [
+        `/web/node_modules/(?!(${esModules.join("|")})/)`,
+      ],
+    },
+    {
+      ...(await createJestConfig(endToEndServerTestConfig)()),
+      transformIgnorePatterns: [
+        `/web/node_modules/(?!(${esModules.join("|")})/)`,
+      ],
+    },
   ],
 };