security: upgrade dompurify v2 (#5570)

security: upgrade dompurify

.env.dev-azure.example

@@ -0,0 +1,92 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 storage
+S3_ENDPOINT=http://localhost:10000/devstoreaccount1
+S3_ACCESS_KEY_ID=devstoreaccount1
+S3_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+S3_BUCKET_NAME=langfuse
+S3_REGION=auto
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=false
+LANGFUSE_RETURN_FROM_CLICKHOUSE=true
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
+
+LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_DISABLED="true"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -43,7 +43,7 @@ S3_REGION=us-east-1
 ## Necessary for minio compatibility
 S3_FORCE_PATH_STYLE=true
 
-# # S3 Media Upload LOCAL
+# S3 Media Upload LOCAL
 LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
@@ -80,8 +80,10 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 
-
 LANGFUSE_READ_FROM_POSTGRES_ONLY=false
 LANGFUSE_RETURN_FROM_CLICKHOUSE=true
 LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
 LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
+
+LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.legacy.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,12 +92,17 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -245,4 +255,4 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion

.github/workflows/codespell.yml

@@ -3,9 +3,14 @@ name: Codespell
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
   pull_request:
-    branches: [main]
+    branches:
+      - "**"
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -82,7 +82,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -137,11 +137,12 @@ jobs:
   tests-web-async:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -150,7 +151,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -172,11 +173,11 @@ jobs:
           pnpm install
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
@@ -205,11 +206,12 @@ jobs:
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -235,17 +237,17 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status
@@ -326,9 +328,15 @@ jobs:
       - name: install dependencies
         run: |
           pnpm install
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
@@ -337,14 +345,29 @@ jobs:
           docker compose -f docker-compose.dev.yml up -d
           docker compose ps
           sleep 5 # Wait for PostgreSQL to accept connections
+      - name: Ensure Docker dependencies are healthy
+        run: |
+          if docker-compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
       - name: Seed DB
         run: |
           pnpm run db:migrate
+          pnpm --filter=shared run ch:up
           pnpm run db:seed:examples
       - name: Build
         run: pnpm run build
       - name: Run server
         run: (pnpm run start&)
+      - name: Check worker health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3030/api/health; do sleep 2; done'
+      - name: Check server health
+        run: |
+          timeout 10 bash -c 'until curl -f http://localhost:3000/api/public/health; do sleep 2; done'
       - name: Run e2e tests
         run: pnpm --filter=web run test:e2e:server
 
@@ -418,6 +441,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -425,6 +450,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.gitignore

@@ -37,6 +37,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env*
 !.env.dev.example
+!.env.dev-azure.example
 !.env.local.example
 !.env.prod.example
 !.env.dev.legacy.example

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.v3preview.yml

@@ -25,7 +25,7 @@ services:
       CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
       CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      CLICKHOUSE_MIGRATION_CLUSTER_DISABLED: ${CLICKHOUSE_MIGRATION_CLUSTER_DISABLED:-true}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
       LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}

ee/package.json

@@ -27,8 +27,9 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "https-proxy-agent": "^7.0.6",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +48,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.2.0"
     }
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.91.0",
+  "version": "2.95.1",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -83,7 +83,12 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
     }
   },
   "packageManager": "pnpm@9.5.0"

packages/shared/clickhouse/scripts/down.sh

@@ -19,7 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
   else

packages/shared/clickhouse/scripts/seed.ts

@@ -65,7 +65,7 @@ async function main() {
   } catch (error) {
     console.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     console.log("Disconnected from Clickhouse.");

packages/shared/clickhouse/scripts/up.sh

@@ -19,7 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
   else

packages/shared/package.json

@@ -59,11 +59,12 @@
     "@aws-sdk/client-s3": "^3.675.0",
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.1",
-    "@langchain/aws": "^0.1.0",
-    "@langchain/core": "^0.3.9",
-    "@langchain/openai": "^0.3.0",
+    "@langchain/anthropic": "^0.3.8",
+    "@langchain/aws": "^0.1.2",
+    "@langchain/core": "^0.3.18",
+    "@langchain/openai": "^0.3.14",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "@prisma/client": "^5.22.0",
     "@react-email/components": "^0.0.19",
@@ -75,11 +76,13 @@
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.2",
+    "langchain": "^0.3.6",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",
@@ -118,7 +121,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
   }
 }

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/scripts/load-seed.ts

@@ -124,7 +124,7 @@ async function main() {
   } catch (error) {
     logger.error("Error during Clickhouse preparation:", error);
   } finally {
-    await clickhouseClient.close();
+    await clickhouseClient().close();
     await prisma.$disconnect();
     redis?.disconnect();
     logger.info("Disconnected from Clickhouse.");

packages/shared/scripts/prepareClickhouse.ts

@@ -160,7 +160,7 @@ export const prepareClickhouse = async (
 
     for (const query of queries) {
       logger.info(`Executing query: ${query}`);
-      await clickhouseClient.command({
+      await clickhouseClient().command({
         query,
         clickhouse_settings: {
           wait_end_of_query: 1,
@@ -174,7 +174,7 @@ export const prepareClickhouse = async (
       FROM traces 
       WHERE session_id IS NOT NULL;
     `;
-    const sessionResult = await clickhouseClient.query({
+    const sessionResult = await clickhouseClient().query({
       query: sessionQuery,
       format: "JSONEachRow",
     });
@@ -212,7 +212,7 @@ export const prepareClickhouse = async (
           ORDER BY count() desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });
@@ -243,7 +243,7 @@ export const prepareClickhouse = async (
           ORDER BY event_date desc
           `;
 
-    const result = await clickhouseClient.query({
+    const result = await clickhouseClient().query({
       query,
       format: "TabSeparated",
     });

packages/shared/src/env.ts

@@ -22,7 +22,7 @@ const EnvSchema = z.object({
     .string()
     .length(
       64,
-      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32"
+      "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
     )
     .optional(),
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
@@ -30,6 +30,16 @@ const EnvSchema = z.object({
   CLICKHOUSE_URL: z.string().url().optional(),
   CLICKHOUSE_USER: z.string().optional(),
   CLICKHOUSE_PASSWORD: z.string().optional(),
+  LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_ASYNC_INGESTION_PROCESSING: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(15_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -48,6 +58,8 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
 });
 
 export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/index.ts

@@ -11,6 +11,7 @@ export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";

packages/shared/src/server/auth/customSsoProvider.ts

@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
     authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
-    idToken: true,
     profile(profile) {
       return {
         id: profile.sub,

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/clickhouse/client.ts

@@ -1,19 +1,23 @@
 import { createClient } from "@clickhouse/client";
 import { env } from "../../env";
+import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
 export type ClickhouseClientType = ReturnType<typeof createClient>;
 
-export const clickhouseClient = createClient({
-  url: env.CLICKHOUSE_URL,
-  username: env.CLICKHOUSE_USER,
-  password: env.CLICKHOUSE_PASSWORD,
-  database: "default",
-  clickhouse_settings: {
-    async_insert: 1,
-    wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
-  },
-});
+export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
+  createClient({
+    ...opts,
+    url: env.CLICKHOUSE_URL,
+    username: env.CLICKHOUSE_USER,
+    password: env.CLICKHOUSE_PASSWORD,
+    database: "default",
+    clickhouse_settings: {
+      async_insert: 1,
+      wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
+    },
+  });
 
+export const defaultClickhouseClient = clickhouseClient();
 /**
  * Accepts a JavaScript date and returns the DateTime in format YYYY-MM-DD HH:MM:SS
  */

packages/shared/src/server/index.ts

@@ -1,10 +1,12 @@
-export * from "./services/S3StorageService";
+export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
+export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -15,14 +17,18 @@ export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
 export * from "./ingestion/modelMatch";
+export * from "./ingestion/processEventBatch";
 export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
+export * from "./redis/CloudUsageMeteringQueue";
+export * from "./redis/getQueue";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
 export * from "./ingestion/legacy/index";
 export * from "./queues";
@@ -33,3 +39,4 @@ export * from "./instrumentation";
 export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
+export * from "./redis/evalExecutionQueue";

packages/shared/src/server/ingestion/legacy/index.ts

@@ -3,13 +3,7 @@ import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
 import { getProcessorForEvent } from "./EventProcessor";
-import { TraceUpsertEventType } from "../../queues";
-import {
-  convertTraceUpsertEventsToRedisEvents,
-  TraceUpsertQueue,
-} from "../../redis/traceUpsert";
 import { ApiAccessScope } from "../../auth/types";
-import { redis } from "../../redis/redis";
 import { backOff } from "exponential-backoff";
 import { Model } from "../../..";
 import { logger } from "../../logger";
@@ -163,45 +157,5 @@ export function cleanEvent(obj: unknown): unknown {
   }
 }
 
-export const isNotNullOrUndefined = <T>(
-  val?: T | null,
-): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
-
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
-
-export const addTracesToTraceUpsertQueue = async (
-  batchResults: BatchResult[],
-  projectId: string,
-): Promise<void> => {
-  const traceEvents: TraceUpsertEventType[] = batchResults
-    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
-    .map((result) =>
-      result.result &&
-      typeof result.result === "object" &&
-      "id" in result.result
-        ? // ingestion API only gets traces for one projectId
-          {
-            traceId: result.result.id as string,
-            projectId,
-          }
-        : null,
-    )
-    .filter(isNotNullOrUndefined);
-
-  try {
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
-
-      const queue = TraceUpsertQueue.getInstance();
-      if (!queue) {
-        logger.error("TraceUpsertQueue not initialized");
-        return;
-      }
-
-      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    }
-  } catch (error) {
-    logger.error("Error sending events to worker", error);
-  }
-};

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -0,0 +1,396 @@
+import { randomUUID } from "crypto";
+import { z } from "zod";
+
+import { type Model } from "../../db";
+import { env } from "../../env";
+import {
+  InvalidRequestError,
+  LangfuseNotFoundError,
+  UnauthorizedError,
+} from "../../errors";
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
+import {
+  getCurrentSpan,
+  instrumentAsync,
+  instrumentSync,
+  recordIncrement,
+  traceException,
+} from "../instrumentation";
+import { logger } from "../logger";
+import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { IngestionQueue } from "../redis/ingestionQueue";
+import { LegacyIngestionQueue } from "../redis/legacyIngestion";
+import { redis } from "../redis/redis";
+import { handleBatch } from "./legacy";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
+import { getProcessorForEvent } from "./legacy/EventProcessor";
+import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+
+export type TokenCountDelegate = (p: {
+  model: Model;
+  text: unknown;
+}) => number | undefined;
+
+let s3StorageServiceClient: StorageService;
+
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
+  }
+  return s3StorageServiceClient;
+};
+
+export const processEventBatch = async (
+  input: unknown[],
+  authCheck: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): Promise<{
+  successes: { id: string; status: number }[];
+  errors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[];
+}> => {
+  // add context of api call to the span
+  const currentSpan = getCurrentSpan();
+  recordIncrement("langfuse.ingestion.event", input.length);
+  currentSpan?.setAttribute("event_count", input.length);
+
+  /**************
+   * VALIDATION *
+   **************/
+  const validationErrors: { id: string; error: unknown }[] = [];
+  const authenticationErrors: { id: string; error: unknown }[] = [];
+
+  const batch: z.infer<typeof ingestionEvent>[] = input
+    .flatMap((event) => {
+      const parsed = instrumentSync(
+        { name: "ingestion-zod-parse-individual-event" },
+        (span) => {
+          const parsedBody = ingestionEvent.safeParse(event);
+          if (parsedBody.data?.id !== undefined) {
+            span.setAttribute("object.id", parsedBody.data.id);
+          }
+          return parsedBody;
+        },
+      );
+      if (!parsed.success) {
+        validationErrors.push({
+          id:
+            typeof event === "object" && event && "id" in event
+              ? typeof event.id === "string"
+                ? event.id
+                : "unknown"
+              : "unknown",
+          error: new InvalidRequestError(parsed.error.message),
+        });
+        return [];
+      }
+      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+        authenticationErrors.push({
+          id: parsed.data.id,
+          error: new UnauthorizedError("Access Scope Denied"),
+        });
+        return [];
+      }
+      return [parsed.data];
+    })
+    .flatMap((event) => {
+      if (event.type === eventTypes.SDK_LOG) {
+        // Log SDK_LOG events, but remove them from further processing
+        logger.info("SDK Log Event", { event });
+        return [];
+      }
+      return [event];
+    });
+
+  const sortedBatch = sortBatch(batch);
+
+  // We group events by eventBodyId which allows us to store and process them
+  // as one which reduces infra interactions per event. Only used in the S3 case.
+  const sortedBatchByEventBodyId = sortedBatch.reduce(
+    (
+      acc: Record<
+        string,
+        {
+          data: IngestionEventType[];
+          key: string;
+          eventBodyId: string;
+          type: (typeof eventTypes)[keyof typeof eventTypes];
+        }
+      >,
+      event,
+    ) => {
+      if (!event.body?.id) {
+        return acc;
+      }
+      const key = `${getClickhouseEntityType(event.type)}-${event.body.id}`;
+      if (!acc[key]) {
+        acc[key] = {
+          data: [],
+          key: event.id,
+          type: event.type,
+          eventBodyId: event.body.id,
+        };
+      }
+      acc[key].data.push(event);
+      return acc;
+    },
+    {},
+  );
+
+  /********************
+   * ASYNC PROCESSING *
+   ********************/
+  let s3UploadErrored = false;
+  if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
+    await instrumentAsync({ name: "s3-upload-events" }, async () => {
+      if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
+        throw new Error("S3 event store is enabled but no bucket is set");
+      }
+      const s3Client = getS3StorageServiceClient(
+        env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+      );
+      // S3 Event Upload is currently blocking, but non-failing.
+      // If a promise rejects, we log it below, but do not throw an error.
+      // In this case, we upload the full batch into the Redis queue.
+      const results = await Promise.allSettled(
+        Object.keys(sortedBatchByEventBodyId).map(async (id) => {
+          // We upload the event in an array to the S3 bucket grouped by the eventBodyId.
+          // That way we batch updates from the same invocation into a single file and reduce
+          // write operations on S3.
+          const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+          return s3Client.uploadJson(
+            `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+            data,
+          );
+        }),
+      );
+      results.forEach((result) => {
+        if (result.status === "rejected") {
+          s3UploadErrored = true;
+          logger.error("Failed to upload event to S3", {
+            error: result.reason,
+          });
+        }
+      });
+    });
+  }
+
+  // Send each event individually to IngestionQueue for new processing
+  if (
+    env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" &&
+    env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" &&
+    env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING === "true" &&
+    redis &&
+    !s3UploadErrored
+  ) {
+    const queue = IngestionQueue.getInstance();
+    const results = await Promise.allSettled(
+      Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+        queue
+          ? queue.add(
+              QueueJobs.IngestionJob,
+              {
+                id: randomUUID(),
+                timestamp: new Date(),
+                name: QueueJobs.IngestionJob as const,
+                payload: {
+                  data: {
+                    type: sortedBatchByEventBodyId[id].type,
+                    eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                  },
+                  authCheck,
+                },
+              },
+              {
+                delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
+              },
+            )
+          : Promise.reject("Failed to instantiate queue"),
+      ),
+    );
+    results.forEach((result) => {
+      if (result.status === "rejected") {
+        logger.error("Failed to add event to IngestionQueue", {
+          error: result.reason,
+        });
+      }
+    });
+  }
+
+  // As part of the legacy processing we sent the entire batch to the worker.
+  if (env.LANGFUSE_ASYNC_INGESTION_PROCESSING === "true" && redis) {
+    const queue = LegacyIngestionQueue.getInstance();
+
+    if (queue) {
+      let addToQueueFailed = false;
+
+      const queuePayload: LegacyIngestionEventType =
+        env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true" && !s3UploadErrored
+          ? {
+              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
+                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
+                return {
+                  type,
+                  eventBodyId,
+                  eventId: key,
+                };
+              }),
+              authCheck,
+              useS3EventStore: true,
+            }
+          : { data: sortedBatch, authCheck, useS3EventStore: false };
+
+      try {
+        await queue.add(QueueJobs.LegacyIngestionJob, {
+          payload: queuePayload,
+          id: randomUUID(),
+          timestamp: new Date(),
+          name: QueueJobs.LegacyIngestionJob as const,
+        });
+      } catch (e: unknown) {
+        logger.warn(
+          "Failed to add batch to queue, falling back to sync processing",
+          e,
+        );
+        addToQueueFailed = true;
+      }
+
+      if (!addToQueueFailed) {
+        return aggregateBatchResult(
+          // we are not sending additional server errors to the client in case of early return
+          [...validationErrors, ...authenticationErrors],
+          sortedBatch.map((event) => ({ id: event.id, result: event })),
+        );
+      }
+    } else {
+      logger.error(
+        "Ingestion queue not initialized, falling back to sync processing",
+      );
+    }
+  }
+
+  /*******************
+   * SYNC PROCESSING *
+   *******************/
+  const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
+
+  //  in case we did not return early, we return the result here
+  return aggregateBatchResult(
+    [...validationErrors, ...authenticationErrors, ...result.errors],
+    result.results,
+  );
+};
+
+const isAuthorized = (
+  event: IngestionEventType,
+  authScope: AuthHeaderValidVerificationResult,
+  tokenCountDelegate: TokenCountDelegate,
+): boolean => {
+  try {
+    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+    return true;
+  } catch (error) {
+    return false;
+  }
+};
+
+/**
+ * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
+ */
+const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+  const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
+    eventTypes.GENERATION_UPDATE,
+    eventTypes.SPAN_UPDATE,
+    eventTypes.OBSERVATION_UPDATE, // legacy event type
+  ];
+  const updates = batch
+    .filter((event) => updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+  const others = batch
+    .filter((event) => !updateEvents.includes(event.type))
+    .sort((a, b) => {
+      return new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime();
+    });
+
+  // Return the array with non-update events first, followed by update events
+  return [...others, ...updates];
+};
+
+export const aggregateBatchResult = (
+  errors: Array<{ id: string; error: unknown }>,
+  results: Array<{ id: string; result: unknown }>,
+) => {
+  const returnedErrors: {
+    id: string;
+    status: number;
+    message?: string;
+    error?: string;
+  }[] = [];
+
+  const successes: {
+    id: string;
+    status: number;
+  }[] = [];
+
+  errors.forEach((error) => {
+    if (error.error instanceof InvalidRequestError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 400,
+        message: "Invalid request data",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof UnauthorizedError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 401,
+        message: "Authentication error",
+        error: error.error.message,
+      });
+    } else if (error.error instanceof LangfuseNotFoundError) {
+      returnedErrors.push({
+        id: error.id,
+        status: 404,
+        message: "Resource not found",
+        error: error.error.message,
+      });
+    } else {
+      returnedErrors.push({
+        id: error.id,
+        status: 500,
+        error: "Internal Server Error",
+      });
+    }
+  });
+
+  if (returnedErrors.length > 0) {
+    traceException(errors);
+    logger.error("Error processing events", returnedErrors);
+  }
+
+  results.forEach((result) => {
+    successes.push({
+      id: result.id,
+      status: 201,
+    });
+  });
+
+  return { successes, errors: returnedErrors };
+};

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,5 +1,7 @@
 import type { ZodSchema } from "zod";
 
+import { CallbackHandler } from "langfuse-langchain";
+
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatBedrockConverse } from "@langchain/aws";
 import {
@@ -17,11 +19,27 @@ import {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-
+import { AuthHeaderValidVerificationResult } from "../auth/types";
+import {
+  processEventBatch,
+  type TokenCountDelegate,
+} from "../ingestion/processEventBatch";
+import { logger } from "../logger";
 import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
 
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
+type ProcessTracedEvents = () => Promise<void>;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};
+
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
@@ -31,6 +49,7 @@ type LLMCompletionParams = {
   apiKey: string;
   maxRetries?: number;
   config?: Record<string, string> | null;
+  traceParams?: TraceParams;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -40,25 +59,34 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: true;
-  }
-): Promise<IterableReadableStream<Uint8Array>>;
+  },
+): Promise<{
+  completion: IterableReadableStream<Uint8Array>;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
-  }
-): Promise<string>;
+  },
+): Promise<{ completion: string; processTracedEvents: ProcessTracedEvents }>;
 
 export async function fetchLLMCompletion(
   params: LLMCompletionParams & {
     streaming: false;
     structuredOutputSchema: ZodSchema;
-  }
-): Promise<unknown>;
+  },
+): Promise<{
+  completion: unknown;
+  processTracedEvents: ProcessTracedEvents;
+}>;
 
 export async function fetchLLMCompletion(
-  params: FetchLLMCompletionParams
-): Promise<string | IterableReadableStream<Uint8Array> | unknown> {
+  params: FetchLLMCompletionParams,
+): Promise<{
+  completion: string | IterableReadableStream<Uint8Array> | unknown;
+  processTracedEvents: ProcessTracedEvents;
+}> {
   // the apiKey must never be printed to the console
   const {
     messages,
@@ -69,8 +97,39 @@ export async function fetchLLMCompletion(
     baseURL,
     maxRetries,
     config,
+    traceParams,
   } = params;
 
+  let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
+  let processTracedEvents: ProcessTracedEvents = () => Promise.resolve();
+
+  if (traceParams) {
+    const handler = new CallbackHandler({
+      _projectId: traceParams.projectId,
+      _isLocalEventExportEnabled: true,
+      tags: traceParams.tags,
+    });
+
+    finalCallbacks.push(handler);
+
+    processTracedEvents = async () => {
+      try {
+        const events = await handler.langfuse._exportLocalEvents(
+          traceParams.projectId,
+        );
+        await processEventBatch(
+          JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
+          traceParams.authCheck,
+          traceParams.tokenCountDelegate,
+        );
+      } catch (e) {
+        logger.error("Failed to process traced events", { error: e });
+      }
+    };
+  }
+
+  finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
+
   const finalMessages = messages.map((message) => {
     if (message.role === ChatMessageRole.User)
       return new HumanMessage(message.content);
@@ -89,7 +148,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       clientOptions: { maxRetries },
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
@@ -100,7 +159,7 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       streamUsage: false, // https://github.com/langchain-ai/langchainjs/issues/6533
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
       configuration: {
         baseURL,
@@ -115,7 +174,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
@@ -129,7 +188,7 @@ export async function fetchLLMCompletion(
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
-      callbacks,
+      callbacks: finalCallbacks,
       maxRetries,
     });
   } else {
@@ -138,10 +197,19 @@ export async function fetchLLMCompletion(
     throw new Error("This model provider is not supported.");
   }
 
+  const runConfig = {
+    callbacks: finalCallbacks,
+    runId: traceParams?.traceId,
+    runName: traceParams?.traceName,
+  };
+
   if (params.structuredOutputSchema) {
-    return await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-      .withStructuredOutput(params.structuredOutputSchema)
-      .invoke(finalMessages);
+    return {
+      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+        .withStructuredOutput(params.structuredOutputSchema)
+        .invoke(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
   /*
@@ -157,27 +225,41 @@ export async function fetchLLMCompletion(
   Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
   */
   if (modelParams.model.startsWith("o1-")) {
-    return await new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      temperature: 1,
-      maxTokens: undefined,
-      topP: undefined,
-      callbacks,
-      maxRetries,
-      configuration: {
-        baseURL,
-      },
-    })
-      .pipe(new StringOutputParser())
-      .invoke(
-        finalMessages.filter((message) => message._getType() !== "system")
-      );
+    return {
+      completion: await new ChatOpenAI({
+        openAIApiKey: apiKey,
+        modelName: modelParams.model,
+        temperature: 1,
+        maxTokens: undefined,
+        topP: undefined,
+        callbacks,
+        maxRetries,
+        configuration: {
+          baseURL,
+        },
+      })
+        .pipe(new StringOutputParser())
+        .invoke(
+          finalMessages.filter((message) => message._getType() !== "system"),
+          runConfig,
+        ),
+      processTracedEvents,
+    };
   }
 
   if (streaming) {
-    return chatModel.pipe(new BytesOutputParser()).stream(finalMessages);
+    return {
+      completion: await chatModel
+        .pipe(new BytesOutputParser())
+        .stream(finalMessages, runConfig),
+      processTracedEvents,
+    };
   }
 
-  return await chatModel.pipe(new StringOutputParser()).invoke(finalMessages);
+  return {
+    completion: await chatModel
+      .pipe(new StringOutputParser())
+      .invoke(finalMessages, runConfig),
+    processTracedEvents,
+  };
 }

packages/shared/src/server/llm/types.ts

@@ -26,6 +26,20 @@ export enum ChatMessageRole {
 
 export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
 
+const ChatMessageSchema = z.object({
+  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+
+export const PromptContentSchema = z.union([
+  ChatMessageListSchema,
+  TextPromptSchema,
+]);
+export type PromptContent = z.infer<typeof PromptContentSchema>;
+
 export type ModelParams = {
   provider: string;
   adapter: LLMAdapter;
@@ -49,6 +63,17 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
+// Experiment config
+export const ExperimentMetadataSchema = z
+  .object({
+    prompt_id: z.string(),
+    provider: z.string(),
+    model: z.string(),
+    model_params: ZodModelConfig,
+  })
+  .strict();
+export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
+
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -1,20 +1,13 @@
 import { filterOperators } from "../../../interfaces/filters";
+import { clickhouseCompliantRandomCharacters } from "../../repositories";
 
-function randomCharacters() {
-  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
-  let result = "";
-  const randomArray = new Uint8Array(5);
-  crypto.getRandomValues(randomArray);
-  randomArray.forEach((number) => {
-    result += chars[number % chars.length];
-  });
-  return result;
-}
-
+export type ClickhouseOperator =
+  | (typeof filterOperators)[keyof typeof filterOperators][number]
+  | "!=";
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
-  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  operator: ClickhouseOperator;
   field: string;
 }
 type ClickhouseFilter = {
@@ -44,7 +37,7 @@ export class StringFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varName = `stringFilter${randomCharacters()}`;
+    const varName = `stringFilter${clickhouseCompliantRandomCharacters()}`;
 
     const fieldWithPrefix = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     let query: string;
@@ -79,29 +72,33 @@ export class NumberFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
   public value: number;
-  public operator: (typeof filterOperators)["number"][number];
+  public operator: (typeof filterOperators)["number"][number] | "!=";
+  public clickhouseTypeOverwrite?: string;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["number"][number];
+    operator: (typeof filterOperators)["number"][number] | "!=";
     value: number;
     tablePrefix?: string;
+    clickhouseTypeOverwrite?: string;
   }) {
     this.clickhouseTable = opts.clickhouseTable;
     this.field = opts.field;
     this.value = opts.value;
     this.operator = opts.operator;
     this.tablePrefix = opts.tablePrefix;
+    this.clickhouseTypeOverwrite = opts.clickhouseTypeOverwrite;
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `numberFilter${uid}`;
+    const type = this.clickhouseTypeOverwrite ?? "Decimal64(12)";
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Decimal}`,
-      params: { [varName]: this.value },
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: ${type}}`,
+      params: { [varName]: this.value.toString() },
     };
   }
 }
@@ -128,7 +125,7 @@ export class DateTimeFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `dateTimeFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: DateTime64(3)}`,
@@ -159,7 +156,7 @@ export class StringOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `stringOptionsFilter${uid}`;
     return {
       query:
@@ -198,8 +195,8 @@ export class StringObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `stringObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `stringObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `stringObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `stringObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
 
     //  const query: `${column}['{varKeyName: String}'] ${this.operator} {${varValueName}: String}`,
@@ -254,7 +251,7 @@ export class ArrayOptionsFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `arrayOptionsFilter${uid}`;
     let query: string;
 
@@ -266,7 +263,7 @@ export class ArrayOptionsFilter implements Filter {
         query = `hasAny({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`;
         break;
       case "all of":
-        query = `arrayAll(x -> has({${varName}: Array(String)}, x), ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`;
+        query = `hasAll(${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}, {${varName}: Array(String)}) = True`;
         break;
       default:
         throw new Error(`Unsupported operator: ${this.operator}`);
@@ -310,13 +307,13 @@ export class NumberObjectFilter implements Filter {
   public field: string;
   public key: string;
   public value: number;
-  public operator: (typeof filterOperators)["numberObject"][number];
+  public operator: (typeof filterOperators)["numberObject"][number] | "!=";
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["numberObject"][number];
+    operator: (typeof filterOperators)["numberObject"][number] | "!=";
     key: string;
     value: number;
     tablePrefix?: string;
@@ -330,11 +327,11 @@ export class NumberObjectFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const varKeyName = `numberObjectKeyFilter${randomCharacters()}`;
-    const varValueName = `numberObjectValueFilter${randomCharacters()}`;
+    const varKeyName = `numberObjectKeyFilter${clickhouseCompliantRandomCharacters()}`;
+    const varValueName = `numberObjectValueFilter${clickhouseCompliantRandomCharacters()}`;
     const column = `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}`;
     return {
-      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal})), ${column})) = 0`,
+      query: `empty(arrayFilter(x -> (((x.1) = {${varKeyName}: String}) AND ((x.2) ${this.operator} {${varValueName}: Decimal64(12)})), ${column})) = 0`,
       params: { [varKeyName]: this.key, [varValueName]: this.value },
     };
   }
@@ -362,7 +359,7 @@ export class BooleanFilter implements Filter {
   }
 
   apply(): ClickhouseFilter {
-    const uid = randomCharacters();
+    const uid = clickhouseCompliantRandomCharacters();
     const varName = `booleanFilter${uid}`;
     return {
       query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Boolean}`,
@@ -374,7 +371,7 @@ export class BooleanFilter implements Filter {
 export class FilterList {
   private filters: Filter[];
 
-  constructor(filters: Filter[]) {
+  constructor(filters: Filter[] = []) {
     this.filters = filters;
   }
 
@@ -386,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  length() {
+    return this.filters.length;
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -67,6 +67,7 @@ export const createFilterFromFilterState = (
           operator: frontEndFilter.operator,
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
+          clickhouseTypeOverwrite: column.clickhouseTypeOverwrite,
         });
       case "arrayOptions":
         return new ArrayOptionsFilter({

packages/shared/src/server/queries/createTracesQuery.ts

@@ -11,13 +11,13 @@ export function parseTraceAllFilters(input: TableFilters) {
   const filterCondition = tableColumnsToSqlFilterAndPrefix(
     input.filter ?? [],
     tracesTableCols,
-    "traces"
+    "traces",
   );
   const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
 
   // to improve query performance, add timeseries filter to observation queries as well
   const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime"
+    (f) => f.column === "Timestamp" && f.type === "datetime",
   );
 
   const observationTimeseriesFilter =
@@ -25,7 +25,7 @@ export function parseTraceAllFilters(input: TableFilters) {
       ? datetimeFilterToPrismaSql(
           "start_time",
           timeseriesFilter.operator,
-          timeseriesFilter.value
+          timeseriesFilter.value,
         )
       : Prisma.empty;
 
@@ -130,6 +130,6 @@ export function createTracesQuery({
     ${filterCondition}
   ${orderByCondition}
   ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
+  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
 `;
 }

packages/shared/src/server/queries/index.ts

@@ -7,3 +7,16 @@ export {
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
+export {
+  FilterList,
+  StringFilter,
+  DateTimeFilter,
+  StringOptionsFilter,
+  NumberFilter,
+  ArrayOptionsFilter,
+  BooleanFilter,
+  NumberObjectFilter,
+  StringObjectFilter,
+  NullFilter,
+  type ClickhouseOperator,
+} from "./clickhouse-sql/clickhouse-filter";

packages/shared/src/server/queues.ts

@@ -7,6 +7,7 @@ export enum EventName {
   EvaluationExecution = "EvaluationExecution",
   LegacyIngestion = "LegacyIngestion",
   CloudUsageMetering = "CloudUsageMetering",
+  ExperimentCreate = "ExperimentCreate",
 }
 
 export const LegacyIngestionEventFull = z.object({
@@ -78,6 +79,13 @@ export const EvalExecutionEvent = z.object({
   delay: z.number().nullish(),
 });
 
+export const ExperimentCreateEventSchema = z.object({
+  projectId: z.string(),
+  datasetId: z.string(),
+  runId: z.string(),
+  description: z.string().optional(),
+});
+
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
@@ -86,6 +94,9 @@ export type DatasetRunItemUpsertEventType = z.infer<
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
 export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
+export type ExperimentCreateEventType = z.infer<
+  typeof ExperimentCreateEventSchema
+>;
 
 export const EventBodySchema = z.union([
   z.object({
@@ -100,6 +111,10 @@ export const EventBodySchema = z.union([
     name: z.literal(EventName.BatchExport),
     payload: BatchExportJobSchema,
   }),
+  z.object({
+    name: z.literal(EventName.ExperimentCreate),
+    payload: ExperimentCreateEventSchema,
+  }),
 ]);
 export type EventBodyType = z.infer<typeof EventBodySchema>;
 
@@ -111,6 +126,7 @@ export enum QueueName {
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
   LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  ExperimentCreate = "experiment-create-queue",
 }
 
 export enum QueueJobs {
@@ -122,6 +138,7 @@ export enum QueueJobs {
   LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  ExperimentCreateJob = "experiment-create-job",
 }
 
 export type TQueueJobTypes = {
@@ -161,4 +178,10 @@ export type TQueueJobTypes = {
     payload: IngestionEventQueueType;
     name: QueueJobs.IngestionJob;
   };
+  [QueueName.ExperimentCreate]: {
+    timestamp: Date;
+    id: string;
+    payload: ExperimentCreateEventType;
+    name: QueueJobs.ExperimentCreateJob;
+  };
 };

packages/shared/src/server/redis/CloudUsageMeteringQueue.ts

@@ -0,0 +1,61 @@
+import { Queue } from "bullmq";
+import { env } from "../..";
+import { logger } from "@azure/storage-blob";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class CloudUsageMeteringQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudUsageMeteringQueue.instance) {
+      return CloudUsageMeteringQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudUsageMeteringQueue.instance = newRedis
+      ? new Queue(QueueName.CloudUsageMeteringQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudUsageMeteringQueue.instance?.on("error", (err) => {
+      logger.error("CloudUsageMeteringQueue error", err);
+    });
+
+    if (CloudUsageMeteringQueue.instance) {
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {
+          repeat: { pattern: "5 * * * *" },
+        },
+      );
+
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {},
+      );
+    }
+
+    return CloudUsageMeteringQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -27,7 +27,8 @@ export class DatasetRunItemUpsertQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 5,
+              delay: 30_000, // 30 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class EvalExecutionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.EvaluationExecution]
+  > | null {
+    if (EvalExecutionQueue.instance) return EvalExecutionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    EvalExecutionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.EvaluationExecution]>(
+          QueueName.EvaluationExecution,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 10,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    EvalExecutionQueue.instance?.on("error", (err) => {
+      logger.error("EvalExecutionQueue error", err);
+    });
+
+    return EvalExecutionQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class ExperimentCreateQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null {
+    if (ExperimentCreateQueue.instance) return ExperimentCreateQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ExperimentCreateQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ExperimentCreate]>(
+          QueueName.ExperimentCreate,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ExperimentCreateQueue.instance?.on("error", (err) => {
+      logger.error("ExperimentCreateQueue error", err);
+    });
+
+    return ExperimentCreateQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -0,0 +1,34 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { BatchExportQueue } from "./batchExport";
+import { CloudUsageMeteringQueue } from "./CloudUsageMeteringQueue";
+import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
+import { EvalExecutionQueue } from "./evalExecutionQueue";
+import { ExperimentCreateQueue } from "./experimentCreateQueue";
+import { IngestionQueue } from "./ingestionQueue";
+import { LegacyIngestionQueue } from "./legacyIngestion";
+import { TraceUpsertQueue } from "./traceUpsert";
+
+export function getQueue(queueName: QueueName): Queue | null {
+  switch (queueName) {
+    case QueueName.LegacyIngestionQueue:
+      return LegacyIngestionQueue.getInstance();
+    case QueueName.BatchExport:
+      return BatchExportQueue.getInstance();
+    case QueueName.CloudUsageMeteringQueue:
+      return CloudUsageMeteringQueue.getInstance();
+    case QueueName.DatasetRunItemUpsert:
+      return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.EvaluationExecution:
+      return EvalExecutionQueue.getInstance();
+    case QueueName.ExperimentCreate:
+      return ExperimentCreateQueue.getInstance();
+    case QueueName.TraceUpsert:
+      return TraceUpsertQueue.getInstance();
+    case QueueName.IngestionQueue:
+      return IngestionQueue.getInstance();
+    default:
+      const exhaustiveCheckDefault: never = queueName;
+      throw new Error(`Queue ${queueName} not found`);
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -32,6 +32,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
+              delay: 10_000, // 10 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,
@@ -48,43 +49,3 @@ export class TraceUpsertQueue {
     return TraceUpsertQueue.instance;
   }
 }
-
-export function convertTraceUpsertEventsToRedisEvents(
-  events: TraceUpsertEventType[],
-) {
-  const uniqueTracesPerProject = events.reduce((acc, event) => {
-    if (!acc.get(event.projectId)) {
-      acc.set(event.projectId, new Set());
-    }
-    acc.get(event.projectId)?.add(event.traceId);
-    return acc;
-  }, new Map<string, Set<string>>());
-
-  return [...uniqueTracesPerProject.entries()]
-    .map((tracesPerProject) => {
-      const [projectId, traceIds] = tracesPerProject;
-
-      return [...traceIds].map((traceId) => ({
-        name: QueueJobs.TraceUpsert,
-        data: {
-          payload: {
-            projectId,
-            traceId,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.TraceUpsert as const,
-        },
-        opts: {
-          removeOnFail: 1_000,
-          removeOnComplete: true,
-          attempts: 5,
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-        },
-      }));
-    })
-    .flat();
-}

packages/shared/src/server/repositories/clickhouse.ts

@@ -5,15 +5,19 @@ import {
 } from "../clickhouse/client";
 import { logger } from "../logger";
 import { instrumentAsync } from "../instrumentation";
-import { S3StorageService } from "../services/S3StorageService";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
+import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -28,7 +32,7 @@ const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces"; // TODO: Modify eventType logic to support more tables going forward
+  table: "scores" | "traces" | "observations";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>;
 }): Promise<void> {
@@ -36,10 +40,6 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // drop trailing s and pretend it's always a create.
-    // Only applicable to scores and traces.
-    const eventType = `${opts.table.slice(0, -1)}-create`;
-
     // If event upload is enabled, we store all rows in S3 to have a backup
     if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
       if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
@@ -50,6 +50,13 @@ export async function upsertClickhouse<
       );
       await Promise.all(
         opts.records.map((record) => {
+          // drop trailing s and pretend it's always a create.
+          // Only applicable to scores and traces.
+          let eventType = `${opts.table.slice(0, -1)}-create`;
+          if (opts.table === "observations") {
+            // @ts-ignore - If it's an observation we now that `type` is a string
+            eventType = `${record["type"].toLowerCase()}-create`;
+          }
           s3Client.uploadJson(
             `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
             [
@@ -65,7 +72,7 @@ export async function upsertClickhouse<
       );
     }
 
-    const res = await clickhouseClient.insert({
+    const res = await clickhouseClient().insert({
       table: opts.table,
       values: opts.records.map((record) => ({
         ...record,
@@ -103,12 +110,13 @@ export async function upsertClickhouse<
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
 }): Promise<T[]> {
   return await instrumentAsync({ name: "clickhouse-query" }, async (span) => {
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.text", opts.query);
 
-    const res = await clickhouseClient.query({
+    const res = await clickhouseClient(opts.clickhouseConfigs).query({
       query: opts.query,
       format: "JSONEachRow",
       query_params: opts.params,
@@ -145,12 +153,12 @@ export async function queryClickhouse<T>(opts: {
 export async function commandClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
 }): Promise<void> {
   return await instrumentAsync({ name: "clickhouse-command" }, async (span) => {
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.text", opts.query);
-
-    const res = await clickhouseClient.command({
+    const res = await clickhouseClient(opts.clickhouseConfigs).command({
       query: opts.query,
       query_params: opts.params,
     });
@@ -184,3 +192,14 @@ export async function commandClickhouse<T>(opts: {
 export function parseClickhouseUTCDateTimeFormat(dateStr: string): Date {
   return new Date(`${dateStr.replace(" ", "T")}Z`);
 }
+
+export function clickhouseCompliantRandomCharacters() {
+  const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+  let result = "";
+  const randomArray = new Uint8Array(5);
+  crypto.getRandomValues(randomArray);
+  randomArray.forEach((number) => {
+    result += chars[number % chars.length];
+  });
+  return result;
+}

packages/shared/src/server/repositories/dashboards.ts

@@ -256,8 +256,8 @@ export const getDistinctModels = async (
       ) as DateTimeFilter | undefined)
     : undefined;
 
-    // No need for final as duplicates are caught by distinct anyway.
-    const query = `
+  // No need for final as duplicates are caught by distinct anyway.
+  const query = `
     SELECT distinct(provided_model_name) as model
     FROM observations o
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}

packages/shared/src/server/repositories/definitions.ts

@@ -129,8 +129,8 @@ export const scoreRecordBaseSchema = z.object({
   project_id: z.string(),
   trace_id: z.string(),
   observation_id: z.string().nullish(),
-  name: z.string().nullish(),
-  value: z.union([z.number(), z.string()]).nullish(),
+  name: z.string(),
+  value: z.number().nullish(),
   source: z.string(),
   comment: z.string().nullish(),
   author_user_id: z.string().nullish(),

packages/shared/src/server/repositories/index.ts

@@ -5,3 +5,5 @@ export * from "./types";
 export * from "./dashboards";
 export * from "./traces_converters";
 export * from "./scores_converters";
+export * from "./observations_converters";
+export * from "./clickhouse";

packages/shared/src/server/repositories/observations.ts

@@ -2,8 +2,9 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  upsertClickhouse,
 } from "./clickhouse";
-import { Observation, ObservationLevel } from "@prisma/client";
+import { ObservationLevel } from "@prisma/client";
 import { logger } from "../logger";
 import { InternalServerError, LangfuseNotFoundError } from "../../errors";
 import { prisma } from "../../db";
@@ -38,6 +39,58 @@ import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
 
+export const checkObservationExists = async (
+  projectId: string,
+  id: string,
+  startTime: Date | undefined,
+): Promise<boolean> => {
+  const query = `
+    SELECT id, project_id
+    FROM observations o
+    WHERE project_id = {projectId: String}
+    AND id = {id: String}
+    ${startTime ? `AND start_time >= {startTime: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    ORDER BY event_ts DESC
+    LIMIT 1 BY id, project_id
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
+
+/**
+ * Accepts a trace in a Clickhouse-ready format.
+ * id, project_id, and timestamp must always be provided.
+ */
+export const upsertObservation = async (
+  observation: Partial<ObservationRecordReadType>,
+) => {
+  if (
+    !["id", "project_id", "start_time", "type"].every(
+      (key) => key in observation,
+    )
+  ) {
+    throw new Error(
+      "Identifier fields must be provided to upsert Observation.",
+    );
+  }
+  await upsertClickhouse({
+    table: "observations",
+    records: [observation as ObservationRecordReadType],
+    eventBodyMapper: convertObservation,
+  });
+};
+
 export const getObservationsViewForTrace = async (
   traceId: string,
   projectId: string,
@@ -94,10 +147,170 @@ export const getObservationsViewForTrace = async (
   return records.map(convertObservationToView);
 };
 
+export const getObservationForTraceIdByName = async (
+  traceId: string,
+  projectId: string,
+  name: string,
+  timestamp?: Date,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations 
+  WHERE trace_id = {traceId: String}
+  AND project_id = {projectId: String}
+  AND name = {name: String}
+   ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  ORDER BY event_ts DESC
+  LIMIT 1 BY id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: {
+      traceId,
+      projectId,
+      name,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return records.map(convertObservationToView);
+};
+
 export const getObservationById = async (
   id: string,
   projectId: string,
   fetchWithInputOutput: boolean = false,
+) => {
+  const records = await getObservationByIdInternal(
+    id,
+    projectId,
+    fetchWithInputOutput,
+  );
+  const mapped = records.map(convertObservation);
+
+  if (mapped.length === 0) {
+    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
+  }
+
+  if (mapped.length > 1) {
+    logger.error(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+    throw new InternalServerError(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+  }
+  return mapped.shift();
+};
+
+export const getObservationsById = async (
+  ids: string[],
+  projectId: string,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations
+  WHERE id IN ({ids: Array(String)})
+  AND project_id = {projectId: String}
+  ORDER BY event_ts desc
+  LIMIT 1 by id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: { ids, projectId },
+  });
+  return records.map(convertObservation);
+};
+
+export const getObservationViewById = async (
+  id: string,
+  projectId: string,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const records = await getObservationByIdInternal(
+    id,
+    projectId,
+    fetchWithInputOutput,
+  );
+  const mapped = records.map(convertObservationToView);
+
+  if (mapped.length === 0) {
+    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
+  }
+
+  if (mapped.length > 1) {
+    logger.error(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+    throw new InternalServerError(
+      `Multiple observations found for id ${id} and project ${projectId}`,
+    );
+  }
+  return mapped.shift();
+};
+
+const getObservationByIdInternal = async (
+  id: string,
+  projectId: string,
+  fetchWithInputOutput: boolean = false,
 ) => {
   const query = `
   SELECT
@@ -134,26 +347,10 @@ export const getObservationById = async (
   AND project_id = {projectId: String}
   ORDER BY event_ts desc
   LIMIT 1 by id, project_id`;
-  const records = await queryClickhouse<ObservationRecordReadType>({
+  return await queryClickhouse<ObservationRecordReadType>({
     query,
     params: { id, projectId },
   });
-
-  const mapped = records.map(convertObservation);
-
-  if (mapped.length === 0) {
-    throw new LangfuseNotFoundError(`Observation with id ${id} not found`);
-  }
-
-  if (mapped.length > 1) {
-    logger.error(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-    throw new InternalServerError(
-      `Multiple observations found for id ${id} and project ${projectId}`,
-    );
-  }
-  return mapped.shift() as Observation;
 };
 
 export type ObservationTableQuery = {
@@ -242,21 +439,19 @@ export const getObservationsTable = async (
     opts.projectId,
   );
 
-  return await Promise.all(
-    observationRecords.map(async (o) => {
-      const trace = traces.find((t) => t.id === o.trace_id);
-      return {
-        ...convertObservationToView({ ...o, type: "GENERATION" }),
-        latency: o.latency ? Number(o.latency) / 1000 : null,
-        timeToFirstToken: o.time_to_first_token
-          ? Number(o.time_to_first_token) / 1000
-          : null,
-        traceName: trace?.name ?? null,
-        traceTags: trace?.tags ?? [],
-        userId: trace?.userId ?? null,
-      };
-    }),
-  );
+  return observationRecords.map((o) => {
+    const trace = traces.find((t) => t.id === o.trace_id);
+    return {
+      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      latency: o.latency ? Number(o.latency) / 1000 : null,
+      timeToFirstToken: o.time_to_first_token
+        ? Number(o.time_to_first_token) / 1000
+        : null,
+      traceName: trace?.name ?? null,
+      traceTags: trace?.tags ?? [],
+      userId: trace?.userId ?? null,
+    };
+  });
 };
 
 export const getObservationsTableWithModelData = async (
@@ -324,29 +519,27 @@ export const getObservationsTableWithModelData = async (
     ),
   ]);
 
-  return await Promise.all(
-    observationRecords.map(async (o) => {
-      const trace = traces.find((t) => t.id === o.trace_id);
-      const model = models.find((m) => m.id === o.internal_model_id);
-      return {
-        ...convertObservationToView({ ...o, type: "GENERATION" }),
-        latency: o.latency ? Number(o.latency) / 1000 : null,
-        timeToFirstToken: o.time_to_first_token
-          ? Number(o.time_to_first_token) / 1000
-          : null,
-        traceName: trace?.name ?? null,
-        traceTags: trace?.tags ?? [],
-        userId: trace?.userId ?? null,
-        modelId: model?.id ?? null,
-        inputPrice:
-          model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
-        outputPrice:
-          model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
-        totalPrice:
-          model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
-      };
-    }),
-  );
+  return observationRecords.map((o) => {
+    const trace = traces.find((t) => t.id === o.trace_id);
+    const model = models.find((m) => m.id === o.internal_model_id);
+    return {
+      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      latency: o.latency ? Number(o.latency) / 1000 : null,
+      timeToFirstToken: o.time_to_first_token
+        ? Number(o.time_to_first_token) / 1000
+        : null,
+      traceName: trace?.name ?? null,
+      traceTags: trace?.tags ?? [],
+      userId: trace?.userId ?? null,
+      modelId: model?.id ?? null,
+      inputPrice:
+        model?.Price?.find((m) => m.usageType === "input")?.price ?? null,
+      outputPrice:
+        model?.Price?.find((m) => m.usageType === "output")?.price ?? null,
+      totalPrice:
+        model?.Price?.find((m) => m.usageType === "total")?.price ?? null,
+    };
+  });
 };
 
 const getObservationsTableInternal = async <T>(

packages/shared/src/server/repositories/observations_converters.ts

@@ -3,20 +3,24 @@ import {
   ObservationView,
   ObservationType,
   ObservationLevel,
+  Prisma,
 } from "@prisma/client";
 import Decimal from "decimal.js";
-import { jsonSchema } from "../../utils/zod";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
+import { parseJsonPrioritised } from "../../utils/json";
+import { jsonSchema } from "../../utils/zod";
 
 export const convertObservationToView = (
   record: ObservationRecordReadType,
-): Omit<
-  ObservationView,
-  "inputPrice" | "outputPrice" | "totalPrice" | "modelId"
-> => {
+): Omit<ObservationView, "inputPrice" | "outputPrice" | "totalPrice"> => {
+  // these cost are not used from the view. They are in the select statement but not in the
+  // Prisma file. We will not clean this up but keep it as it is for now.
+  // eslint-disable-next-line no-unused-vars
+  const { inputCost, outputCost, totalCost, internalModelId, ...rest } =
+    convertObservation(record ?? undefined);
   return {
-    ...convertObservation(record ?? undefined),
+    ...rest,
     latency: record.end_time
       ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
@@ -27,12 +31,18 @@ export const convertObservationToView = (
       : null,
     promptName: record.prompt_name ?? null,
     promptVersion: record.prompt_version ?? null,
+    modelId: record.internal_model_id ?? null,
   };
 };
 
 export const convertObservation = (
   record: ObservationRecordReadType,
-): Omit<Observation, "internalModel"> => {
+): Omit<Observation, "internalModel"> & {
+  promptName: string | null;
+  promptVersion: number | null;
+  latency: number | null;
+  timeToFirstToken: number | null;
+} => {
   return {
     id: record.id,
     traceId: record.trace_id ?? null,
@@ -48,8 +58,12 @@ export const convertObservation = (
     level: record.level as ObservationLevel,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: jsonSchema.nullish().parse(record.input) ?? null,
-    output: jsonSchema.nullish().parse(record.output) ?? null,
+    input: (record.input
+      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      : null) as Prisma.JsonValue | null,
     modelParameters: record.model_parameters
       ? JSON.parse(record.model_parameters)
       : null,
@@ -87,5 +101,15 @@ export const convertObservation = (
     model: record.provided_model_name ?? null,
     internalModelId: record.internal_model_id ?? null,
     unit: "TOKENS", // to be removed.
+    promptName: record.prompt_name ?? null,
+    promptVersion: record.prompt_version ?? null,
+    latency: record.end_time
+      ? parseClickhouseUTCDateTimeFormat(record.end_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
+      : null,
+    timeToFirstToken: record.completion_start_time
+      ? parseClickhouseUTCDateTimeFormat(record.start_time).getTime() -
+        parseClickhouseUTCDateTimeFormat(record.completion_start_time).getTime()
+      : null,
   };
 };

packages/shared/src/server/repositories/scores.ts

@@ -24,28 +24,7 @@ import {
 } from "./scores_converters";
 import { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./constants";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-
-export type FetchScoresReturnType = {
-  id: string;
-  timestamp: string;
-  project_id: string;
-  trace_id: string;
-  observation_id: string | null;
-  name: string;
-  value: number;
-  source: string;
-  comment: string | null;
-  author_user_id: string | null;
-  config_id: string | null;
-  data_type: string;
-  string_value: string | null;
-  queue_id: string | null;
-  created_at: string;
-  updated_at: string;
-  event_ts: string;
-  is_deleted: number;
-  projectId: string;
-};
+import { ScoreRecordReadType } from "./definitions";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -74,7 +53,7 @@ export const searchExistingAnnotationScore = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -103,7 +82,7 @@ export const getScoreById = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -118,13 +97,13 @@ export const getScoreById = async (
  * Accepts a score in a Clickhouse-ready format.
  * id, project_id, name, and timestamp must always be provided.
  */
-export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
+export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   if (!["id", "project_id", "name", "timestamp"].every((key) => key in score)) {
     throw new Error("Identifier fields must be provided to upsert Score.");
   }
   await upsertClickhouse({
     table: "scores",
-    records: [score as FetchScoresReturnType],
+    records: [score as ScoreRecordReadType],
     eventBodyMapper: convertToScore,
   });
 };
@@ -148,7 +127,7 @@ export const getScoresForTraces = async (
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId,
@@ -181,7 +160,7 @@ export const getScoresForObservations = async (
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId: projectId,

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,5 +1,5 @@
 import { ScoreSource, ScoreDataType } from "@prisma/client";
-import { FetchScoresReturnType } from "./scores";
+import { ScoreRecordReadType } from "./definitions";
 
 export type ScoreAggregation = {
   id: string;
@@ -11,22 +11,22 @@ export type ScoreAggregation = {
   comment: string | null;
 };
 
-export const convertToScore = (row: FetchScoresReturnType) => {
+export const convertToScore = (row: ScoreRecordReadType) => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),
     projectId: row.project_id,
     traceId: row.trace_id,
-    observationId: row.observation_id,
+    observationId: row.observation_id ?? null,
     name: row.name,
-    value: row.value,
+    value: row.value ?? null,
     source: row.source as ScoreSource,
-    comment: row.comment,
-    authorUserId: row.author_user_id,
-    configId: row.config_id,
+    comment: row.comment ?? null,
+    authorUserId: row.author_user_id ?? null,
+    configId: row.config_id ?? null,
     dataType: row.data_type as ScoreDataType,
-    stringValue: row.string_value,
-    queueId: row.queue_id,
+    stringValue: row.string_value ?? null,
+    queueId: row.queue_id ?? null,
     createdAt: new Date(row.created_at),
     updatedAt: new Date(row.updated_at),
   };

packages/shared/src/server/repositories/traces.ts

@@ -8,14 +8,11 @@ import {
   createFilterFromFilterState,
   getProjectIdDefaultFilter,
 } from "../queries/clickhouse-sql/factory";
-import { ObservationLevel, Trace } from "@prisma/client";
 import { FilterState } from "../../types";
-import { logger } from "../logger";
 import {
   DateTimeFilter,
   FilterList,
   StringFilter,
-  StringOptionsFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
@@ -24,241 +21,51 @@ import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factor
 import { UiColumnMapping } from "../../tableDefinitions";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-import {
-  convertClickhouseToDomain,
-  convertToDomain,
-} from "./traces_converters";
+import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
+import { FetchTracesTableProps } from "../services/traces-ui-table-service";
 
-export type TracesTableReturnType = Pick<
-  TraceRecordReadType,
-  | "project_id"
-  | "id"
-  | "name"
-  | "timestamp"
-  | "bookmarked"
-  | "release"
-  | "version"
-  | "user_id"
-  | "session_id"
-  | "tags"
-  | "public"
-> & {
-  level: ObservationLevel;
-  observation_count: number | null;
-  latency_milliseconds: string | null;
-  usage_details: Record<string, number>;
-  cost_details: Record<string, number>;
-  scores_avg: Array<{ name: string; avg_value: number }>;
-};
-
-export const getTracesTableCount = async (props: {
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  offset?: number;
-}) => {
-  const countRows = await getTracesTableGeneric<{ count: string }>({
-    select: "count(*) as count",
-    ...props,
-  });
-
-  const converted = countRows.map((row) => ({
-    count: Number(row.count),
-  }));
-
-  return converted.length > 0 ? converted[0].count : 0;
-};
-
-export const getTracesTable = async (
+export const checkTraceExists = async (
   projectId: string,
+  traceId: string,
+  timestamp: Date | undefined,
   filter: FilterState,
-  searchQuery?: string,
-  orderBy?: OrderByState,
-  limit?: number,
-  offset?: number,
-) => {
-  const rows = await getTracesTableGeneric<TracesTableReturnType>({
-    select: `
-    t.id, 
-    t.project_id as project_id, 
-    t.timestamp, 
-    t.tags, 
-    t.bookmarked, 
-    t.name, 
-    t.release, 
-    t.version, 
-    t.user_id, 
-    t.session_id,
-    os.latency_milliseconds,
-    os.cost_details as cost_details,
-    os.usage_details as usage_details,
-    os.level as level,
-    os.observation_count as observation_count,
-    s.scores_avg as scores_avg,
-    t.public`,
-    projectId,
-    filter,
-    searchQuery,
-    orderBy,
-    limit,
-    offset,
+): Promise<boolean> => {
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
   });
 
-  return rows.map(convertToDomain);
-};
-
-type FetchTracesTableProps = {
-  select: string;
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  offset?: number;
-};
-
-const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, offset, searchQuery } =
-    props;
-
-  const { tracesFilter, scoresFilter, observationsFilter } =
-    getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
-
   tracesFilter.push(
     ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+    new StringFilter({
+      clickhouseTable: "t",
+      field: "id",
+      operator: "=",
+      value: traceId,
+    }),
   );
 
-  const traceIdFilter = tracesFilter.find(
-    (f) => f.clickhouseTable === "traces" && f.field === "id",
-  ) as StringFilter | StringOptionsFilter | undefined;
-
-  traceIdFilter
-    ? scoresFilter.push(
-        new StringOptionsFilter({
-          clickhouseTable: "scores",
-          field: "trace_id",
-          operator: "any of",
-          values:
-            traceIdFilter instanceof StringFilter
-              ? [traceIdFilter.value]
-              : traceIdFilter.values,
-        }),
-      )
-    : null;
-  traceIdFilter
-    ? observationsFilter.push(
-        new StringOptionsFilter({
-          clickhouseTable: "observations",
-          field: "trace_id",
-          operator: "any of",
-          values:
-            traceIdFilter instanceof StringFilter
-              ? [traceIdFilter.value]
-              : traceIdFilter.values,
-        }),
-      )
-    : null;
-
-  // for query optimisation, we have to add the timeseries filter to observations + scores as well
-  // stats show, that 98% of all observations have their start_time larger than trace.timestamp - 5 min
-  const timeStampFilter = tracesFilter.find(
-    (f) =>
-      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  timeStampFilter
-    ? scoresFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "scores",
-          field: "timestamp",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
-
-  timeStampFilter
-    ? observationsFilter.push(
-        new DateTimeFilter({
-          clickhouseTable: "observations",
-          field: "start_time",
-          operator: ">=",
-          value: timeStampFilter.value,
-        }),
-      )
-    : null;
-
   const tracesFilterRes = tracesFilter.apply();
-  const scoresFilterRes = scoresFilter.apply();
-  const observationFilterRes = observationsFilter.apply();
-
-  const search = clickhouseSearchCondition(searchQuery);
 
   const query = `
-    WITH observations_stats AS (
-      SELECT
-        COUNT(*) AS observation_count,
-          sumMap(usage_details) as usage_details,
-          SUM(total_cost) AS total_cost,
-          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
-          multiIf(
-            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
-            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
-            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
-            'DEBUG'
-          ) AS level,
-          sumMap(cost_details) as cost_details,
-          trace_id,
-          project_id
-      FROM observations FINAL
-      WHERE ${observationFilterRes.query}
-      GROUP BY trace_id, project_id
-    ),
-    scores_avg AS (
-      SELECT
-        project_id,
-        trace_id,
-        groupArray(tuple(name, avg_value)) AS "scores_avg"
-      FROM (
-        SELECT project_id,
-                trace_id,
-                name,
-                avg(value) avg_value
-        FROM scores final
-        WHERE ${scoresFilterRes.query}
-        GROUP BY project_id,
-                  trace_id,
-                  name
-      ) tmp
-      GROUP BY project_id, trace_id
-    )
-    SELECT ${select}
-    FROM traces t final
-    LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
-    LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
+    SELECT id, project_id
+    FROM traces t FINAL
     WHERE ${tracesFilterRes.query}
-    ${search.query}
-    ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
-    ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
   `;
 
-  const res = await queryClickhouse<T>({
-    query: query,
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
     params: {
-      limit: limit,
-      offset: offset,
       ...tracesFilterRes.params,
-      ...observationFilterRes.params,
-      ...scoresFilterRes.params,
-      ...search.params,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
     },
   });
 
-  return res;
+  return rows.length > 0;
 };
 
 /**
@@ -535,7 +342,7 @@ export const getSessionsTableCount = async (props: {
   filter: FilterState;
   orderBy?: OrderByState;
   limit?: number;
-  offset?: number;
+  page?: number;
 }) => {
   const rows = await getSessionsTableGeneric<{ count: string }>({
     select: `
@@ -545,7 +352,7 @@ export const getSessionsTableCount = async (props: {
     filter: props.filter,
     orderBy: props.orderBy,
     limit: props.limit,
-    offset: props.offset,
+    page: props.page,
   });
 
   return rows.length > 0 ? Number(rows[0].count) : 0;
@@ -556,7 +363,7 @@ export const getSessionsTable = async (props: {
   filter: FilterState;
   orderBy?: OrderByState;
   limit?: number;
-  offset?: number;
+  page?: number;
 }) => {
   const rows = await getSessionsTableGeneric<SessionDataReturnType>({
     select: `
@@ -582,14 +389,14 @@ export const getSessionsTable = async (props: {
     filter: props.filter,
     orderBy: props.orderBy,
     limit: props.limit,
-    offset: props.offset,
+    page: props.page,
   });
 
   return rows;
 };
 
 const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
-  const { select, projectId, filter, orderBy, limit, offset } = props;
+  const { select, projectId, filter, orderBy, limit, page } = props;
 
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
@@ -664,7 +471,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
     FROM session_data s
     WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
     ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && offset !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
     `;
 
   const obsStartTimeValue = traceTimestampFilter
@@ -676,7 +483,7 @@ const getSessionsTableGeneric = async <T>(props: FetchTracesTableProps) => {
     params: {
       projectId,
       limit: limit,
-      offset: offset,
+      offset: limit && page ? limit * page : 0,
       ...tracesFilterRes.params,
       ...observationsStatsRes.params,
       ...scoresAvgFilterRes.params,

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,10 +1,10 @@
 import { ObservationLevel, Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
-import { TracesTableReturnType } from "./traces";
 import Decimal from "decimal.js";
 import { ScoreAggregate } from "../../features/scores";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { TracesTableReturnType } from "../services/traces-ui-table-service";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -81,7 +81,7 @@ export const convertToDomain = (row: TracesTableReturnType) => {
     version: row.version ?? null,
     userId: row.user_id ?? null,
     sessionId: row.session_id ?? null,
-    latencyMilliseconds: Number(row.latency_milliseconds),
+    latency: Number(row.latency),
     usageDetails: row.usage_details,
     costDetails: row.cost_details,
     level: row.level,

packages/shared/src/server/services/S3StorageService.ts

@@ -1,172 +0,0 @@
-import type { Readable } from "stream";
-import {
-  GetObjectCommand,
-  ListObjectsV2Command,
-  PutObjectCommand,
-  S3Client,
-} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { logger } from "../logger";
-
-type UploadFile = {
-  fileName: string;
-  fileType: string;
-  data: Readable | string;
-  expiresInSeconds: number;
-};
-
-export class S3StorageService {
-  private client: S3Client;
-  private bucketName: string;
-
-  constructor(params: {
-    accessKeyId: string | undefined;
-    secretAccessKey: string | undefined;
-    bucketName: string;
-    endpoint: string | undefined;
-    region: string | undefined;
-    forcePathStyle: boolean;
-  }) {
-    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
-    const { accessKeyId, secretAccessKey } = params;
-    const credentials =
-      accessKeyId !== undefined && secretAccessKey !== undefined
-        ? {
-            accessKeyId,
-            secretAccessKey,
-          }
-        : undefined;
-
-    this.client = new S3Client({
-      credentials,
-      endpoint: params.endpoint,
-      region: params.region,
-      forcePathStyle: params.forcePathStyle,
-    });
-    this.bucketName = params.bucketName;
-  }
-
-  public async uploadFile({
-    fileName,
-    fileType,
-    data,
-    expiresInSeconds,
-  }: UploadFile): Promise<{ signedUrl: string }> {
-    try {
-      await new Upload({
-        client: this.client,
-        params: {
-          Bucket: this.bucketName,
-          Key: fileName,
-          Body: data,
-          ContentType: fileType,
-        },
-      }).done();
-
-      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
-
-      return { signedUrl };
-    } catch (err) {
-      logger.error(`Failed to upload file to ${fileName}`, err);
-      throw new Error("Failed to upload to S3 or generate signed URL");
-    }
-  }
-
-  public async uploadJson(path: string, body: Record<string, unknown>[]) {
-    const putCommand = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-      Body: JSON.stringify(body),
-      ContentType: "application/json",
-    });
-
-    try {
-      await this.client.send(putCommand);
-    } catch (err) {
-      logger.error(`Failed to upload JSON to S3 ${path}`, err);
-      throw Error("Failed to upload JSON to S3");
-    }
-  }
-
-  public async download(path: string): Promise<string> {
-    const getCommand = new GetObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-    });
-
-    try {
-      const response = await this.client.send(getCommand);
-      return (await response.Body?.transformToString()) ?? "";
-    } catch (err) {
-      logger.error(`Failed to download file from S3 ${path}`, err);
-      throw Error("Failed to download file from S3");
-    }
-  }
-
-  public async listFiles(prefix: string): Promise<string[]> {
-    const listCommand = new ListObjectsV2Command({
-      Bucket: this.bucketName,
-      Prefix: prefix,
-    });
-
-    try {
-      const response = await this.client.send(listCommand);
-      return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
-      );
-    } catch (err) {
-      logger.error(`Failed to list files from S3 ${prefix}`, err);
-      throw Error("Failed to list files from S3");
-    }
-  }
-
-  public async getSignedUrl(
-    fileName: string,
-    ttlSeconds: number,
-    asAttachment: boolean = true,
-  ): Promise<string> {
-    try {
-      return await getSignedUrl(
-        this.client,
-        new GetObjectCommand({
-          Bucket: this.bucketName,
-          Key: fileName,
-          ResponseContentDisposition: asAttachment
-            ? `attachment; filename="${fileName}"`
-            : undefined,
-        }),
-        { expiresIn: ttlSeconds },
-      );
-    } catch (err) {
-      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
-      throw Error("Failed to generate signed URL");
-    }
-  }
-
-  public async getSignedUploadUrl(params: {
-    path: string;
-    ttlSeconds: number;
-    sha256Hash: string;
-    contentType: string;
-    contentLength: number;
-  }): Promise<string> {
-    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
-
-    return await getSignedUrl(
-      this.client,
-      new PutObjectCommand({
-        Bucket: this.bucketName,
-        Key: path,
-        ContentType: contentType,
-        ChecksumSHA256: sha256Hash,
-        ContentLength: contentLength,
-      }),
-      {
-        expiresIn: ttlSeconds,
-        signableHeaders: new Set(["content-type", "content-length"]),
-        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
-      },
-    );
-  }
-}

packages/shared/src/server/services/StorageService.ts

@@ -0,0 +1,432 @@
+import { Readable } from "stream";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from "@azure/storage-blob";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+type UploadFile = {
+  fileName: string;
+  fileType: string;
+  data: Readable | string;
+  expiresInSeconds: number;
+};
+
+export interface StorageService {
+  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>;
+
+  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>;
+
+  download(path: string): Promise<string>;
+
+  listFiles(prefix: string): Promise<string[]>;
+
+  getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string>;
+
+  getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string>;
+}
+
+export class StorageServiceFactory {
+  public static getInstance(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }): StorageService {
+    if (env.LANGFUSE_USE_AZURE_BLOB === "true") {
+      return new AzureBlobStorageService(params);
+    }
+    return new S3StorageService(params);
+  }
+}
+
+class AzureBlobStorageService implements StorageService {
+  private client: ContainerClient;
+  private container: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    const { accessKeyId, secretAccessKey, endpoint } = params;
+    if (!accessKeyId || !secretAccessKey || !endpoint) {
+      throw new Error(
+        `Endpoint, account and account key must be configured to use Azure Blob Storage`,
+      );
+    }
+
+    const sharedKeyCredential = new StorageSharedKeyCredential(
+      accessKeyId,
+      secretAccessKey,
+    );
+    const blobServiceClient = new BlobServiceClient(
+      endpoint,
+      sharedKeyCredential,
+    );
+    this.container = params.bucketName;
+    this.client = blobServiceClient.getContainerClient(this.container);
+  }
+
+  private async createContainerIfNotExists(): Promise<void> {
+    try {
+      await this.client.createIfNotExists();
+    } catch (err) {
+      logger.error(
+        `Failed to create Azure Blob Storage container ${this.container}`,
+        err,
+      );
+      throw Error("Failed to create Azure Blob Storage container ");
+    }
+  }
+
+  public async uploadFile(params: UploadFile): Promise<{ signedUrl: string }> {
+    const { fileName, data, expiresInSeconds } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+
+      if (typeof data === "string") {
+        await blockBlobClient.upload(data, data.length);
+      } else if (data instanceof Readable) {
+        let offset = 0;
+        const blockIds = [];
+        for await (const chunk of data) {
+          const blockId = Buffer.from(`block-${offset}`).toString("base64");
+          const bufferChunk = Buffer.isBuffer(chunk)
+            ? chunk
+            : Buffer.from(chunk);
+
+          await blockBlobClient.stageBlock(
+            blockId,
+            bufferChunk,
+            bufferChunk.length,
+          );
+          blockIds.push(blockId);
+
+          offset += bufferChunk.length;
+        }
+
+        await blockBlobClient.commitBlockList(blockIds);
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+
+      return {
+        signedUrl: await this.getSignedUrl(fileName, expiresInSeconds, false),
+      };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to upload file to Azure Blob Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    await this.createContainerIfNotExists();
+
+    const blockBlobClient = this.client.getBlockBlobClient(path);
+    const content = JSON.stringify(body);
+    try {
+      await blockBlobClient.upload(content, content.length);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to Azure Blob Storage ${path}`, err);
+      throw Error("Failed to upload JSON to Azure Blob Storage");
+    }
+  }
+
+  private async streamToString(
+    readableStream: NodeJS.ReadableStream,
+  ): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const chunks: string[] = [];
+      readableStream.on("data", (data) => {
+        chunks.push(data.toString());
+      });
+      readableStream.on("end", () => {
+        resolve(chunks.join(""));
+      });
+      readableStream.on("error", reject);
+    });
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blobClient = this.client.getBlobClient(path);
+      const downloadResponse = await blobClient.download();
+      if (!downloadResponse.readableStreamBody) {
+        throw Error("No stream body available");
+      }
+      return this.streamToString(downloadResponse.readableStreamBody);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const result = await this.client.listBlobsFlat({ prefix });
+      const files = [];
+      for await (const blob of result) {
+        if (blob.name.startsWith(prefix)) {
+          files.push(blob.name);
+        }
+      }
+      return files;
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Azure Blob Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("r"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentDisposition: asAttachment
+          ? `attachment; filename="${fileName}"`
+          : undefined,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL for Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate presigned URL for Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(path);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("w"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentType: contentType,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL for Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate presigned upload URL for Azure Blob Storage",
+      );
+    }
+  }
+}
+
+class S3StorageService implements StorageService {
+  private client: S3Client;
+  private bucketName: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
+    const { accessKeyId, secretAccessKey } = params;
+    const credentials =
+      accessKeyId !== undefined && secretAccessKey !== undefined
+        ? {
+            accessKeyId,
+            secretAccessKey,
+          }
+        : undefined;
+
+    this.client = new S3Client({
+      credentials,
+      endpoint: params.endpoint,
+      region: params.region,
+      forcePathStyle: params.forcePathStyle,
+    });
+    this.bucketName = params.bucketName;
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      await new Upload({
+        client: this.client,
+        params: {
+          Bucket: this.bucketName,
+          Key: fileName,
+          Body: data,
+          ContentType: fileType,
+        },
+      }).done();
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(`Failed to upload file to ${fileName}`, err);
+      throw new Error("Failed to upload to S3 or generate signed URL");
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    const putCommand = new PutObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+      Body: JSON.stringify(body),
+      ContentType: "application/json",
+    });
+
+    try {
+      await this.client.send(putCommand);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to S3 ${path}`, err);
+      throw Error("Failed to upload JSON to S3");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+
+    try {
+      const response = await this.client.send(getCommand);
+      return (await response.Body?.transformToString()) ?? "";
+    } catch (err) {
+      logger.error(`Failed to download file from S3 ${path}`, err);
+      throw Error("Failed to download file from S3");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: prefix,
+    });
+
+    try {
+      const response = await this.client.send(listCommand);
+      return (
+        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
+      );
+    } catch (err) {
+      logger.error(`Failed to list files from S3 ${prefix}`, err);
+      throw Error("Failed to list files from S3");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      return await getSignedUrl(
+        this.client,
+        new GetObjectCommand({
+          Bucket: this.bucketName,
+          Key: fileName,
+          ResponseContentDisposition: asAttachment
+            ? `attachment; filename="${fileName}"`
+            : undefined,
+        }),
+        { expiresIn: ttlSeconds },
+      );
+    } catch (err) {
+      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
+      throw Error("Failed to generate signed URL");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
+
+    return await getSignedUrl(
+      this.client,
+      new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: path,
+        ContentType: contentType,
+        ChecksumSHA256: sha256Hash,
+        ContentLength: contentLength,
+      }),
+      {
+        expiresIn: ttlSeconds,
+        signableHeaders: new Set(["content-type", "content-length"]),
+        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
+      },
+    );
+  }
+}

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -0,0 +1,248 @@
+import { ObservationLevel } from "@prisma/client";
+import { OrderByState } from "../../interfaces/orderBy";
+import { tracesTableUiColumnDefinitions } from "../../tableDefinitions";
+import { FilterState } from "../../types";
+import {
+  StringFilter,
+  StringOptionsFilter,
+  DateTimeFilter,
+} from "../queries/clickhouse-sql/clickhouse-filter";
+import {
+  getProjectIdDefaultFilter,
+  createFilterFromFilterState,
+} from "../queries/clickhouse-sql/factory";
+import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
+import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
+import { convertToDomain } from "../repositories";
+import { queryClickhouse } from "../repositories/clickhouse";
+import { TraceRecordReadType } from "../repositories/definitions";
+
+export type TracesTableReturnType = Pick<
+  TraceRecordReadType,
+  | "project_id"
+  | "id"
+  | "name"
+  | "timestamp"
+  | "bookmarked"
+  | "release"
+  | "version"
+  | "user_id"
+  | "session_id"
+  | "tags"
+  | "public"
+> & {
+  level: ObservationLevel;
+  observation_count: number | null;
+  latency: string | null;
+  usage_details: Record<string, number>;
+  cost_details: Record<string, number>;
+  scores_avg: Array<{ name: string; avg_value: number }>;
+};
+
+export type FetchTracesTableProps = {
+  select: string;
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+export const getTracesTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const countRows = await getTracesTableGeneric<{ count: string }>({
+    select: "count(*) as count",
+    ...props,
+  });
+
+  const converted = countRows.map((row) => ({
+    count: Number(row.count),
+  }));
+
+  return converted.length > 0 ? converted[0].count : 0;
+};
+
+export const getTracesTable = async (
+  projectId: string,
+  filter: FilterState,
+  searchQuery?: string,
+  orderBy?: OrderByState,
+  limit?: number,
+  page?: number,
+) => {
+  const rows = await getTracesTableGeneric<TracesTableReturnType>({
+    select: `
+    t.id, 
+    t.project_id as project_id, 
+    t.timestamp, 
+    t.tags, 
+    t.bookmarked, 
+    t.name, 
+    t.release, 
+    t.version, 
+    t.user_id, 
+    t.session_id,
+    os.latency_milliseconds / 1000 as latency,
+    os.cost_details as cost_details,
+    os.usage_details as usage_details,
+    os.level as level,
+    os.observation_count as observation_count,
+    s.scores_avg as scores_avg,
+    t.public`,
+    projectId,
+    filter,
+    searchQuery,
+    orderBy,
+    limit,
+    page,
+  });
+
+  return rows.map(convertToDomain);
+};
+
+const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
+  const { select, projectId, filter, orderBy, limit, page, searchQuery } =
+    props;
+
+  const { tracesFilter, scoresFilter, observationsFilter } =
+    getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+
+  const traceIdFilter = tracesFilter.find(
+    (f) => f.clickhouseTable === "traces" && f.field === "id",
+  ) as StringFilter | StringOptionsFilter | undefined;
+
+  traceIdFilter
+    ? scoresFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "scores",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+  traceIdFilter
+    ? observationsFilter.push(
+        new StringOptionsFilter({
+          clickhouseTable: "observations",
+          field: "trace_id",
+          operator: "any of",
+          values:
+            traceIdFilter instanceof StringFilter
+              ? [traceIdFilter.value]
+              : traceIdFilter.values,
+        }),
+      )
+    : null;
+
+  // for query optimisation, we have to add the timeseries filter to observations + scores as well
+  // stats show, that 98% of all observations have their start_time larger than trace.timestamp - 5 min
+  const timeStampFilter = tracesFilter.find(
+    (f) =>
+      f.field === "timestamp" && (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  timeStampFilter
+    ? scoresFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "scores",
+          field: "timestamp",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
+  timeStampFilter
+    ? observationsFilter.push(
+        new DateTimeFilter({
+          clickhouseTable: "observations",
+          field: "start_time",
+          operator: ">=",
+          value: timeStampFilter.value,
+        }),
+      )
+    : null;
+
+  const tracesFilterRes = tracesFilter.apply();
+  const scoresFilterRes = scoresFilter.apply();
+  const observationFilterRes = observationsFilter.apply();
+
+  const search = clickhouseSearchCondition(searchQuery);
+
+  const query = `
+    WITH observations_stats AS (
+      SELECT
+        COUNT(*) AS observation_count,
+          sumMap(usage_details) as usage_details,
+          SUM(total_cost) AS total_cost,
+          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          multiIf(
+            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
+            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
+            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
+            'DEBUG'
+          ) AS level,
+          sumMap(cost_details) as cost_details,
+          trace_id,
+          project_id
+      FROM observations FINAL
+      WHERE ${observationFilterRes.query}
+      GROUP BY trace_id, project_id
+    ),
+    scores_avg AS (
+      SELECT
+        project_id,
+        trace_id,
+        groupArray(tuple(name, avg_value)) AS "scores_avg"
+      FROM (
+        SELECT project_id,
+                trace_id,
+                name,
+                avg(value) avg_value
+        FROM scores final
+        WHERE ${scoresFilterRes.query}
+        GROUP BY project_id,
+                  trace_id,
+                  name
+      ) tmp
+      GROUP BY project_id, trace_id
+    )
+    SELECT ${select}
+    FROM traces t final
+    LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id
+    LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id
+    WHERE ${tracesFilterRes.query}
+    ${search.query}
+    ${orderByToClickhouseSql(orderBy ?? null, tracesTableUiColumnDefinitions)}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+  `;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      limit: limit,
+      offset: limit && page ? limit * page : 0,
+      ...tracesFilterRes.params,
+      ...observationFilterRes.params,
+      ...scoresFilterRes.params,
+      ...search.params,
+    },
+  });
+
+  return res;
+};

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -69,6 +69,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Latency (s)",
@@ -76,6 +78,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(end_time), NULL, date_diff('seconds', start_time, end_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Tokens per second",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -106,27 +106,26 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableName: "Latency (s)",
     uiTableId: "latency",
     clickhouseTableName: "traces",
-    clickhouseSelect: "latency_milliseconds",
+    clickhouseSelect: "latency_milliseconds / 1000",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Input Cost ($)",
     uiTableId: "inputCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'input'), cost_details), usage_details['input'], NULL)",
+    clickhouseSelect: "cost_details['input']",
   },
   {
     uiTableName: "Output Cost ($)",
     uiTableId: "outputCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'output'), cost_details), usage_details['output'], NULL)",
+    clickhouseSelect: "cost_details['output']",
   },
   {
     uiTableName: "Total Cost ($)",
     uiTableId: "totalCost",
     clickhouseTableName: "traces",
-    clickhouseSelect:
-      "if(mapExists((k, v) -> (k = 'total'), cost_details), usage_details['total'], NULL)",
+    clickhouseSelect: "cost_details['total']",
   },
 ];

packages/shared/src/tableDefinitions/tracesTable.ts

@@ -138,16 +138,16 @@ export const tracesTableCols: ColumnDefinition[] = [
   },
 ];
 
-// Used only for dataset evaluator form, not on any table
-export const datasetOnlyCols: ColumnDefinition[] = [
-  {
-    name: "Dataset",
-    id: "datasetId",
-    type: "stringOptions",
-    internal: 'di."dataset_id"',
-    options: [], // to be filled in at runtime
-  },
-];
+export const datasetCol: ColumnDefinition = {
+  name: "Dataset",
+  id: "datasetId",
+  type: "stringOptions",
+  internal: 'di."dataset_id"',
+  options: [], // to be filled in at runtime
+};
+
+// Used only for dataset evaluator, not on dataset table
+export const datasetOnlyCols: ColumnDefinition[] = [datasetCol];
 
 export const evalTraceTableCols: ColumnDefinition[] = tracesOnlyCols;
 export const evalDatasetFormFilterCols: ColumnDefinition[] = datasetOnlyCols;

packages/shared/src/tableDefinitions/types.ts

@@ -3,6 +3,7 @@ export type UiColumnMapping = {
   uiTableId: string;
   clickhouseTableName: string;
   clickhouseSelect: string;
+  clickhouseTypeOverwrite?: string;
   queryPrefix?: string;
 };
 

packages/shared/src/utils/stringChecks.ts

@@ -0,0 +1,23 @@
+export function getIsCharOrUnderscore(value: string): boolean {
+  const charOrUnderscore = /^[A-Za-z_]+$/;
+
+  return charOrUnderscore.test(value);
+}
+
+export function extractVariables(mustacheString: string): string[] {
+  const mustacheRegex = /\{\{(.*?)\}\}/g;
+  const uniqueVariables = new Set<string>();
+
+  for (const match of mustacheString.matchAll(mustacheRegex)) {
+    uniqueVariables.add(match[1]);
+  }
+
+  for (const variable of uniqueVariables) {
+    // if validated fails, remove from set
+    if (!getIsCharOrUnderscore(variable)) {
+      uniqueVariables.delete(variable);
+    }
+  }
+
+  return Array.from(uniqueVariables);
+}

patches/next-auth@4.24.11.patch

@@ -0,0 +1,26 @@
+diff --git a/core/lib/oauth/client.js b/core/lib/oauth/client.js
+index 52c51eb6ff422dc0899ccec31baf3fa39e42eeae..bc50c35bb617d0e86b68ca42f64d44b475ba4abb 100644
+--- a/core/lib/oauth/client.js
++++ b/core/lib/oauth/client.js
+@@ -1,5 +1,7 @@
+ "use strict";
+ 
++var HttpsProxyAgent = require('https-proxy-agent').HttpsProxyAgent;
++
+ Object.defineProperty(exports, "__esModule", {
+   value: true
+ });
+@@ -7,7 +9,12 @@ exports.openidClient = openidClient;
+ var _openidClient = require("openid-client");
+ async function openidClient(options) {
+   const provider = options.provider;
+-  if (provider.httpOptions) _openidClient.custom.setHttpOptionsDefaults(provider.httpOptions);
++  let httpOptions = {};
++  if (provider.httpOptions) httpOptions = { ...provider.httpOptions };
++  if (process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY) {
++    httpOptions.agent = new HttpsProxyAgent(process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY);
++  }
++  _openidClient.custom.setHttpOptionsDefaults(httpOptions);
+   let issuer;
+   if (provider.wellKnown) {
+     issuer = await _openidClient.Issuer.discover(provider.wellKnown);

web/Dockerfile

@@ -109,7 +109,7 @@ RUN adduser --system --uid 1001 nextjs
 RUN npm install -g --no-package-lock --no-save prisma@5.22.0
 
 RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
-    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.0/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
+    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
     mv migrate /usr/bin/migrate
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .

web/entrypoint.sh

@@ -12,11 +12,16 @@ if [ -z "$DATABASE_URL" ]; then
         echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
         exit 1
     fi
+    if [ -n "$DATABASE_ARGS" ]; then
+        # Append ARGS to DATABASE_URL
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
+        export DATABASE_URL
+    fi
 fi
 
 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
-    export DIRECT_URL=$DATABASE_URL
+    export DIRECT_URL="${DATABASE_URL}"
 fi
 
 # Always execute the postgres migration, except when disabled.

web/next.config.mjs

@@ -28,6 +28,9 @@ const cspHeader = `
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
 `;
 
+// Match rules for Hugging Face
+const huggingFaceHosts = ["huggingface.co", ".*\\.hf\\.space$"];
+
 const reportToHeader = {
   key: "Report-To",
   value: JSON.stringify({
@@ -78,10 +81,6 @@ const nextConfig = {
       {
         source: "/:path*",
         headers: [
-          {
-            key: "x-frame-options",
-            value: "SAMEORIGIN",
-          },
           {
             key: "X-Content-Type-Options",
             value: "nosniff",
@@ -97,6 +96,21 @@ const nextConfig = {
           ...(env.SENTRY_CSP_REPORT_URI ? [reportToHeader] : []),
         ],
       },
+      {
+        source: "/:path*",
+        headers: [
+          {
+            key: "x-frame-options",
+            value: "SAMEORIGIN",
+          },
+        ],
+        // Disable x-frame-options on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
+      },
+      // CSP header
       {
         source: "/:path((?!api).*)*",
         headers: [
@@ -105,26 +119,31 @@ const nextConfig = {
             value: cspHeader.replace(/\n/g, ""),
           },
         ],
+        // Disable CSP on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
       },
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
       // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
       {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.91.0",
+  "version": "2.95.1",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -35,9 +35,9 @@
     "@headlessui/tailwindcss": "0.2.1",
     "@heroicons/react": "^2.1.5",
     "@hookform/resolvers": "^3.3.4",
-    "@langchain/anthropic": "^0.3.1",
-    "@langchain/core": "^0.3.9",
-    "@langchain/openai": "^0.3.0",
+    "@langchain/anthropic": "^0.3.8",
+    "@langchain/core": "^0.3.18",
+    "@langchain/openai": "^0.3.14",
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
     "@marsidev/react-turnstile": "^0.5.4",
@@ -83,7 +83,7 @@
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.33.1",
+    "@sentry/nextjs": "^8.52.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -107,17 +107,18 @@
     "date-fns": "^3.3.1",
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.5",
+    "dompurify": "^3.2.4",
     "graphql": "^16.9.0",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "ip-address": "^9.0.5",
     "js-tiktoken": "^1.0.15",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.2",
+    "langchain": "^0.3.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.447.0",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.176.0",
@@ -192,7 +193,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.0"
+      "jsonpath-plus": "10.0.7"
     }
   }
 }

web/src/__e2e__/api.servertest.ts

@@ -1,6 +1,10 @@
 import { v4 } from "uuid";
 import { JobExecutionStatus, Prisma, prisma } from "@langfuse/shared/src/db";
-import { OrgEnrichedApiKey, redis } from "@langfuse/shared/src/server";
+import {
+  clickhouseClient,
+  OrgEnrichedApiKey,
+  redis,
+} from "@langfuse/shared/src/server";
 import waitForExpect from "wait-for-expect";
 
 const generateAuth = (username: string, password: string) => {
@@ -86,70 +90,85 @@ describe("Ingestion Pipeline", () => {
       body: JSON.stringify(event),
     });
 
-    await waitForExpect(async () => {
-      // we need a second call to the public API with the API key, so that it is stored in redis
-      // first call (ingestion above) generates the new, fast API hash
-      // second call (below) stores the API key in redis
-      const traceUrl = `http://localhost:3000/api/public/traces/${traceId}`;
+    await waitForExpect(
+      async () => {
+        // we need a second call to the public API with the API key, so that it is stored in redis
+        // first call (ingestion above) generates the new, fast API hash
+        // second call (below) stores the API key in redis
+        const traceUrl = `http://localhost:3000/api/public/traces/${traceId}`;
 
-      const traceResponse = await fetch(traceUrl, {
-        headers: {
-          Authorization: userApiKeyAuth,
-        },
-      });
+        const traceResponse = await fetch(traceUrl, {
+          headers: {
+            Authorization: userApiKeyAuth,
+          },
+        });
 
-      expect(traceResponse.status).toBe(200);
-      expect(traceResponse.body).not.toBeNull();
-      expect((await traceResponse.json()).id).toBe(traceId);
+        const traces = await clickhouseClient().query({
+          query: "SELECT * FROM traces",
+          format: "JSONEachRow",
+        });
+        console.log("traces", await traces.text());
+        expect(traceResponse.status).toBe(200);
+        expect(traceResponse.body).not.toBeNull();
+        expect((await traceResponse.json()).id).toBe(traceId);
 
-      const trace = await prisma.trace.findUnique({
-        where: {
-          id: traceId,
-        },
-      });
-      expect(trace).not.toBeNull();
-      expect(trace?.name).toBe("test trace");
+        const trace = await prisma.trace.findUnique({
+          where: {
+            id: traceId,
+          },
+        });
+        expect(trace).not.toBeNull();
+        expect(trace?.name).toBe("test trace");
 
-      const observation = await prisma.observation.findUnique({
-        where: {
-          id: spanId,
-          traceId: traceId,
-        },
-      });
-      expect(observation).not.toBeNull();
-      expect(observation?.name).toBe("test span");
+        const observation = await prisma.observation.findUnique({
+          where: {
+            id: spanId,
+            traceId: traceId,
+          },
+        });
+        expect(observation).not.toBeNull();
+        expect(observation?.name).toBe("test span");
 
-      expect(redis).not.toBeNull();
+        expect(redis).not.toBeNull();
 
-      const redisKeys = await redis?.keys(`api-key:*`);
-      expect(redisKeys?.length).toBe(1);
-      const redisValue = await redis?.get(redisKeys![0]);
+        const redisKeys = await redis?.keys(`api-key:*`);
+        expect(redisKeys?.length).toBe(1);
+        const redisValue = await redis?.get(redisKeys![0]);
 
-      const llmApiKey = OrgEnrichedApiKey.parse(JSON.parse(redisValue!));
-      expect(llmApiKey.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
-    });
+        const llmApiKey = OrgEnrichedApiKey.parse(JSON.parse(redisValue!));
+        expect(llmApiKey.projectId).toBe(
+          "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+        );
+      },
+      40000,
+      1000,
+    );
 
     // check for eval
-    await waitForExpect(async () => {
-      const evalExecution = await prisma.jobExecution.findFirst({
-        where: {
-          jobInputTraceId: traceId,
-          projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        },
-      });
+    await waitForExpect(
+      async () => {
+        const evalExecution = await prisma.jobExecution.findFirst({
+          where: {
+            jobInputTraceId: traceId,
+            projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+          },
+        });
 
-      expect(evalExecution).not.toBeNull();
+        expect(evalExecution).not.toBeNull();
 
-      if (!evalExecution) {
-        return;
-      }
+        if (!evalExecution) {
+          return;
+        }
 
-      // failure due to missing openai key in the pipeline. Expected
-      expect(evalExecution.status).toBe(JobExecutionStatus.ERROR);
-    }, 20000);
+        // failure due to missing openai key in the pipeline. Expected
+        expect(evalExecution.status).toBe(JobExecutionStatus.ERROR);
+      },
+      50000,
+      10000,
+    );
 
     expect(response.status).toBe(207);
-  }, 25000);
+  }, 60000);
 
   it("rate limit ingestion", async () => {
     // update the org in the database and set the rate limit to 1 for ingestion

web/src/__tests__/async/dataset-service.servertest.ts

@@ -0,0 +1,602 @@
+import {
+  createObservations,
+  createScores,
+  createTraces,
+} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { v4 } from "uuid";
+import { prisma } from "@langfuse/shared/src/db";
+import {
+  createObservation,
+  createScore,
+  createTrace,
+} from "@/src/__tests__/fixtures/tracing-factory";
+import {
+  createDatasetRunsTable,
+  fetchDatasetItems,
+  getRunItemsByRunIdOrItemId,
+} from "@/src/features/datasets/server/service";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("Fetch datasets for UI presentation", () => {
+  it("should fetch dataset runs for UI", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    const datasetRun2Id = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRun2Id,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const datasetRunItemId2 = v4();
+    const datasetRunItemId3 = v4();
+    const datasetRunItemId4 = v4();
+    const observationId = v4();
+    const traceId = v4();
+    const traceId2 = v4();
+    const traceId3 = v4();
+    const traceId4 = v4();
+    const scoreId = v4();
+    const scoreId2 = v4();
+    const scoreId3 = v4();
+    const scoreName = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        observationId: observationId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRunId,
+        traceId: traceId2,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId3,
+        datasetRunId: datasetRunId,
+        traceId: traceId3,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    // linked to the second run
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId4,
+        datasetRunId: datasetRun2Id,
+        observationId: null,
+        traceId: traceId4,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const observation = createObservation({
+      id: observationId,
+      trace_id: traceId,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
+      end_time: new Date().getTime(),
+    });
+    const observation2 = createObservation({
+      trace_id: traceId2,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
+      end_time: new Date().getTime(),
+    });
+    const observation3 = createObservation({
+      trace_id: traceId2,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 30,
+      end_time: new Date().getTime(),
+      total_cost: 200,
+    });
+    const observation4 = createObservation({
+      trace_id: traceId3,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 300,
+      end_time: new Date().getTime(),
+      total_cost: 50,
+    });
+    const observation5 = createObservation({
+      trace_id: traceId4,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000,
+      end_time: new Date().getTime(),
+    });
+    await createObservations([
+      observation,
+      observation2,
+      observation3,
+      observation4,
+      observation5,
+    ]);
+    const score = createScore({
+      id: scoreId,
+      observation_id: observationId,
+      trace_id: traceId,
+      project_id: projectId,
+      name: scoreName,
+    });
+    const score2 = createScore({
+      id: scoreId2,
+      observation_id: null,
+      trace_id: traceId,
+      project_id: projectId,
+      name: scoreName,
+      value: 1,
+      comment: "some other comment",
+    });
+    const observationId2 = v4(); // this one is not related to a run
+    const anotherScoreName = v4();
+
+    const score3 = createScore({
+      id: scoreId3,
+      observation_id: observationId2,
+      trace_id: traceId,
+      project_id: projectId,
+      name: anotherScoreName,
+      value: 1,
+      comment: "some other comment for non run related score",
+    });
+    await createScores([score, score2, score3]);
+
+    const runs = await createDatasetRunsTable({
+      projectId,
+      datasetId,
+      queryClickhouse: false,
+      page: 0,
+      limit: 10,
+    });
+
+    console.log("runs", JSON.stringify(runs));
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.run_id === datasetRunId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+    expect(firstRun.run_id).toEqual(datasetRunId);
+
+    expect(firstRun.run_description).toBeNull();
+    expect(firstRun.run_metadata).toEqual({});
+
+    expect(firstRun.avgLatency).toBeGreaterThanOrEqual(10800);
+    expect(firstRun.avgTotalCost.toString()).toStrictEqual("275");
+
+    const expectedObject = {
+      [`${scoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([1, 100.5]),
+        average: 50.75,
+      },
+      [`${anotherScoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([1]),
+        average: 1,
+        comment: "some other comment for non run related score",
+      },
+    };
+
+    expect(firstRun.scores).toEqual(expectedObject);
+
+    const secondRun = runs.find((run) => run.run_id === datasetRun2Id);
+
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("second run is not defined");
+    }
+
+    expect(secondRun.run_id).toEqual(datasetRun2Id);
+    expect(secondRun.run_description).toBeNull();
+    expect(secondRun.run_metadata).toEqual({});
+    expect(secondRun.avgLatency).toEqual(1);
+    expect(secondRun.avgTotalCost.toString()).toStrictEqual("300");
+
+    expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
+  });
+
+  it("should fetch dataset run items for UI", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const traceId = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const traceId2 = v4();
+    const observationId = v4();
+    const datasetRunItemId2 = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRunId,
+        traceId: traceId2,
+        projectId,
+        datasetItemId,
+        observationId,
+      },
+    });
+
+    const trace1 = createTrace({
+      id: traceId,
+      project_id: projectId,
+    });
+
+    const trace2 = createTrace({
+      id: traceId2,
+      project_id: projectId,
+    });
+
+    await createTraces([trace1, trace2]);
+
+    const observation = createObservation({
+      id: observationId,
+      trace_id: traceId2,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
+      end_time: new Date().getTime(),
+    });
+
+    const observation2 = createObservation({
+      trace_id: traceId,
+    });
+
+    await createObservations([observation]);
+
+    const score = createScore({
+      observation_id: observation2.id,
+      trace_id: traceId2,
+      project_id: projectId,
+    });
+
+    await createScores([score]);
+
+    const runs = await getRunItemsByRunIdOrItemId(
+      projectId,
+      // fetch directly from the db to have realistic data.
+      await prisma.datasetRunItems.findMany({
+        where: {
+          id: {
+            in: [datasetRunItemId, datasetRunItemId2],
+          },
+        },
+      }),
+    );
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.id === datasetRunItemId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+
+    expect(firstRun.id).toEqual(datasetRunItemId);
+    expect(firstRun.datasetItemId).toEqual(datasetItemId);
+    expect(firstRun.observation).toBeUndefined();
+    expect(firstRun.trace).toBeDefined();
+    expect(firstRun.trace?.id).toEqual(traceId);
+
+    const secondRun = runs.find((run) => run.id === datasetRunItemId2);
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("secondRun is not defined");
+    }
+
+    expect(secondRun.id).toEqual(datasetRunItemId2);
+    expect(secondRun.datasetItemId).toEqual(datasetItemId);
+    expect(secondRun.trace?.id).toEqual(traceId2);
+    expect(secondRun.observation?.id).toEqual(observationId);
+
+    const expectedObject = {
+      [`${score.name.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([100.5]),
+        average: 100.5,
+        comment: "comment",
+      },
+    };
+
+    expect(secondRun.scores).toEqual(expectedObject);
+  });
+
+  it("should fetch dataset run items for UI with missing tracing data", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const traceId = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const traceId2 = v4();
+    const observationId = v4();
+    const datasetRunItemId2 = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRunId,
+        traceId: traceId2,
+        projectId,
+        datasetItemId,
+        observationId,
+      },
+    });
+
+    const runs = await getRunItemsByRunIdOrItemId(
+      projectId,
+      // fetch directly from the db to have realistic data.
+      await prisma.datasetRunItems.findMany({
+        where: {
+          id: {
+            in: [datasetRunItemId, datasetRunItemId2],
+          },
+        },
+      }),
+    );
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.id === datasetRunItemId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+
+    expect(firstRun.id).toEqual(datasetRunItemId);
+    expect(firstRun.datasetItemId).toEqual(datasetItemId);
+    expect(firstRun.observation).toBeUndefined();
+    expect(firstRun.trace).toBeDefined();
+    expect(firstRun.trace?.id).toEqual(traceId);
+
+    const secondRun = runs.find((run) => run.id === datasetRunItemId2);
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("secondRun is not defined");
+    }
+
+    expect(secondRun.id).toEqual(datasetRunItemId2);
+    expect(secondRun.datasetItemId).toEqual(datasetItemId);
+    expect(secondRun.trace?.id).toEqual(traceId2);
+    expect(secondRun.observation?.id).toEqual(observationId);
+  });
+
+  it("should fetch dataset items correctly", async () => {
+    // Create test data in the database
+
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+
+    const traceId1 = v4();
+    const traceId2 = v4();
+    const observationId2 = v4();
+
+    const datasetItemId = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+        sourceTraceId: traceId1,
+      },
+    });
+
+    const datasetItemId2 = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId2,
+        datasetId,
+        metadata: {},
+        projectId,
+        sourceTraceId: traceId2,
+        sourceObservationId: observationId2,
+      },
+    });
+
+    const datasetItemId3 = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId3,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const observation = createObservation({
+      id: observationId2,
+      trace_id: traceId2,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
+      end_time: new Date().getTime(),
+    });
+
+    await createObservations([observation]);
+
+    const trace1 = createTrace({
+      id: traceId1,
+      project_id: projectId,
+    });
+    const trace2 = createTrace({
+      id: traceId2,
+      project_id: projectId,
+    });
+
+    await createTraces([trace1, trace2]);
+
+    const input = {
+      projectId: projectId,
+      datasetId: datasetId,
+      limit: 10,
+      page: 0,
+      prisma: prisma,
+    };
+
+    const result = await fetchDatasetItems(input);
+
+    expect(result.totalDatasetItems).toEqual(3);
+    expect(result.datasetItems).toHaveLength(3);
+
+    const firstDatasetItem = result.datasetItems.find(
+      (item) => item.id === datasetItemId,
+    );
+    expect(firstDatasetItem).toBeDefined();
+    if (!firstDatasetItem) {
+      throw new Error("firstDatasetItem is not defined");
+    }
+    expect(firstDatasetItem.sourceTraceId).toEqual(traceId1);
+    expect(firstDatasetItem.sourceObservationId).toBeNull();
+
+    const secondDatasetItem = result.datasetItems.find(
+      (item) => item.id === datasetItemId2,
+    );
+    expect(secondDatasetItem).toBeDefined();
+    if (!secondDatasetItem) {
+      throw new Error("secondDatasetItem is not defined");
+    }
+    expect(secondDatasetItem.sourceTraceId).toEqual(traceId2);
+    expect(secondDatasetItem.sourceObservationId).toEqual(observationId2);
+
+    const thirdDatasetItem = result.datasetItems.find(
+      (item) => item.id === datasetItemId3,
+    );
+    expect(thirdDatasetItem).toBeDefined();
+    if (!thirdDatasetItem) {
+      throw new Error("thirdDatasetItem is not defined");
+    }
+    expect(thirdDatasetItem.sourceTraceId).toBeNull();
+    expect(thirdDatasetItem.sourceObservationId).toBeNull();
+  });
+});

web/src/__tests__/async/observation-api.servertest.ts

@@ -0,0 +1,253 @@
+import { createObservation as createObservationObject } from "@/src/__tests__/fixtures/tracing-factory";
+import { createObservations as createObservationsInClickhouse } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
+import { GetObservationV1Response } from "@/src/features/public-api/types/observations";
+import { v4 } from "uuid";
+import { v4 as uuidv4 } from "uuid";
+import { GetObservationsV1Response } from "@/src/features/public-api/types/observations";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("/api/public/observations API Endpoint", () => {
+  describe("GET /api/public/observations/:id", () => {
+    it("should GET an observation", async () => {
+      const observationId = v4();
+      const traceId = v4();
+
+      const observation = createObservationObject({
+        id: observationId,
+        project_id: projectId,
+        trace_id: traceId,
+        internal_model_id: "b9854a5c92dc496b997d99d21",
+        provided_model_name: "gpt-4o-2024-05-13",
+        input: "input",
+        output: "output",
+      });
+
+      await createObservationsInClickhouse([observation]);
+
+      const getEventRes = await makeZodVerifiedAPICall(
+        GetObservationV1Response,
+        "GET",
+        "/api/public/observations/" + observationId,
+      );
+      expect(getEventRes.body).toMatchObject({
+        id: observationId,
+        traceId: traceId,
+        type: observation.type,
+        modelId: observation.internal_model_id,
+        inputPrice: 0.000005,
+        input: observation.input,
+        output: observation.output,
+      });
+    });
+  });
+});
+
+describe("/api/public/observations API Endpoint", () => {
+  it("should fetch all observations", async () => {
+    const traceId = uuidv4();
+
+    const observation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      internal_model_id: "clrkwk4cb000408l576jl7koo",
+      provided_model_name: "gpt-3.5-turbo",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      usage_details: {
+        input: 10,
+        output: 20,
+        total: 30,
+      },
+      version: "2.0.0",
+      type: "GENERATION",
+    });
+
+    await createObservationsInClickhouse([observation]);
+
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      "/api/public/observations?traceId=" + traceId,
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
+    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
+    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
+    expect(fetchedObservations.body.data[0]?.model).toEqual("gpt-3.5-turbo");
+    expect(fetchedObservations.body.data[0]?.modelId).toEqual(
+      "clrkwk4cb000408l576jl7koo",
+    );
+    expect(
+      fetchedObservations.body.data[0]?.calculatedInputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedOutputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedTotalCost,
+    ).toBeGreaterThan(0);
+  });
+
+  it("should fetch all observations, filtered by generations", async () => {
+    const traceId = uuidv4();
+
+    const generationObservation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      internal_model_id: "model-1",
+      provided_model_name: "gpt-3.5-turbo",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      usage_details: {
+        input: 10,
+        output: 20,
+        total: 30,
+      },
+      version: "2.0.0",
+      type: "GENERATION",
+    });
+
+    const spanObservation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      version: "2.0.0",
+      type: "SPAN",
+    });
+
+    await createObservationsInClickhouse([
+      generationObservation,
+      spanObservation,
+    ]);
+
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      "/api/public/observations?type=GENERATION&traceId=" + traceId,
+      undefined,
+    );
+
+    console.log(fetchedObservations.body);
+
+    expect(fetchedObservations.status).toBe(200);
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
+    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
+    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
+    expect(fetchedObservations.body.data[0]?.type).toEqual("GENERATION");
+  });
+
+  it("GET /observations with timestamp filters and pagination", async () => {
+    const traceId = v4();
+    const obs1 = createObservationObject({
+      id: "observation-2021-01-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "GENERATION",
+    });
+
+    const obs2 = createObservationObject({
+      id: "observation-2021-02-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "SPAN",
+    });
+    const obs3 = createObservationObject({
+      id: "observation-2021-03-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-03-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "EVENT",
+    });
+    const obs4 = createObservationObject({
+      id: "observation-2021-04-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "GENERATION",
+    });
+
+    await createObservationsInClickhouse([obs1, obs2, obs3, obs4]);
+
+    const fromTimestamp = "2021-02-01T00:00:00.000Z";
+    const toTimestamp = "2021-04-01T00:00:00.000Z";
+
+    // Test with both fromTimestamp and toTimestamp
+    let fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(2);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(2);
+
+    // Test with only fromTimestamp
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?fromStartTime=${fromTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(3);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+    // Test with only toTimestamp
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?toStartTime=${toTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(3);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+    // test pagination only
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?limit=1&page=2&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.meta).toMatchObject({
+      totalItems: 4,
+      totalPages: 4,
+      page: 2,
+      limit: 1,
+    });
+  });
+});

web/src/__tests__/async/scores-api.servertest.ts

@@ -1,13 +1,27 @@
-import { createScores } from "@/src/__tests__/server/repositories/clickhouse-helpers";
-import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
-import { GetScoreResponse } from "@langfuse/shared";
+import {
+  createObservation,
+  createScore,
+  createTrace,
+} from "@/src/__tests__/fixtures/tracing-factory";
+import {
+  createObservations,
+  createScores,
+  createTraces,
+} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import {
+  createOrgProjectAndApiKey,
+  makeZodVerifiedAPICall,
+} from "@/src/__tests__/test-utils";
+import { GetScoreResponse, GetScoresResponse } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
-
-const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+import { z } from "zod";
 
 describe("/api/public/scores API Endpoint", () => {
   describe("GET /api/public/scores/:scoreId", () => {
     it("should GET a score", async () => {
+      const { projectId: projectId, auth } = await createOrgProjectAndApiKey();
+
       const scoreId = v4();
       const traceId = v4();
       const score = {
@@ -33,6 +47,8 @@ describe("/api/public/scores API Endpoint", () => {
         GetScoreResponse,
         "GET",
         `/api/public/scores/${scoreId}`,
+        undefined,
+        auth,
       );
 
       expect(getScore.status).toBe(200);
@@ -49,3 +65,653 @@ describe("/api/public/scores API Endpoint", () => {
     });
   });
 });
+
+describe("/api/public/scores API Endpoint", () => {
+  it("should create score for a trace", async () => {
+    const traceId = v4();
+
+    const { projectId: projectId, auth } = await createOrgProjectAndApiKey();
+
+    const trace = createTrace({
+      id: traceId,
+      project_id: projectId,
+    });
+    await createTraces([trace]);
+
+    const scoreId = v4();
+
+    const score = createScore({
+      id: scoreId,
+      project_id: projectId,
+      trace_id: traceId,
+      name: "score-name",
+      value: 100.5,
+      source: "API",
+      comment: "comment",
+      observation_id: null,
+    });
+    await createScores([score]);
+
+    const fetchedScore = await makeZodVerifiedAPICall(
+      GetScoreResponse,
+      "GET",
+      `/api/public/scores/${scoreId}`,
+      undefined,
+      auth,
+    );
+
+    expect(fetchedScore.body?.id).toBe(scoreId);
+    expect(fetchedScore.body?.traceId).toBe(traceId);
+    expect(fetchedScore.body?.name).toBe("score-name");
+    expect(fetchedScore.body?.value).toBe(100.5);
+    expect(fetchedScore.body?.observationId).toBeNull();
+    expect(fetchedScore.body?.comment).toBe("comment");
+    expect(fetchedScore.body?.source).toBe("API");
+    expect(fetchedScore.body?.projectId).toBe(projectId);
+  });
+  it("should GET score with minimal score data and minimal trace data", async () => {
+    const { projectId, auth } = await createOrgProjectAndApiKey();
+
+    const minimalTraceId = v4();
+
+    const trace = createTrace({
+      id: minimalTraceId,
+      project_id: projectId,
+    });
+    await createTraces([trace]);
+
+    const minimalScoreId = v4();
+
+    const score = createScore({
+      id: minimalScoreId,
+      project_id: projectId,
+      trace_id: minimalTraceId,
+      name: "score-name",
+      value: 100.5,
+      source: "API",
+      comment: null,
+      observation_id: null,
+    });
+    await createScores([score]);
+
+    const fetchedScore = await makeZodVerifiedAPICall(
+      GetScoreResponse,
+      "GET",
+      `/api/public/scores/${minimalScoreId}`,
+      undefined,
+      auth,
+    );
+
+    expect(fetchedScore.status).toBe(200);
+  });
+  describe("should Filter scores", () => {
+    let configId = "";
+    const userId = "user-name";
+    const traceTags = ["prod", "test"];
+    const traceTags_2 = ["staging", "dev"];
+    const scoreName = "score-name";
+    const queryUserName = `userId=${userId}&name=${scoreName}`;
+    const traceId = v4();
+    const traceId_2 = v4();
+    const traceId_3 = v4();
+    const generationId = v4();
+    const scoreId_1 = v4();
+    const scoreId_2 = v4();
+    const scoreId_3 = v4();
+    const scoreId_4 = v4();
+    const scoreId_5 = v4();
+    let authentication: string;
+    let newProjectId: string;
+
+    beforeEach(async () => {
+      const { projectId, auth } = await createOrgProjectAndApiKey();
+      authentication = auth;
+      newProjectId = projectId;
+
+      const trace = createTrace({
+        id: traceId,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: traceTags,
+      });
+
+      const trace_2 = createTrace({
+        id: traceId_2,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: traceTags_2,
+      });
+
+      const trace_3 = createTrace({
+        id: traceId_3,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: ["staging"],
+      });
+
+      await createTraces([trace, trace_2, trace_3]);
+
+      const generation = createObservation({
+        id: generationId,
+        project_id: newProjectId,
+        type: "GENERATION",
+      });
+
+      await createObservations([generation]);
+
+      const config = await prisma.scoreConfig.create({
+        data: {
+          name: scoreName,
+          dataType: "NUMERIC",
+          maxValue: 100,
+          projectId: newProjectId,
+        },
+      });
+
+      configId = config.id;
+
+      const score1 = createScore({
+        id: scoreId_1,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 10.5,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        config_id: config.id,
+        comment: "comment",
+      });
+
+      const score2 = createScore({
+        id: scoreId_2,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 50.5,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        comment: "comment",
+      });
+
+      const score3 = createScore({
+        id: scoreId_3,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 100.8,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        comment: "comment",
+      });
+
+      const score4 = createScore({
+        id: scoreId_4,
+        project_id: newProjectId,
+        trace_id: traceId_2,
+        name: "other-score-name",
+        value: 0,
+        string_value: "best",
+        data_type: "CATEGORICAL",
+        comment: "comment",
+      });
+
+      const score5 = createScore({
+        id: scoreId_5,
+        project_id: newProjectId,
+        trace_id: traceId_3,
+        name: "other-score-name",
+        value: 0,
+        data_type: "CATEGORICAL",
+        string_value: "test",
+        comment: "comment",
+      });
+
+      await createScores([score1, score2, score3, score4, score5]);
+    });
+
+    it("get all scores", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}`,
+        undefined,
+        authentication,
+      );
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+        });
+      }
+    });
+
+    it("get all scores for config", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?configId=${configId}`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+          configId: configId,
+        });
+      }
+    });
+
+    it("get all scores for numeric data type", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?dataType=NUMERIC`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+          dataType: "NUMERIC",
+        });
+      }
+    });
+
+    it("get all scores for trace tag 'prod'", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?traceTags=prod`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          trace: { tags: ["prod", "test"], userId: "user-name" },
+        });
+      }
+    });
+
+    it("get all scores for trace tags 'staging' and 'dev'", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?traceTags=${["staging", "dev"]}`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId_2,
+          trace: {
+            tags: expect.arrayContaining(["dev", "staging"]),
+            userId: "user-name",
+          },
+        });
+      }
+    });
+
+    describe("should Filter scores by queueId", () => {
+      describe("queueId filtering", () => {
+        let queueId: string;
+
+        beforeEach(async () => {
+          queueId = v4();
+          const score = createScore({
+            id: v4(),
+            project_id: newProjectId,
+            trace_id: traceId,
+            name: "score-name",
+            value: 100.5,
+            source: "ANNOTATION",
+            comment: "comment",
+            observation_id: generationId,
+            queue_id: queueId,
+          });
+          const score2 = createScore({
+            id: v4(),
+            project_id: newProjectId,
+            trace_id: traceId,
+            name: "score-name",
+            value: 75.0,
+            source: "ANNOTATION",
+            comment: "comment",
+            observation_id: generationId,
+            queue_id: queueId,
+          });
+
+          await createScores([score, score2]);
+        });
+
+        it("get all scores for queueId", async () => {
+          const getAllScore = await makeZodVerifiedAPICall(
+            GetScoresResponse,
+            "GET",
+            `/api/public/scores?queueId=${queueId}`,
+            undefined,
+            authentication,
+          );
+          expect(getAllScore.status).toBe(200);
+          expect(getAllScore.body.meta).toMatchObject({
+            page: 1,
+            limit: 50,
+            totalItems: 2,
+            totalPages: 1,
+          });
+          for (const val of getAllScore.body.data) {
+            expect(val).toMatchObject({
+              traceId: traceId,
+              observationId: generationId,
+              queueId: queueId,
+              source: "ANNOTATION",
+            });
+          }
+        });
+      });
+    });
+
+    it("test only operator", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test only value", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&value=0.8`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test operator <", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<&value=50`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
+    it("test operator >", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>&value=100`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_3,
+          name: scoreName,
+          value: 100.8,
+        },
+      ]);
+    });
+    it("test operator <=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator >=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_3,
+            name: scoreName,
+            value: 100.8,
+          }),
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator !=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=!=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_3,
+            name: scoreName,
+            value: 100.8,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator =", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator==&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+      ]);
+    });
+
+    it("test invalid operator", async () => {
+      try {
+        await makeZodVerifiedAPICall(
+          z.object({
+            message: z.string(),
+            error: z.array(z.object({})),
+          }),
+          "GET",
+          `/api/public/scores?${queryUserName}&operator=op&value=50.5`,
+          undefined,
+          authentication,
+        );
+      } catch (error) {
+        expect((error as Error).message).toBe(
+          `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"received\":\"op\",\"code\":\"invalid_enum_value\",\"options\":[\"<\",\">\",\"<=\",\">=\",\"!=\",\"=\"],\"path\":[\"operator\"],\"message\":\"Invalid enum value. Expected '<' | '>' | '<=' | '>=' | '!=' | '=', received 'op'\"}]}`,
+        );
+      }
+    });
+    it("test invalid value", async () => {
+      try {
+        await makeZodVerifiedAPICall(
+          z.object({
+            message: z.string(),
+            error: z.array(z.object({})),
+          }),
+          "GET",
+          `/api/public/scores?${queryUserName}&operator=<&value=myvalue`,
+          undefined,
+          authentication,
+        );
+      } catch (error) {
+        expect((error as Error).message).toBe(
+          'API call did not return 200, returned status 400, body {"message":"Invalid request data","error":[{"code":"invalid_type","expected":"number","received":"nan","path":["value"],"message":"Expected number, received nan"}]}',
+        );
+      }
+    });
+
+    it("should filter scores by score IDs", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+  });
+});

web/src/__tests__/async/traces-api.servertest.ts

@@ -0,0 +1,76 @@
+import {
+  createObservation,
+  createTrace,
+} from "@/src/__tests__/fixtures/tracing-factory";
+import {
+  createObservations,
+  createTraces,
+} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
+import { GetTraceV1Response } from "@/src/features/public-api/types/traces";
+
+describe("/api/public/traces API Endpoint", () => {
+  it("should create and get a trace via /traces", async () => {
+    const createdTrace = createTrace({
+      name: "trace-name",
+      user_id: "user-1",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    });
+
+    const observations = [
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        input: "input",
+        output: "output",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-2",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 100000,
+        input: "input-2",
+        output: "output-2",
+      }),
+    ];
+
+    await createTraces([createdTrace]);
+    await createObservations(observations);
+
+    const trace = await makeZodVerifiedAPICall(
+      GetTraceV1Response,
+      "GET",
+      "/api/public/traces/" + createdTrace.id,
+    );
+
+    expect(trace.body.name).toBe("trace-name");
+    expect(trace.body.release).toBe("1.0.0");
+    expect(trace.body.externalId).toBeNull();
+    expect(trace.body.version).toBe("2.0.0");
+    expect(trace.body.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
+    expect(trace.body.latency).toBe(100);
+    expect(trace.body.observations.length).toBe(2);
+    expect(trace.body.scores.length).toBe(0);
+    expect(trace.body.observations).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          name: "observation-name-2",
+          input: "input-2",
+          output: "output-2",
+        }),
+        expect.objectContaining({
+          name: "observation-name",
+          input: "input",
+          output: "output",
+        }),
+      ]),
+    );
+  });
+});

web/src/__tests__/async/traces-ui-table.servertest.ts

@@ -0,0 +1,277 @@
+import { v4 } from "uuid";
+import {
+  createObservations,
+  createTraces,
+} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import {
+  createObservation,
+  createTrace,
+} from "@/src/__tests__/fixtures/tracing-factory";
+import {
+  getTracesTable,
+  type ObservationRecordInsertType,
+  type TraceRecordInsertType,
+  type TracesTableReturnType,
+} from "@langfuse/shared/src/server";
+import { type FilterState } from "@langfuse/shared";
+
+describe("Traces table API test", () => {
+  it("should get a correct trace without observation", async () => {
+    const project_id = v4();
+    const trace_id = v4();
+
+    const trace = createTrace({ id: trace_id, project_id });
+    await createTraces([trace]);
+
+    const tableRows = await getTracesTable(
+      project_id,
+      [],
+      undefined,
+      undefined,
+      1,
+      0,
+    );
+
+    expect(tableRows).toHaveLength(1);
+    expect(tableRows[0].id).toEqual(trace_id);
+    expect(tableRows[0].projectId).toEqual(project_id);
+    expect(tableRows[0].tags).toEqual(trace.tags);
+    expect(tableRows[0].name).toEqual(trace.name);
+    expect(tableRows[0].bookmarked).toEqual(trace.bookmarked);
+    expect(tableRows[0].release).toEqual(trace.release);
+    expect(tableRows[0].version).toEqual(trace.version);
+    expect(tableRows[0].userId).toEqual(trace.user_id);
+    expect(tableRows[0].sessionId).toEqual(trace.session_id);
+    expect(tableRows[0].public).toEqual(trace.public);
+    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].usageDetails).toEqual({});
+    expect(tableRows[0].costDetails).toEqual({});
+    expect(tableRows[0].level).toBeDefined();
+    expect(tableRows[0].observationCount).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].scoresAvg).toEqual([]);
+  });
+
+  it("should get a correct trace with observations", async () => {
+    const project_id = v4();
+    const trace_id = v4();
+
+    const trace = createTrace({ id: trace_id, project_id });
+    await createTraces([trace]);
+
+    const obs1 = createObservation({ trace_id, project_id });
+    const obs2 = createObservation({ trace_id, project_id });
+    await createObservations([obs1, obs2]);
+
+    const tableRows = await getTracesTable(
+      project_id,
+      [],
+      undefined,
+      undefined,
+      1,
+      0,
+    );
+
+    expect(tableRows).toHaveLength(1);
+    expect(tableRows[0].id).toEqual(trace_id);
+    expect(tableRows[0].projectId).toEqual(project_id);
+    expect(tableRows[0].tags).toEqual(trace.tags);
+    expect(tableRows[0].name).toEqual(trace.name);
+    expect(tableRows[0].bookmarked).toEqual(trace.bookmarked);
+    expect(tableRows[0].release).toEqual(trace.release);
+    expect(tableRows[0].version).toEqual(trace.version);
+    expect(tableRows[0].userId).toEqual(trace.user_id);
+    expect(tableRows[0].sessionId).toEqual(trace.session_id);
+    expect(tableRows[0].public).toEqual(trace.public);
+    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].usageDetails).toEqual({
+      input: (obs1.usage_details.input + obs2.usage_details.input).toString(),
+      output: (
+        obs1.usage_details.output + obs2.usage_details.output
+      ).toString(),
+      total: (obs1.usage_details.total + obs2.usage_details.total).toString(),
+    });
+    expect(tableRows[0].costDetails).toEqual({
+      input: obs1.cost_details.input + obs2.cost_details.input,
+      output: obs1.cost_details.output + obs2.cost_details.output,
+      total: obs1.cost_details.total + obs2.cost_details.total,
+    });
+    expect(tableRows[0].level).toBeDefined();
+    expect(tableRows[0].observationCount).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].scoresAvg).toEqual([]);
+  });
+
+  type TestCase = {
+    traceInput: Partial<TraceRecordInsertType>;
+    observationInput: Partial<ObservationRecordInsertType>[];
+    filterstate: FilterState;
+    expected: Partial<TracesTableReturnType>[];
+  };
+
+  [
+    {
+      traceInput: {},
+      observationInput: [
+        { cost_details: { total: 100 } },
+        { cost_details: { total: 200 } },
+      ],
+      filterstate: [
+        {
+          column: "totalCost",
+          operator: ">" as const,
+          value: 100000,
+          type: "number" as const,
+        },
+      ],
+      expected: [],
+    },
+    {
+      traceInput: {},
+      observationInput: [
+        {
+          cost_details: { total: 0.000001 },
+          usage_details: { total: 0 },
+        },
+        {
+          cost_details: { total: 0.000002 },
+          usage_details: { total: 0 },
+        },
+      ],
+      filterstate: [
+        {
+          column: "Total Cost ($)",
+          operator: ">" as const,
+          value: 0.000002,
+          type: "number" as const,
+        },
+      ],
+      expected: [
+        {
+          cost_details: { total: 0.000003 },
+        },
+      ],
+    },
+    {
+      traceInput: {},
+      observationInput: [],
+      filterstate: [
+        {
+          column: "id",
+          operator: "=" as const,
+          value: "some-id",
+          type: "string" as const,
+        },
+      ],
+      expected: [],
+    },
+    {
+      traceInput: {},
+      observationInput: [],
+      filterstate: [
+        {
+          column: "Latency (s)",
+          operator: ">" as const,
+          value: 5_000_000, // Verify that we can pass large values
+          type: "number" as const,
+        },
+      ],
+      expected: [],
+    },
+  ].forEach(async (testConfig: TestCase) => {
+    it(`should get a correct trace with filters ${JSON.stringify(testConfig)}`, async () => {
+      const project_id = v4();
+      const trace_id = v4();
+
+      const trace = createTrace({
+        id: trace_id,
+        project_id,
+        ...testConfig.traceInput,
+      });
+      await createTraces([trace]);
+
+      expect(testConfig.observationInput.length).not.toBeGreaterThan(2);
+
+      const obs1 = createObservation({
+        trace_id,
+        project_id,
+        ...(testConfig.observationInput.length > 0
+          ? testConfig.observationInput[0]
+          : {}),
+      });
+      const obs2 = createObservation({
+        trace_id,
+        project_id,
+        ...(testConfig.observationInput.length > 1
+          ? testConfig.observationInput[1]
+          : {}),
+      });
+      await createObservations([obs1, obs2]);
+
+      const tableRows = await getTracesTable(
+        project_id,
+        testConfig.filterstate,
+        undefined,
+        undefined,
+        1,
+        0,
+      );
+
+      expect(tableRows).toHaveLength(testConfig.expected.length);
+      testConfig.expected.forEach((expectedTrace, index) => {
+        if (expectedTrace.id !== undefined) {
+          expect(tableRows[index].id).toEqual(expectedTrace.id);
+        }
+        if (expectedTrace.project_id !== undefined) {
+          expect(tableRows[index].projectId).toEqual(expectedTrace.project_id);
+        }
+        if (expectedTrace.tags !== undefined) {
+          expect(tableRows[index].tags).toEqual(expectedTrace.tags);
+        }
+        if (expectedTrace.name !== undefined) {
+          expect(tableRows[index].name).toEqual(expectedTrace.name);
+        }
+        if (expectedTrace.bookmarked !== undefined) {
+          expect(tableRows[index].bookmarked).toEqual(expectedTrace.bookmarked);
+        }
+        if (expectedTrace.release !== undefined) {
+          expect(tableRows[index].release).toEqual(expectedTrace.release);
+        }
+        if (expectedTrace.version !== undefined) {
+          expect(tableRows[index].version).toEqual(expectedTrace.version);
+        }
+        if (expectedTrace.user_id !== undefined) {
+          expect(tableRows[index].userId).toEqual(expectedTrace.user_id);
+        }
+        if (expectedTrace.session_id !== undefined) {
+          expect(tableRows[index].sessionId).toEqual(expectedTrace.session_id);
+        }
+        if (expectedTrace.public !== undefined) {
+          expect(tableRows[index].public).toEqual(expectedTrace.public);
+        }
+        if (expectedTrace.latency !== undefined) {
+          expect(tableRows[index].latency).toEqual(expectedTrace.latency);
+        }
+        if (expectedTrace.usage_details !== undefined) {
+          expect(tableRows[index].usageDetails).toEqual(
+            expectedTrace.usage_details,
+          );
+        }
+        if (expectedTrace.cost_details !== undefined) {
+          expect(tableRows[index].costDetails).toEqual(
+            expectedTrace.cost_details,
+          );
+        }
+        if (expectedTrace.level !== undefined) {
+          expect(tableRows[index].level).toEqual(expectedTrace.level);
+        }
+        if (expectedTrace.observation_count !== undefined) {
+          expect(tableRows[index].observationCount).toEqual(
+            expectedTrace.observation_count,
+          );
+        }
+        if (expectedTrace.scores_avg !== undefined) {
+          expect(tableRows[index].scoresAvg).toEqual(expectedTrace.scores_avg);
+        }
+      });
+    });
+  });
+});

web/src/__tests__/fixtures/tracing-factory.ts

@@ -1,4 +1,5 @@
 import {
+  type ScoreRecordInsertType,
   type ObservationRecordInsertType,
   type TraceRecordInsertType,
 } from "@langfuse/shared/src/server";
@@ -68,3 +69,23 @@ export const createObservation = (
     ...observation,
   };
 };
+
+export const createScore = (score: Partial<ScoreRecordInsertType>) => {
+  return {
+    id: v4(),
+    project_id: v4(),
+    trace_id: v4(),
+    observation_id: v4(),
+    name: "test-score" + v4(),
+    timestamp: Date.now(),
+    value: 100.5,
+    source: "API",
+    comment: "comment",
+    data_type: "NUMERIC" as const,
+    created_at: Date.now(),
+    updated_at: Date.now(),
+    event_ts: Date.now(),
+    is_deleted: 0,
+    ...score,
+  };
+};

web/src/__tests__/media.servertest.ts

@@ -167,7 +167,12 @@ describe("Media Upload API", () => {
       );
       result.getDownloadUrlResponse = getDownloadUrlResponse;
 
-      if (getDownloadUrlResponse.status !== 200) {
+      if (
+        !(
+          getDownloadUrlResponse.status === 200 ||
+          getDownloadUrlResponse.status === 201
+        )
+      ) {
         return result;
       }
 

web/src/__tests__/observations.servertest.ts

@@ -1,441 +0,0 @@
-/** @jest-environment node */
-
-import { v4 as uuidv4 } from "uuid";
-
-import {
-  makeZodVerifiedAPICall,
-  pruneDatabase,
-} from "@/src/__tests__/test-utils";
-import { ModelUsageUnit } from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
-import {
-  GetObservationV1Response,
-  GetObservationsV1Response,
-} from "@/src/features/public-api/types/observations";
-
-describe("/api/public/observations API Endpoint", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
-
-  it("should fetch all observations", async () => {
-    await pruneDatabase();
-
-    const traceId = uuidv4();
-
-    await prisma.trace.create({
-      data: {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    });
-
-    const model = await prisma.model.create({
-      data: {
-        id: "model-1",
-        modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0000010",
-        outputPrice: "0.0000020",
-        totalPrice: "0.1",
-        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        projectId: null,
-        unit: ModelUsageUnit.Tokens,
-      },
-    });
-
-    const prompt = await prisma.prompt.create({
-      data: {
-        name: "prompt-name",
-        prompt: "prompt-one",
-        isActive: false,
-        version: 1,
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "gpt-3.5-turbo",
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        promptTokens: 10,
-        completionTokens: 20,
-        totalTokens: 30,
-        version: "2.0.0",
-        type: "GENERATION",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        internalModel: "gpt-3.5-turbo",
-        unit: ModelUsageUnit.Tokens,
-        promptId: prompt.id,
-      },
-    });
-
-    const fetchedObservations = await makeZodVerifiedAPICall(
-      GetObservationsV1Response,
-      "GET",
-      "/api/public/observations",
-      undefined,
-    );
-
-    expect(fetchedObservations.status).toBe(200);
-
-    expect(fetchedObservations.body.data.length).toBe(1);
-    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
-    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
-    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
-    expect(fetchedObservations.body.data[0]?.model).toEqual("gpt-3.5-turbo");
-    expect(fetchedObservations.body.data[0]?.modelId).toEqual(model.id);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedInputCost,
-    ).toBeGreaterThan(0);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedOutputCost,
-    ).toBeGreaterThan(0);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedTotalCost,
-    ).toBeGreaterThan(0);
-    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
-    expect(fetchedObservations.body.data[0]?.promptName).toBe(prompt.name);
-    expect(fetchedObservations.body.data[0]?.promptVersion).toBe(
-      prompt.version,
-    );
-  });
-  it("should fetch all observations, filtered by generations", async () => {
-    await pruneDatabase();
-
-    const traceId = uuidv4();
-
-    await prisma.trace.create({
-      data: {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    });
-
-    await prisma.model.create({
-      data: {
-        id: "model-1",
-        modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0000010",
-        outputPrice: "0.0000020",
-        totalPrice: "0.1",
-        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        projectId: null,
-        unit: ModelUsageUnit.Tokens,
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "gpt-3.5-turbo",
-        internalModel: "gpt-3.5-turbo",
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        promptTokens: 10,
-        completionTokens: 20,
-        totalTokens: 30,
-        version: "2.0.0",
-        unit: ModelUsageUnit.Tokens,
-        type: "GENERATION",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        version: "2.0.0",
-        type: "SPAN",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-      },
-    });
-
-    const fetchedObservations = await makeZodVerifiedAPICall(
-      GetObservationsV1Response,
-      "GET",
-      "/api/public/observations?type=GENERATION",
-      undefined,
-    );
-
-    console.log(fetchedObservations.body);
-
-    expect(fetchedObservations.status).toBe(200);
-
-    expect(fetchedObservations.body.data.length).toBe(1);
-    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
-    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
-    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
-    expect(fetchedObservations.body.data[0]?.type).toEqual("GENERATION");
-  });
-});
-
-it("GET /observations with timestamp filters and pagination", async () => {
-  await prisma.trace.create({
-    data: {
-      id: "trace-id",
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    },
-  });
-  await prisma.observation.createMany({
-    data: [
-      {
-        id: "observation-2021-01-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "GENERATION",
-      },
-      {
-        id: "observation-2021-02-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-02-01T00:00:00.000Z"),
-        endTime: new Date("2021-02-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "SPAN",
-      },
-      {
-        id: "observation-2021-03-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-03-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "EVENT",
-      },
-      {
-        id: "observation-2021-04-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-04-01T00:00:00.000Z"),
-        endTime: new Date("2021-04-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "GENERATION",
-      },
-    ],
-  });
-
-  const fromTimestamp = "2021-02-01T00:00:00.000Z";
-  const toTimestamp = "2021-04-01T00:00:00.000Z";
-
-  // Test with both fromTimestamp and toTimestamp
-  let fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(2);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(2);
-
-  // Test with only fromTimestamp
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?fromStartTime=${fromTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(3);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(3);
-
-  // Test with only toTimestamp
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?toStartTime=${toTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(3);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(3);
-
-  // test pagination only
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?limit=1&page=2`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(1);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.meta).toMatchObject({
-    totalItems: 4,
-    totalPages: 4,
-    page: 2,
-    limit: 1,
-  });
-});
-
-it("Get a single EVENT from /observations/:id", async () => {
-  const traceId = uuidv4();
-  await prisma.trace.create({
-    data: {
-      id: traceId,
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    },
-  });
-
-  // EVENT
-  const eventId = uuidv4();
-  await prisma.observation.create({
-    data: {
-      id: eventId,
-      traceId: traceId,
-      name: "generation-name",
-      startTime: new Date("2021-01-01T00:00:00.000Z"),
-      type: "EVENT",
-      project: {
-        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-      },
-    },
-  });
-  const getEventRes = await makeZodVerifiedAPICall(
-    GetObservationV1Response,
-    "GET",
-    "/api/public/observations/" + eventId,
-  );
-  expect(getEventRes.body).toMatchObject({
-    id: eventId,
-    traceId: traceId,
-    type: "EVENT",
-  });
-});
-it("Get a single GENERATION from /observations/:id", async () => {
-  const traceId = uuidv4();
-  await prisma.trace.create({
-    data: {
-      id: traceId,
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    },
-  });
-
-  const generationId = uuidv4();
-  await prisma.observation.create({
-    data: {
-      id: generationId,
-      traceId: traceId,
-      name: "generation-name",
-      startTime: new Date("2021-01-01T00:00:00.000Z"),
-      endTime: new Date("2021-01-01T00:00:00.000Z"),
-      model: "gpt-3.5-turbo",
-      internalModel: "gpt-3.5-turbo",
-      modelParameters: { key: "value" },
-      input: { key: "input" },
-      output: { key: "output" },
-      promptTokens: 10,
-      completionTokens: 20,
-      totalTokens: 30,
-      version: "2.0.0",
-      unit: ModelUsageUnit.Tokens,
-      type: "GENERATION",
-      project: {
-        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-      },
-    },
-  });
-  const getGenerationRes = await makeZodVerifiedAPICall(
-    GetObservationV1Response,
-    "GET",
-    "/api/public/observations/" + generationId,
-  );
-  expect(getGenerationRes.body).toMatchObject({
-    id: generationId,
-    traceId: traceId,
-    input: { key: "input" },
-    output: { key: "output" },
-    type: "GENERATION",
-  });
-});
-it("Get a single SPAN from /observations/:id", async () => {
-  const traceId = uuidv4();
-  await prisma.trace.create({
-    data: {
-      id: traceId,
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    },
-  });
-
-  const spanId = uuidv4();
-  await prisma.observation.create({
-    data: {
-      id: spanId,
-      traceId: traceId,
-      name: "generation-name",
-      startTime: new Date("2021-01-01T00:00:00.000Z"),
-      endTime: new Date("2021-01-01T00:00:00.000Z"),
-      modelParameters: { key: "value" },
-      input: { key: "input" },
-      output: { key: "output" },
-      version: "2.0.0",
-      type: "SPAN",
-      project: {
-        connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-      },
-    },
-  });
-  const getSpanRes = await makeZodVerifiedAPICall(
-    GetObservationV1Response,
-    "GET",
-    "/api/public/observations/" + spanId,
-  );
-  expect(getSpanRes.body).toMatchObject({
-    id: spanId,
-    traceId: traceId,
-    input: { key: "input" },
-    output: { key: "output" },
-    type: "SPAN",
-  });
-});

web/src/__tests__/scores.servertest.ts

@@ -7,12 +7,7 @@ import {
   pruneDatabase,
 } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
-import {
-  DeleteScoreResponse,
-  GetScoreResponse,
-  GetScoresResponse,
-} from "@langfuse/shared";
-import { z } from "zod";
+import { DeleteScoreResponse, GetScoreResponse } from "@langfuse/shared";
 import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 
 const traceId = "de98afa2-89dc-47e9-9924-33f1490fdaf4";
@@ -111,39 +106,6 @@ describe("/api/public/scores API Endpoint", () => {
     expect(fetchedScore.body?.observationId).toBeNull();
   });
 
-  it("should GET score with minimal score data and minimal trace data", async () => {
-    const minimalTraceId = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: minimalTraceId,
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    });
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        id: minimalTraceId,
-      },
-    });
-
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.id).toBe(minimalTraceId);
-
-    const minimalScoreId = uuidv4();
-
-    await makeAPICall("POST", "/api/public/scores", {
-      id: minimalScoreId,
-      name: "score-name",
-      value: 100,
-      traceId: minimalTraceId,
-    });
-
-    const fetchedScore = await makeZodVerifiedAPICall(
-      GetScoreResponse,
-      "GET",
-      `/api/public/scores/${minimalScoreId}`,
-    );
-
-    expect(fetchedScore.status).toBe(200);
-  });
-
   it("should create score for a generation", async () => {
     await pruneDatabase();
 
@@ -826,573 +788,4 @@ describe("/api/public/scores API Endpoint", () => {
     });
     expect(deletedScore).toBeNull();
   });
-
-  describe("should Filter scores", () => {
-    let configId = "";
-    const userId = "user-name";
-    const traceTags = ["prod", "test"];
-    const traceTags_2 = ["staging", "dev"];
-    const scoreName = "score-name";
-    const queryUserName = `userId=${userId}&name=${scoreName}`;
-    const traceId = uuidv4();
-    const traceId_2 = uuidv4();
-    const traceId_3 = uuidv4();
-    const generationId = uuidv4();
-    const scoreId_1 = uuidv4();
-    const scoreId_2 = uuidv4();
-    const scoreId_3 = uuidv4();
-    const scoreId_4 = uuidv4();
-    const scoreId_5 = uuidv4();
-
-    beforeAll(async () => {
-      should_prune_db = false;
-      await pruneDatabase();
-
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId,
-          userId: userId,
-          tags: traceTags,
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId_2,
-          userId: userId,
-          tags: traceTags_2,
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId_3,
-          userId: userId,
-          tags: ["staging"],
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/generations",
-        {
-          id: generationId,
-        },
-      );
-
-      await makeAPICall("POST", "/api/public/score-configs", {
-        name: scoreName,
-        dataType: "NUMERIC",
-        maxValue: 100,
-      });
-
-      const config = await prisma.scoreConfig.findFirst({
-        where: {
-          name: scoreName,
-        },
-      });
-      configId = config?.id ?? "";
-
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_1,
-        observationId: generationId,
-        name: scoreName,
-        value: 10.5,
-        traceId: traceId,
-        comment: "comment",
-        configId,
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_2,
-        observationId: generationId,
-        name: scoreName,
-        value: 50.5,
-        traceId: traceId,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_3,
-        observationId: generationId,
-        name: scoreName,
-        value: 100.8,
-        traceId: traceId,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_4,
-        name: "other-score-name",
-        value: "best",
-        traceId: traceId_2,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_5,
-        name: "other-score-name",
-        value: "test",
-        traceId: traceId_3,
-        comment: "comment",
-      });
-    });
-    afterAll(async () => {
-      await pruneDatabase();
-    });
-
-    it("get all scores", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}`,
-      );
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-        });
-      }
-    });
-
-    it("get all scores for config", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?configId=${configId}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-          configId: configId,
-        });
-      }
-    });
-
-    it("get all scores for numeric data type", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?dataType=${"NUMERIC"}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-          dataType: "NUMERIC",
-        });
-      }
-    });
-
-    it("get all scores for trace tag 'prod'", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?traceTags=${"prod"}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          trace: { tags: ["prod", "test"], userId: "user-name" },
-        });
-      }
-    });
-
-    it("get all scores for trace tags 'staging' and 'dev'", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?traceTags=${["staging", "dev"]}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId_2,
-          trace: { tags: ["dev", "staging"], userId: "user-name" },
-        });
-      }
-    });
-
-    describe("should Filter scores by queueId", () => {
-      describe("queueId filtering", () => {
-        let queueId: string;
-        const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-
-        beforeEach(async () => {
-          queueId = uuidv4();
-
-          await Promise.all([
-            prisma.score.create({
-              data: {
-                observationId: generationId,
-                name: "annotation-score-1",
-                value: 100.5,
-                traceId: traceId,
-                comment: "comment 1",
-                queueId,
-                source: "ANNOTATION",
-                project: { connect: { id: projectId } },
-                dataType: "NUMERIC",
-              },
-            }),
-            prisma.score.create({
-              data: {
-                observationId: generationId,
-                name: "annotation-score-2",
-                value: 75.0,
-                traceId: traceId,
-                comment: "comment 2",
-                queueId,
-                source: "ANNOTATION",
-                project: { connect: { id: projectId } },
-                dataType: "NUMERIC",
-              },
-            }),
-          ]);
-        });
-
-        afterEach(async () => {
-          await prisma.score.deleteMany({
-            where: { queueId, projectId },
-          });
-        });
-
-        it("get all scores for queueId", async () => {
-          const getAllScore = await makeZodVerifiedAPICall(
-            GetScoresResponse,
-            "GET",
-            `/api/public/scores?queueId=${queueId}`,
-          );
-
-          expect(getAllScore.status).toBe(200);
-          expect(getAllScore.body.meta).toMatchObject({
-            page: 1,
-            limit: 50,
-            totalItems: 2,
-            totalPages: 1,
-          });
-          for (const val of getAllScore.body.data) {
-            expect(val).toMatchObject({
-              traceId: traceId,
-              observationId: generationId,
-              queueId: queueId,
-              source: "ANNOTATION",
-            });
-          }
-        });
-      });
-    });
-
-    it("test only operator", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-    });
-
-    it("test only value", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&value=0.8`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-    });
-
-    it("test operator <", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<&value=50`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator >", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=>&value=100`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-      ]);
-    });
-    it("test operator <=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator >=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=>=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-      ]);
-    });
-    it("test operator !=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=!=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator =", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator==&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-      ]);
-    });
-    it("should GET ALL scores with minimal score data and minimal trace data", async () => {
-      const minimalTraceId = uuidv4();
-      await makeAPICall("POST", "/api/public/traces", {
-        id: minimalTraceId,
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      });
-      const dbTrace = await prisma.trace.findMany({
-        where: {
-          id: minimalTraceId,
-        },
-      });
-
-      expect(dbTrace.length).toBeGreaterThan(0);
-      expect(dbTrace[0]?.id).toBe(minimalTraceId);
-
-      const createScore = await makeAPICall("POST", "/api/public/scores", {
-        name: "score-name",
-        value: 100,
-        traceId: minimalTraceId,
-      });
-
-      expect(createScore.status).toBe(200);
-
-      const fetchedScores = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores`,
-      );
-
-      expect(fetchedScores.status).toBe(200);
-      expect(fetchedScores.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 6,
-        totalPages: 1,
-      });
-      expect(fetchedScores.body.data.length).toBe(6);
-    });
-
-    it("test invalid operator", async () => {
-      try {
-        await makeZodVerifiedAPICall(
-          z.object({
-            message: z.string(),
-            error: z.array(z.object({})),
-          }),
-          "GET",
-          `/api/public/scores?${queryUserName}&operator=op&value=50.5`,
-        );
-      } catch (error) {
-        expect((error as Error).message).toBe(
-          `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"received\":\"op\",\"code\":\"invalid_enum_value\",\"options\":[\"<\",\">\",\"<=\",\">=\",\"!=\",\"=\"],\"path\":[\"operator\"],\"message\":\"Invalid enum value. Expected '<' | '>' | '<=' | '>=' | '!=' | '=', received 'op'\"}]}`,
-        );
-      }
-    });
-    it("test invalid value", async () => {
-      try {
-        await makeZodVerifiedAPICall(
-          z.object({
-            message: z.string(),
-            error: z.array(z.object({})),
-          }),
-          "GET",
-          `/api/public/scores?${queryUserName}&operator=<&value=myvalue`,
-        );
-      } catch (error) {
-        expect((error as Error).message).toBe(
-          'API call did not return 200, returned status 400, body {"message":"Invalid request data","error":[{"code":"invalid_type","expected":"number","received":"nan","path":["value"],"message":"Expected number, received nan"}]}',
-        );
-      }
-    });
-
-    it("should filter scores by score IDs", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-  });
 });

web/src/__tests__/server/api/tables/traces-ui.servertest.ts

@@ -1,157 +0,0 @@
-import {
-  createObservation,
-  createTraces,
-} from "@/src/__tests__/server/repositories/clickhouse-helpers";
-import { pruneDatabase } from "@/src/__tests__/test-utils";
-import { getTracesTable } from "@langfuse/shared/src/server";
-import { v4 } from "uuid";
-
-const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-
-describe("UI Traces table", () => {
-  beforeEach(async () => {
-    await pruneDatabase();
-  });
-
-  it("should return a single trace", async () => {
-    const traceId = v4();
-
-    const trace = {
-      id: traceId,
-      project_id: projectId,
-      session_id: v4(),
-      timestamp: Date.now(),
-      metadata: {},
-      public: false,
-      bookmarked: false,
-      name: "Test Trace",
-      tags: ["tag-alpha", "tag-beta"],
-      release: "alpha",
-      version: "1.0.0",
-      user_id: "user-id-1",
-      created_at: Date.now(),
-      updated_at: Date.now(),
-      event_ts: Date.now(),
-      is_deleted: 0,
-    };
-
-    await createTraces([trace]);
-
-    const result = await getTracesTable(
-      projectId,
-      [],
-      undefined,
-      undefined,
-      10,
-      0,
-    );
-
-    expect(result.length).toEqual(1);
-
-    expect(result[0].id).toEqual(trace.id);
-    expect(result[0].projectId).toEqual(trace.project_id);
-    expect(result[0].name).toEqual(trace.name);
-    expect(result[0].timestamp).toEqual(new Date(trace.timestamp));
-    expect(result[0].tags).toEqual(trace.tags);
-    expect(result[0].bookmarked).toEqual(trace.bookmarked);
-    expect(result[0].release).toEqual(trace.release);
-    expect(result[0].version).toEqual(trace.version);
-    expect(result[0].userId).toEqual(trace.user_id);
-    expect(result[0].sessionId).toEqual(trace.session_id);
-    expect(result[0].public).toEqual(trace.public);
-    expect(result[0].tags).toEqual(trace.tags);
-  });
-
-  it("should return a trace joined with an observation", async () => {
-    const traceId = v4();
-
-    const trace = {
-      id: traceId,
-      project_id: projectId,
-      session_id: v4(),
-      timestamp: Date.now(),
-      metadata: {},
-      public: false,
-      bookmarked: false,
-      name: "Test Trace",
-      tags: ["tag-alpha", "tag-beta"],
-      release: "alpha",
-      version: "1.0.0",
-      user_id: "user-id-1",
-      created_at: Date.now(),
-      updated_at: Date.now(),
-      event_ts: Date.now(),
-      is_deleted: 0,
-    };
-
-    const observation = {
-      id: v4(),
-      project_id: projectId,
-      trace_id: traceId,
-      prompt_id: v4(),
-      type: "GENERATION",
-      created_at: Date.now(),
-      updated_at: Date.now(),
-      event_ts: Date.now(),
-      is_deleted: 0,
-      metadata: {
-        some: "metadata",
-      },
-      start_time: Date.now(),
-      provided_usage_details: {},
-      provided_cost_details: {},
-      usage_details: { input: 100, output: 200 },
-      cost_details: { input: 100, output: 200 },
-      name: "Test Observation",
-      level: "WARNING",
-      input: "some llm input",
-      output: "some llm output",
-      version: "1.2.0",
-      parent_observation_id: null,
-      status_message: "this is an error",
-      provided_model_name: "anthropic",
-      internal_model_id: "some-model-id",
-      model_parameters: JSON.stringify({
-        model: "test-model",
-        model_version: "1",
-        model_parameters: {},
-      }),
-      total_cost: 100,
-      prompt_name: "Test Prompt",
-      prompt_version: 1,
-      end_time: Date.now(),
-      completion_start_time: Date.now(),
-    };
-
-    await createTraces([trace]);
-    await createObservation(observation);
-
-    const result = await getTracesTable(
-      projectId,
-      [],
-      undefined,
-      undefined,
-      10,
-      0,
-    );
-
-    expect(result.length).toEqual(1);
-
-    expect(result[0].id).toEqual(trace.id);
-    expect(result[0].projectId).toEqual(trace.project_id);
-    expect(result[0].name).toEqual(trace.name);
-    expect(result[0].timestamp).toEqual(new Date(trace.timestamp));
-    expect(result[0].tags).toEqual(trace.tags);
-    expect(result[0].bookmarked).toEqual(trace.bookmarked);
-    expect(result[0].release).toEqual(trace.release);
-    expect(result[0].version).toEqual(trace.version);
-    expect(result[0].userId).toEqual(trace.user_id);
-    expect(result[0].sessionId).toEqual(trace.session_id);
-    expect(result[0].public).toEqual(trace.public);
-    expect(result[0].observationCount).toEqual(1);
-    expect(result[0].costDetails).toEqual({
-      input: 100,
-      output: 200,
-    });
-  });
-});

web/src/__tests__/server/repositories/clickhouse-helpers.ts

@@ -6,7 +6,7 @@ import {
 } from "@langfuse/shared/src/server";
 
 export const createTraces = async (trace: TraceRecordInsertType[]) => {
-  return await clickhouseClient.insert({
+  return await clickhouseClient().insert({
     table: "traces",
     format: "JSONEachRow",
     values: trace,
@@ -16,7 +16,7 @@ export const createTraces = async (trace: TraceRecordInsertType[]) => {
 export const createObservation = async (
   observation: ObservationRecordInsertType,
 ) => {
-  return await clickhouseClient.insert({
+  return await clickhouseClient().insert({
     table: "observations",
     format: "JSONEachRow",
     values: [observation],
@@ -26,7 +26,7 @@ export const createObservation = async (
 export const createObservations = async (
   observations: ObservationRecordInsertType[],
 ) => {
-  return await clickhouseClient.insert({
+  return await clickhouseClient().insert({
     table: "observations",
     format: "JSONEachRow",
     values: observations,
@@ -34,7 +34,7 @@ export const createObservations = async (
 };
 
 export const createScores = async (scores: ScoreRecordInsertType[]) => {
-  return await clickhouseClient.insert({
+  return await clickhouseClient().insert({
     table: "scores",
     format: "JSONEachRow",
     values: scores,

web/src/__tests__/test-utils.ts

@@ -1,6 +1,11 @@
 import { env } from "@/src/env.mjs";
 import { prisma } from "@langfuse/shared/src/db";
-import { clickhouseClient } from "@langfuse/shared/src/server";
+import {
+  clickhouseClient,
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server";
+import { v4 } from "uuid";
 import { type z } from "zod";
 
 export const pruneDatabase = async () => {
@@ -27,18 +32,21 @@ export const pruneDatabase = async () => {
     throw new Error("You cannot prune clickhouse unless running on localhost.");
   }
 
-  await clickhouseClient.command({
+  await clickhouseClient().command({
     query: "TRUNCATE TABLE IF EXISTS observations",
   });
-  await clickhouseClient.command({
+  await clickhouseClient().command({
     query: "TRUNCATE TABLE IF EXISTS scores",
   });
-  await clickhouseClient.command({
+  await clickhouseClient().command({
     query: "TRUNCATE TABLE IF EXISTS traces",
   });
 };
 
-function createBasicAuthHeader(username: string, password: string): string {
+export function createBasicAuthHeader(
+  username: string,
+  password: string,
+): string {
   const base64Credentials = Buffer.from(`${username}:${password}`).toString(
     "base64",
   );
@@ -130,3 +138,35 @@ export async function makeZodVerifiedAPICallSilent<T extends z.ZodTypeAny>(
 
   return { body: resBody, status };
 }
+
+export const createOrgProjectAndApiKey = async () => {
+  const projectId = v4();
+  const org = await prisma.organization.create({
+    data: {
+      id: v4(),
+      name: v4(),
+    },
+  });
+  await prisma.project.create({
+    data: {
+      id: projectId,
+      name: v4(),
+      orgId: org.id,
+    },
+  });
+  const publicKey = v4();
+  const secretKey = v4();
+
+  const auth = createBasicAuthHeader(publicKey, secretKey);
+  await prisma.apiKey.create({
+    data: {
+      id: v4(),
+      projectId: projectId,
+      publicKey: publicKey,
+      hashedSecretKey: await hashSecretKey(secretKey),
+      displaySecretKey: getDisplaySecretKey(secretKey),
+    },
+  });
+
+  return { projectId, publicKey, secretKey, auth };
+};

web/src/__tests__/traces.servertest.ts

@@ -19,36 +19,6 @@ describe("/api/public/traces API Endpoint", () => {
   beforeEach(async () => await pruneDatabase());
   afterEach(async () => await pruneDatabase());
 
-  it("should create and get a trace via /traces", async () => {
-    await pruneDatabase();
-
-    const traceCreate = await makeZodVerifiedAPICall(
-      PostTracesV1Response,
-      "POST",
-      "/api/public/traces",
-      {
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    );
-
-    const trace = await makeZodVerifiedAPICall(
-      GetTraceV1Response,
-      "GET",
-      "/api/public/traces/" + traceCreate.body.id,
-    );
-
-    expect(trace.body.name).toBe("trace-name");
-    expect(trace.body.release).toBe("1.0.0");
-    expect(trace.body.externalId).toBeNull();
-    expect(trace.body.version).toBe("2.0.0");
-    expect(trace.body.projectId).toBe("7a88fb47-b4e2-43b8-a06c-a5ce950dc53a");
-  });
-
   it("should upsert second trace", async () => {
     await pruneDatabase();
 

web/src/__tests__/usage.servertest.ts

@@ -53,6 +53,15 @@ describe("Token Count Functions", () => {
       expect(result).toBeUndefined();
     });
 
+    it("should return token count for strings with special characters", () => {
+      const result = tokenCount({
+        model: generateModel("gpt-4-1106-preview", "openai"),
+        text: "Hello <|endoftext|> World!",
+      });
+      expect(result).toBeDefined();
+      expect(result).toBeGreaterThan(0);
+    });
+
     it("check extensive openai chat message", () => {
       const result = tokenCount({
         model: generateModel("gpt-3.5-turbo", "openai"),
@@ -130,6 +139,7 @@ describe("Token Count Functions", () => {
       });
       expect(result).toBeUndefined();
     });
+
     it("should return undefined for undefined text input", () => {
       const result = tokenCount({
         model: generateModel("gpt-4", "openai"),

web/src/components/layouts/layout.tsx

@@ -57,6 +57,7 @@ const unauthenticatedPaths: string[] = [
   "/auth/sign-in",
   "/auth/sign-up",
   "/auth/error",
+  "/auth/hf-spaces",
 ];
 // auth or unauthed
 const publishablePaths: string[] = [

web/src/components/sidebar-notifications.tsx

@@ -23,11 +23,39 @@ type SidebarNotification = {
 };
 
 const notifications: SidebarNotification[] = [
+  {
+    id: "lw2-5",
+    title: "Launch Week 2 – Day 5",
+    description: "Introducing Prompt Experiments to test prompts on datasets",
+    link: "https://langfuse.com/changelog/2024-11-22-prompt-experimentation",
+    linkTitle: "Changelog",
+  },
+  {
+    id: "lw2-4",
+    title: "Launch Week 2 – Day 4",
+    description: "All new docs for datasets, experiments, and evaluations",
+    link: "https://langfuse.com/changelog/2024-11-21-all-new-datasets-and-evals-documentation",
+    linkTitle: "Changelog",
+  },
+  {
+    id: "lw2-3",
+    title: "Launch Week 2 – Day 3",
+    description:
+      "Full multi-modal support including images, audio, and attachments",
+    link: "https://langfuse.com/changelog/2024-11-20-full-multi-modal-images-audio-attachments",
+    linkTitle: "Changelog",
+  },
+  {
+    id: "lw2-2",
+    title: "Launch Week 2 – Day 2",
+    description: "LLM-as-a-Judge Evaluators for Dataset Experiments",
+    link: "https://langfuse.com/changelog/2024-11-19-llm-as-a-judge-for-datasets",
+    linkTitle: "Changelog",
+  },
   {
     id: "lw2-1",
     title: "Launch Week 2 – Day 1",
-    description:
-      "New side-by-side comparison view for dataset experiment runs.",
+    description: "New side-by-side comparison view for dataset experiment runs",
     link: "https://langfuse.com/changelog/2024-11-18-dataset-runs-comparison-view",
     linkTitle: "Changelog",
   },
@@ -35,7 +63,7 @@ const notifications: SidebarNotification[] = [
     id: "github-star",
     title: "Star Langfuse",
     description:
-      "See the latest releases and help grow the community on GitHub.",
+      "See the latest releases and help grow the community on GitHub",
     link: "https://github.com/langfuse/langfuse",
     linkContent: (
       // eslint-disable-next-line @next/next/no-img-element
@@ -84,7 +112,7 @@ export function SidebarNotifications() {
   const currentNotification = notifications[currentNotificationIndex];
 
   return (
-    <Card className="relative max-h-44 overflow-hidden rounded-md bg-opacity-50 shadow-none group-data-[collapsible=icon]:hidden">
+    <Card className="relative max-h-60 overflow-hidden rounded-md bg-opacity-50 shadow-none group-data-[collapsible=icon]:hidden">
       <Button
         variant="ghost"
         size="sm"
@@ -127,7 +155,7 @@ export function SidebarNotifications() {
                 });
               }}
             >
-              {currentNotification.linkTitle ?? "Learn more"}
+              {currentNotification.linkTitle ?? "Learn more"} &rarr;
             </Link>
           </Button>
         )}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.91.0";
+export const VERSION = "v2.95.1";

web/src/ee/features/evals/components/evaluator-detail.tsx

@@ -11,7 +11,6 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { useState } from "react";
-import { Trash } from "lucide-react";
 import TableLink from "@/src/components/table/table-link";
 import EvalLogTable from "@/src/ee/features/evals/components/eval-log";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -20,12 +19,23 @@ import { TableWithMetadataWrapper } from "@/src/components/table/TableWithMetada
 import { StatusBadge } from "@/src/components/layouts/status-badge";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { CardDescription } from "@/src/components/ui/card";
+import { EvaluatorStatus } from "@/src/ee/features/evals/types";
+import { Switch } from "@/src/components/ui/switch";
+import { Edit } from "lucide-react";
+import {
+  Dialog,
+  DialogContent,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
 
 export const EvaluatorDetail = () => {
   const router = useRouter();
   const projectId = router.query.projectId as string;
   const evaluatorId = router.query.evaluatorId as string;
 
+  const [isEditOpen, setIsEditOpen] = useState(false);
+
   // get the current template by id
   const evaluator = api.evals.configById.useQuery({
     projectId: projectId,
@@ -63,79 +73,102 @@ export const EvaluatorDetail = () => {
 
   return (
     <FullScreenPage>
-      <>
-        <Header
-          title={evaluator.data?.id ?? "Loading..."}
-          breadcrumb={[
-            {
-              name: "Evaluators",
-              href: `/project/${router.query.projectId as string}/evals`,
-            },
-            { name: evaluator.data?.id },
-          ]}
-          actionButtons={
-            <>
-              <DeactivateEvaluator
-                projectId={projectId}
-                evaluator={evaluator.data ?? undefined}
-                isLoading={evaluator.isLoading}
+      <Header
+        title={evaluator.data ? `Evaluator ${evaluator.data.id}` : "Loading..."}
+        breadcrumb={[
+          {
+            name: "Evaluators",
+            href: `/project/${router.query.projectId as string}/evals`,
+          },
+          { name: evaluator.data?.id },
+        ]}
+        actionButtons={
+          <>
+            <StatusBadge
+              type={evaluator.data?.status.toLowerCase()}
+              isLive
+              className="max-h-8"
+            />
+            <DeactivateEvaluator
+              projectId={projectId}
+              evaluator={evaluator.data ?? undefined}
+              isLoading={evaluator.isLoading}
+            />
+            {evaluator.data && (
+              <DetailPageNav
+                key="nav"
+                currentId={encodeURIComponent(evaluator.data.id)}
+                path={(entry) =>
+                  `/project/${projectId}/evals/${encodeURIComponent(entry.id)}`
+                }
+                listKey="evals"
               />
-              {evaluator.data && (
-                <DetailPageNav
-                  key="nav"
-                  currentId={encodeURIComponent(evaluator.data.id)}
-                  path={(entry) =>
-                    `/project/${projectId}/evals/${encodeURIComponent(entry.id)}`
-                  }
-                  listKey="evals"
+            )}
+          </>
+        }
+      />
+      {existingEvaluator && (
+        <TableWithMetadataWrapper
+          tableComponent={
+            <EvalLogTable
+              projectId={projectId}
+              jobConfigurationId={existingEvaluator.id}
+            />
+          }
+          cardTitleChildren={
+            <div className="flex w-full flex-row items-center justify-between">
+              <span>Evaluator configuration</span>
+              <Dialog open={isEditOpen} onOpenChange={setIsEditOpen}>
+                <DialogTrigger asChild>
+                  <Button variant="outline" className="flex items-center gap-2">
+                    <Edit className="h-4 w-4" />
+                  </Button>
+                </DialogTrigger>
+                <DialogContent className="max-w-screen-xl">
+                  <DialogTitle>Edit Evaluator</DialogTitle>
+                  <div className="max-h-[80vh] overflow-y-auto">
+                    <EvaluatorForm
+                      key={existingEvaluator.id}
+                      projectId={projectId}
+                      evalTemplates={allTemplates.data?.templates}
+                      existingEvaluator={existingEvaluator}
+                      shouldWrapVariables={true}
+                      mode="edit"
+                      onFormSuccess={() => {
+                        setIsEditOpen(false);
+                        // Force a reload as the form state is not properly updated
+                        void router.reload();
+                      }}
+                    />
+                  </div>
+                </DialogContent>
+              </Dialog>
+            </div>
+          }
+          cardContentChildren={
+            <>
+              <CardDescription className="flex items-center justify-between text-sm">
+                <span className="text-sm font-medium">Eval Template</span>
+                <TableLink
+                  path={`/project/${projectId}/evals/templates/${existingEvaluator.evalTemplateId}`}
+                  value={`${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})`}
+                  className="flex min-h-6 items-center"
                 />
-              )}
+              </CardDescription>
+              <div className="flex w-full flex-col items-start justify-between space-y-2 pb-4">
+                <EvaluatorForm
+                  key={existingEvaluator.id}
+                  projectId={projectId}
+                  evalTemplates={allTemplates.data?.templates}
+                  existingEvaluator={existingEvaluator}
+                  disabled={true}
+                  shouldWrapVariables={true}
+                />
+              </div>
             </>
           }
         />
-        {existingEvaluator && (
-          <TableWithMetadataWrapper
-            tableComponent={
-              <EvalLogTable
-                projectId={projectId}
-                jobConfigurationId={existingEvaluator.id}
-              />
-            }
-            cardTitleChildren={
-              <div className="flex w-full flex-row items-center justify-between">
-                <span>Evaluator</span>
-                <StatusBadge
-                  type={evaluator.data?.status.toLowerCase()}
-                  isLive
-                  className="max-h-8"
-                />
-              </div>
-            }
-            cardContentChildren={
-              <>
-                <CardDescription className="flex items-center justify-between text-sm">
-                  <span className="text-sm font-medium">Eval Template</span>
-                  <TableLink
-                    path={`/project/${projectId}/evals/templates/${existingEvaluator.evalTemplateId}`}
-                    value={`${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})`}
-                    className="flex min-h-6 items-center"
-                  />
-                </CardDescription>
-                <div className="flex w-full flex-col items-start justify-between space-y-2 pb-4">
-                  <EvaluatorForm
-                    key={existingEvaluator.id}
-                    projectId={projectId}
-                    evalTemplates={allTemplates.data?.templates}
-                    existingEvaluator={existingEvaluator}
-                    disabled={true}
-                    shouldWrapVariables={true}
-                  />
-                </div>
-              </>
-            }
-          />
-        )}
-      </>
+      )}
     </FullScreenPage>
   );
 };
@@ -143,7 +176,6 @@ export const EvaluatorDetail = () => {
 export function DeactivateEvaluator({
   projectId,
   evaluator,
-  isLoading,
 }: {
   projectId: string;
   evaluator?: RouterOutputs["evals"]["configById"];
@@ -153,6 +185,7 @@ export function DeactivateEvaluator({
   const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
   const [isOpen, setIsOpen] = useState(false);
   const capture = usePostHogClientCapture();
+  const isActive = evaluator?.status === EvaluatorStatus.ACTIVE;
 
   const mutEvaluator = api.evals.updateEvalJob.useMutation({
     onSuccess: () => {
@@ -165,41 +198,50 @@ export function DeactivateEvaluator({
       console.error("Project ID is missing");
       return;
     }
+
+    const prevStatus = evaluator?.status;
+
     mutEvaluator.mutateAsync({
       projectId,
       evalConfigId: evaluator?.id ?? "",
-      updatedStatus: "INACTIVE",
+      config: {
+        status: isActive ? EvaluatorStatus.INACTIVE : EvaluatorStatus.ACTIVE,
+      },
     });
-    capture("eval_config:delete");
+    capture(
+      prevStatus === EvaluatorStatus.ACTIVE
+        ? "eval_config:deactivate"
+        : "eval_config:activate",
+    );
     setIsOpen(false);
   };
 
   return (
     <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
       <PopoverTrigger asChild>
-        <Button
-          variant="outline"
-          size={"icon"}
-          disabled={!hasAccess || evaluator?.status !== "ACTIVE"}
-          loading={isLoading}
-        >
-          <Trash className="h-5 w-5" />
-        </Button>
+        <div className="flex items-center">
+          <Switch
+            disabled={!hasAccess}
+            checked={isActive}
+            className={isActive ? "data-[state=checked]:bg-dark-green" : ""}
+          />
+        </div>
       </PopoverTrigger>
       <PopoverContent>
         <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
         <p className="mb-3 text-sm">
-          This action permanently deactivates the evaluator. No more traces will
-          be evaluated based on this evaluator.
+          {evaluator?.status === "ACTIVE"
+            ? "This action will deactivate the evaluator. No more traces will be evaluated based on this evaluator."
+            : "This action will activate the evaluator. New traces will be evaluated based on this evaluator."}
         </p>
         <div className="flex justify-end space-x-4">
           <Button
             type="button"
-            variant="destructive"
+            variant={evaluator?.status === "ACTIVE" ? "destructive" : "default"}
             loading={mutEvaluator.isLoading}
             onClick={onClick}
           >
-            Deactivate evaluator
+            {evaluator?.status === "ACTIVE" ? "Deactivate" : "Activate"}
           </Button>
         </div>
       </PopoverContent>

web/src/ee/features/evals/components/evaluator-form.tsx

@@ -69,9 +69,9 @@ import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"
 const formSchema = z.object({
   scoreName: z.string(),
   target: z.string(),
-  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
   mapping: z.array(wipVariableMapping),
-  sampling: z.coerce.number().gte(0).lte(1),
+  sampling: z.coerce.number().gt(0).lte(1),
   delay: z.coerce.number().optional().default(10),
 });
 
@@ -87,6 +87,7 @@ export const EvaluatorForm = (props: {
   disabled?: boolean;
   existingEvaluator?: JobConfiguration & { evalTemplate: EvalTemplate };
   onFormSuccess?: () => void;
+  mode?: "create" | "edit";
   shouldWrapVariables?: boolean;
 }) => {
   const [open, setOpen] = useState(false);
@@ -131,7 +132,7 @@ export const EvaluatorForm = (props: {
           <Popover open={open} onOpenChange={setOpen}>
             <PopoverTrigger asChild>
               <Button
-                disabled={props.disabled}
+                disabled={props.disabled || props.mode === "edit"}
                 variant="outline"
                 role="combobox"
                 aria-expanded={open}
@@ -193,7 +194,11 @@ export const EvaluatorForm = (props: {
           <Popover>
             <PopoverTrigger asChild>
               <Button
-                disabled={props.disabled || !selectedTemplateName}
+                disabled={
+                  props.disabled ||
+                  !selectedTemplateName ||
+                  props.mode === "edit"
+                }
                 variant="outline"
                 role="combobox"
                 className="w-1/3 justify-between px-2 font-normal"
@@ -276,6 +281,7 @@ export const EvaluatorForm = (props: {
           }
           onFormSuccess={props.onFormSuccess}
           shouldWrapVariables={props.shouldWrapVariables}
+          mode={props.mode}
         />
       ) : null}
     </>
@@ -289,6 +295,7 @@ export const InnerEvalConfigForm = (props: {
   existingEvaluator?: JobConfiguration;
   onFormSuccess?: () => void;
   shouldWrapVariables?: boolean;
+  mode?: "create" | "edit";
 }) => {
   const [formError, setFormError] = useState<string | null>(null);
   const capture = usePostHogClientCapture();
@@ -393,6 +400,10 @@ export const InnerEvalConfigForm = (props: {
     onSuccess: () => utils.models.invalidate(),
     onError: (error) => setFormError(error.message),
   });
+  const updateJobMutation = api.evals.updateEvalJob.useMutation({
+    onSuccess: () => utils.evals.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
   const [availableVariables, setAvailableVariables] = useState<
     typeof availableTraceEvalVariables | typeof availableDatasetEvalVariables
   >(
@@ -402,7 +413,11 @@ export const InnerEvalConfigForm = (props: {
   );
 
   function onSubmit(values: z.infer<typeof formSchema>) {
-    capture("eval_config:new_form_submit");
+    capture(
+      props.mode === "edit"
+        ? "eval_config:update"
+        : "eval_config:new_form_submit",
+    );
 
     const validatedFilter = z.array(singleFilter).safeParse(values.filter);
 
@@ -427,21 +442,42 @@ export const InnerEvalConfigForm = (props: {
       return;
     }
 
-    createJobMutation
-      .mutateAsync({
-        projectId: props.projectId,
-        evalTemplateId: props.evalTemplate.id,
-        scoreName: values.scoreName,
-        target: values.target,
-        filter: validatedFilter.data,
-        mapping: validatedVarMapping.data,
-        sampling: values.sampling,
-        delay: values.delay * 1000, // multiply by 1k to convert to ms
-      })
+    const delay = values.delay * 1000; // convert to ms
+    const sampling = values.sampling;
+    const mapping = validatedVarMapping.data;
+    const filter = validatedFilter.data;
+    const scoreName = values.scoreName;
+
+    (props.mode === "edit" && props.existingEvaluator
+      ? updateJobMutation.mutateAsync({
+          projectId: props.projectId,
+          evalConfigId: props.existingEvaluator.id,
+          config: {
+            delay,
+            filter,
+            variableMapping: mapping,
+            sampling,
+            scoreName,
+          },
+        })
+      : createJobMutation.mutateAsync({
+          projectId: props.projectId,
+          target: values.target,
+          evalTemplateId: props.evalTemplate.id,
+          scoreName,
+          filter,
+          mapping,
+          sampling,
+          delay,
+        })
+    )
       .then(() => {
-        props.onFormSuccess?.();
         form.reset();
-        void router.push(`/project/${props.projectId}/evals`);
+        props.onFormSuccess?.();
+
+        if (props.mode !== "edit") {
+          void router.push(`/project/${props.projectId}/evals`);
+        }
       })
       .catch((error) => {
         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
@@ -507,10 +543,16 @@ export const InnerEvalConfigForm = (props: {
                       }}
                     >
                       <TabsList>
-                        <TabsTrigger value="trace" disabled={props.disabled}>
+                        <TabsTrigger
+                          value="trace"
+                          disabled={props.disabled || props.mode === "edit"}
+                        >
                           Trace
                         </TabsTrigger>
-                        <TabsTrigger value="dataset" disabled={props.disabled}>
+                        <TabsTrigger
+                          value="dataset"
+                          disabled={props.disabled || props.mode === "edit"}
+                        >
                           Dataset
                         </TabsTrigger>
                       </TabsList>
@@ -825,7 +867,7 @@ export const InnerEvalConfigForm = (props: {
         {!props.disabled ? (
           <Button
             type="submit"
-            loading={createJobMutation.isLoading}
+            loading={createJobMutation.isLoading || updateJobMutation.isLoading}
             className="mt-3"
           >
             Save

web/src/ee/features/evals/components/template-form.tsx

@@ -15,7 +15,7 @@ import {
 import { Textarea } from "@/src/components/ui/textarea";
 import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
-import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
+import { extractVariables, getIsCharOrUnderscore } from "@langfuse/shared";
 import router from "next/router";
 import { type EvalTemplate } from "@langfuse/shared";
 import { ModelParameters } from "@/src/components/ModelParameters";

web/src/ee/features/evals/server/addDatasetRunItemsToEvalQueue.ts

@@ -1,7 +1,10 @@
 import { env } from "@/src/env.mjs";
-import { DatasetRunItemUpsertQueue } from "../../../../../../packages/shared/dist/src/server/redis/datasetRunItemUpsert";
 import { randomUUID } from "crypto";
-import { QueueJobs, redis } from "@langfuse/shared/src/server";
+import {
+  QueueJobs,
+  DatasetRunItemUpsertQueue,
+  redis,
+} from "@langfuse/shared/src/server";
 
 export const addDatasetRunItemsToEvalQueue = async ({
   projectId,
@@ -18,30 +21,17 @@ export const addDatasetRunItemsToEvalQueue = async ({
     const queue = DatasetRunItemUpsertQueue.getInstance();
 
     if (queue) {
-      await queue.add(
-        QueueJobs.DatasetRunItemUpsert,
-        {
-          payload: {
-            projectId,
-            datasetItemId: datasetItemId,
-            traceId,
-            observationId: observationId ?? undefined,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.DatasetRunItemUpsert as const,
+      await queue.add(QueueJobs.DatasetRunItemUpsert, {
+        payload: {
+          projectId,
+          datasetItemId: datasetItemId,
+          traceId,
+          observationId: observationId ?? undefined,
         },
-        {
-          attempts: 3, // retry 3 times
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-          delay: 10000, // 10 seconds
-          removeOnComplete: true,
-          removeOnFail: 1_000,
-        },
-      );
+        id: randomUUID(),
+        timestamp: new Date(),
+        name: QueueJobs.DatasetRunItemUpsert as const,
+      });
     }
   }
 };

web/src/ee/features/evals/server/router.ts

@@ -12,6 +12,9 @@ import {
   singleFilter,
   variableMapping,
   ChatMessageRole,
+  type JobConfiguration,
+  JobConfigState,
+  JobType,
   Prisma,
 } from "@langfuse/shared";
 import { decrypt } from "@langfuse/shared/encryption";
@@ -23,6 +26,47 @@ import {
 } from "@langfuse/shared/src/server";
 import { TRPCError } from "@trpc/server";
 import { EvalReferencedEvaluators } from "@/src/ee/features/evals/types";
+import { EvaluatorStatus } from "../types";
+import { traceException } from "@langfuse/shared/src/server";
+
+const APIEvaluatorSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  evalTemplateId: z.string(),
+  scoreName: z.string(),
+  targetObject: z.string(),
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
+  variableMapping: z.array(variableMapping),
+  sampling: z.instanceof(Prisma.Decimal),
+  delay: z.number(),
+  status: z.nativeEnum(JobConfigState),
+  jobType: z.nativeEnum(JobType),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+type APIEvaluator = z.infer<typeof APIEvaluatorSchema>;
+
+/**
+ * Use this function when pulling a list of evaluators from the database before using in the application to ensure type safety.
+ * All evaluators are expected to pass the validation. If an evaluator fails validation, it will be logged to Otel.
+ * @param evaluators
+ * @returns list of validated evaluators
+ */
+const filterAndValidateDbEvaluatorList = (
+  evaluators: JobConfiguration[],
+  onParseError?: (error: z.ZodError) => void,
+): APIEvaluator[] =>
+  evaluators.reduce((acc, ts) => {
+    const result = APIEvaluatorSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      console.error("Evaluator parsing error: ", result.error);
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as APIEvaluator[]);
 
 export const CreateEvalTemplate = z.object({
   name: z.string().min(1),
@@ -42,6 +86,26 @@ export const CreateEvalTemplate = z.object({
     .default(EvalReferencedEvaluators.PERSIST),
 });
 
+const CreateEvalJobSchema = z.object({
+  projectId: z.string(),
+  evalTemplateId: z.string(),
+  scoreName: z.string().min(1),
+  target: z.string(),
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
+  mapping: z.array(variableMapping),
+  sampling: z.number().gt(0).lte(1),
+  delay: z.number().gte(0).default(DEFAULT_TRACE_JOB_DELAY), // 10 seconds default
+});
+
+const UpdateEvalJobSchema = z.object({
+  scoreName: z.string().min(1).optional(),
+  filter: z.array(singleFilter).optional(),
+  variableMapping: z.array(variableMapping).optional(),
+  sampling: z.number().gt(0).lte(1).optional(),
+  delay: z.number().gte(0).optional(),
+  status: z.nativeEnum(EvaluatorStatus).optional(),
+});
+
 export const evalRouter = createTRPCRouter({
   allConfigs: protectedProjectProcedure
     .input(
@@ -314,6 +378,31 @@ export const evalRouter = createTRPCRouter({
       }
     }),
 
+  jobConfigsByTarget: protectedProjectProcedure
+    .input(z.object({ projectId: z.string(), targetObject: z.string() }))
+    .query(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "model-based-evaluations",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalJob:read",
+      });
+
+      const evaluators = await ctx.prisma.jobConfiguration.findMany({
+        where: {
+          projectId: input.projectId,
+          targetObject: input.targetObject,
+          status: "ACTIVE",
+        },
+      });
+
+      return filterAndValidateDbEvaluatorList(evaluators, traceException);
+    }),
+
   jobConfigsByTemplateName: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), evalTemplateName: z.string() }))
     .query(async ({ input, ctx }) => {
@@ -360,18 +449,7 @@ export const evalRouter = createTRPCRouter({
     }),
 
   createJob: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        evalTemplateId: z.string(),
-        scoreName: z.string().min(1),
-        target: z.string(),
-        filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
-        mapping: z.array(variableMapping),
-        sampling: z.number().gte(0).lte(1),
-        delay: z.number().gte(0).default(DEFAULT_TRACE_JOB_DELAY), // 10 seconds default
-      }),
-    )
+    .input(CreateEvalJobSchema)
     .mutation(async ({ input, ctx }) => {
       try {
         throwIfNoEntitlement({
@@ -453,29 +531,31 @@ export const evalRouter = createTRPCRouter({
 
       // Make a test structured output call to validate the LLM key
       try {
-        await fetchLLMCompletion({
-          streaming: false,
-          apiKey: decrypt(parsedKey.data.secretKey), // decrypt the secret key
-          baseURL: parsedKey.data.baseURL ?? undefined,
-          messages: [
-            {
-              role: ChatMessageRole.System,
-              content: "You are an expert at evaluating LLM outputs.",
+        (
+          await fetchLLMCompletion({
+            streaming: false,
+            apiKey: decrypt(parsedKey.data.secretKey), // decrypt the secret key
+            baseURL: parsedKey.data.baseURL ?? undefined,
+            messages: [
+              {
+                role: ChatMessageRole.System,
+                content: "You are an expert at evaluating LLM outputs.",
+              },
+              { role: ChatMessageRole.User, content: input.prompt },
+            ],
+            modelParams: {
+              provider: input.provider,
+              model: input.model,
+              adapter: parsedKey.data.adapter,
+              ...input.modelParams,
             },
-            { role: ChatMessageRole.User, content: input.prompt },
-          ],
-          modelParams: {
-            provider: input.provider,
-            model: input.model,
-            adapter: parsedKey.data.adapter,
-            ...input.modelParams,
-          },
-          structuredOutputSchema: z.object({
-            score: z.string(),
-            reasoning: z.string(),
-          }),
-          config: parsedKey.data.config,
-        });
+            structuredOutputSchema: z.object({
+              score: z.string(),
+              reasoning: z.string(),
+            }),
+            config: parsedKey.data.config,
+          })
+        ).completion;
       } catch (err) {
         logger.error(err);
 
@@ -543,7 +623,7 @@ export const evalRouter = createTRPCRouter({
       z.object({
         projectId: z.string(),
         evalConfigId: z.string(),
-        updatedStatus: z.enum(["ACTIVE", "INACTIVE"]),
+        config: UpdateEvalJobSchema,
       }),
     )
     .mutation(async ({ input, ctx }) => {
@@ -563,9 +643,7 @@ export const evalRouter = createTRPCRouter({
           id: input.evalConfigId,
           projectId: input.projectId,
         },
-        data: {
-          status: input.updatedStatus,
-        },
+        data: input.config,
       });
 
       await auditLog({

web/src/ee/features/evals/types.ts

@@ -1,4 +1,11 @@
+import { z } from "zod";
+import { JobConfigState } from "@langfuse/shared";
+
 export enum EvalReferencedEvaluators {
   UPDATE = "update",
   PERSIST = "persist",
 }
+
+export const EvaluatorStatus = JobConfigState;
+export const EvaluatorStatusSchema = z.nativeEnum(EvaluatorStatus);
+export type EvaluatorStatusType = z.infer<typeof EvaluatorStatusSchema>;

web/src/ee/features/experiments/components/CreateExperimentsForm.tsx

@@ -0,0 +1,742 @@
+import React, { useEffect, useMemo, useState } from "react";
+import { Button } from "@/src/components/ui/button";
+import {
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import { Form } from "@/src/components/ui/form";
+import { Textarea } from "@/src/components/ui/textarea";
+import { ModelParameters } from "@/src/components/ModelParameters";
+import {
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandList,
+  Command,
+  CommandItem,
+} from "@/src/components/ui/command";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { z, type ZodSchema } from "zod";
+import { cn } from "@/src/utils/tailwind";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import {
+  ChevronDown,
+  CheckIcon,
+  Info,
+  CircleCheck,
+  Loader2,
+} from "lucide-react";
+import { api } from "@/src/utils/api";
+import {
+  Card,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/src/components/ui/card";
+import { showErrorToast } from "@/src/features/notifications/showErrorToast";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
+import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
+import {
+  type ColumnDefinition,
+  datasetCol,
+  extractVariables,
+  type FilterCondition,
+  stringOptionsFilter,
+  ZodModelConfig,
+} from "@langfuse/shared";
+import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { Input } from "@/src/components/ui/input";
+import { EvaluatorStatus } from "@/src/ee/features/evals/types";
+
+const CreateExperimentData = z.object({
+  name: z.string().min(1, "Please enter a name").optional(),
+  promptId: z.string().min(1, "Please select a prompt"),
+  datasetId: z.string().min(1, "Please select a dataset"),
+  description: z.string().max(1000).optional(),
+  modelConfig: z.object({
+    provider: z.string().min(1, "Please select a provider"),
+    model: z.string().min(1, "Please select a model"),
+    modelParams: ZodModelConfig,
+  }),
+});
+
+export type CreateExperiment = z.infer<typeof CreateExperimentData>;
+
+const isDatasetTarget = <T extends ZodSchema>(
+  filters: FilterCondition[] | null,
+  condition: {
+    column: ColumnDefinition;
+    schema: T;
+    isValid: (filter: z.infer<T>) => boolean;
+  },
+): boolean => {
+  if (!filters) return true;
+
+  const { column, schema, isValid } = condition;
+  const datasetFilters = filters.filter(
+    (filter) =>
+      (filter.column === column.id || column.name) &&
+      schema.safeParse(filter).success,
+  );
+  return datasetFilters.every((filter): boolean => isValid(filter));
+};
+
+export const CreateExperimentsForm = ({
+  projectId,
+  setFormOpen,
+  defaultValues = {},
+  promptDefault,
+  handleExperimentSettled,
+  handleExperimentSuccess,
+}: {
+  projectId: string;
+  setFormOpen: (open: boolean) => void;
+  defaultValues?: Partial<CreateExperiment>;
+  promptDefault?: {
+    name: string;
+    version: number;
+  };
+  handleExperimentSuccess?: (data?: {
+    success: boolean;
+    datasetId: string;
+    runId: string;
+    runName: string;
+  }) => Promise<void>;
+  handleExperimentSettled?: (data?: {
+    success: boolean;
+    datasetId: string;
+    runId: string;
+    runName: string;
+  }) => Promise<void>;
+}) => {
+  const [open, setOpen] = useState(false);
+  const [evaluatorOptions, setEvaluatorOptions] = useState<
+    { key: string; value: string }[]
+  >([]);
+  const [selectedEvaluators, setSelectedEvaluators] = useState<
+    { key: string; value: string }[]
+  >([]);
+  const [selectedPromptName, setSelectedPromptName] = useState<string>(
+    promptDefault?.name ?? "",
+  );
+  const [selectedPromptVersion, setSelectedPromptVersion] = useState<
+    number | null
+  >(promptDefault?.version ?? null);
+
+  const {
+    modelParams,
+    updateModelParamValue,
+    setModelParamEnabled,
+    availableModels,
+    availableProviders,
+  } = useModelParams();
+
+  const form = useForm<CreateExperiment>({
+    resolver: zodResolver(CreateExperimentData),
+    defaultValues: {
+      promptId: "",
+      datasetId: "",
+      modelConfig: {},
+      ...defaultValues,
+    },
+  });
+
+  const hasExperimentWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "experiments:CUD",
+  });
+
+  const hasEvalReadAccess = useHasProjectAccess({
+    projectId,
+    scope: "evalJob:read",
+  });
+
+  const hasEvalWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "evalJob:CUD",
+  });
+
+  const promptMeta = api.prompts.allPromptMeta.useQuery({
+    projectId,
+  });
+
+  const datasets = api.datasets.allDatasetMeta.useQuery(
+    { projectId },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+      staleTime: Infinity,
+    },
+  );
+
+  const promptId = form.watch("promptId");
+  const datasetId = form.watch("datasetId");
+
+  const evaluators = api.evals.jobConfigsByTarget.useQuery(
+    { projectId, targetObject: "dataset" },
+    {
+      enabled: hasEvalReadAccess && !!datasetId,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+    },
+  );
+
+  const expectedColumns = useMemo(() => {
+    const prompt = promptMeta.data?.find((p) => p.id === promptId);
+    if (!prompt) return [];
+
+    return extractVariables(
+      prompt.type === PromptType.Text
+        ? (prompt?.prompt?.toString() ?? "")
+        : JSON.stringify(prompt?.prompt),
+    );
+  }, [promptId, promptMeta.data]);
+
+  useEffect(() => {
+    if (evaluators.data) {
+      const isValidFilter = (filter: z.infer<typeof stringOptionsFilter>) => {
+        const filterIncludesId = filter.value.includes(datasetId);
+        if (filter.operator === "any of") {
+          return filterIncludesId;
+        } else {
+          return !filterIncludesId;
+        }
+      };
+
+      const initialEvaluators = evaluators.data.reduce<
+        { key: string; value: string }[]
+      >((acc, evaluator) => {
+        if (
+          isDatasetTarget(evaluator.filter, {
+            column: datasetCol,
+            schema: stringOptionsFilter,
+            isValid: isValidFilter,
+          })
+        ) {
+          acc.push({
+            key: evaluator.id,
+            value: evaluator.scoreName,
+          });
+        }
+        return acc;
+      }, []);
+
+      setEvaluatorOptions(initialEvaluators);
+      setSelectedEvaluators(initialEvaluators);
+    }
+  }, [evaluators.data, datasetId]);
+
+  const validationResult = api.experiments.validateConfig.useQuery(
+    {
+      projectId,
+      promptId: promptId as string,
+      datasetId: datasetId as string,
+    },
+    {
+      enabled: Boolean(promptId && datasetId),
+    },
+  );
+
+  const experimentMutation = api.experiments.createExperiment.useMutation({
+    onSuccess: handleExperimentSuccess ?? (() => {}),
+    onError: (error) => {
+      showErrorToast(
+        error.message || "Failed to trigger experiment run",
+        "Please try again.",
+      );
+    },
+    onSettled: handleExperimentSettled ?? (() => {}),
+  });
+
+  const archiveEvaluatorMutation = api.evals.updateEvalJob.useMutation();
+
+  // Watch model config changes and update form
+  useEffect(() => {
+    form.setValue("modelConfig", {
+      provider: modelParams.provider.value,
+      model: modelParams.model.value,
+      modelParams: getFinalModelParams(modelParams),
+    });
+  }, [modelParams, form]);
+
+  const onSubmit = async (data: CreateExperiment) => {
+    const experiment = {
+      ...data,
+      projectId,
+    };
+    await experimentMutation.mutateAsync(experiment);
+    form.reset();
+    setFormOpen(false);
+  };
+
+  const handleOnValueChange = (
+    values: { key: string; value: string }[],
+    changedValueId?: string,
+  ) => {
+    if (!changedValueId) return;
+    const evaluator = evaluators.data?.find((e) => e.id === changedValueId);
+    if (!evaluator) return;
+
+    if (evaluator.status === "INACTIVE") {
+      const confirmed = window.confirm(
+        `Are you sure you want to activate "${evaluator.scoreName}"? You can always always archive the evaluator.`,
+      );
+      if (!confirmed) {
+        return;
+      }
+    } else {
+      const confirmed = window.confirm(
+        `Are you sure you want to archive "${evaluator.scoreName}"? You can always always re-activate the evaluator.`,
+      );
+      if (!confirmed) {
+        return;
+      }
+    }
+
+    archiveEvaluatorMutation.mutate({
+      projectId,
+      evalConfigId: changedValueId,
+      config: {
+        status:
+          evaluator.status === EvaluatorStatus.INACTIVE
+            ? EvaluatorStatus.ACTIVE
+            : EvaluatorStatus.INACTIVE,
+      },
+    });
+
+    setSelectedEvaluators(values);
+  };
+
+  const promptsByName = useMemo(
+    () =>
+      promptMeta.data?.reduce<
+        Record<string, Array<{ version: number; id: string }>>
+      >((acc, prompt) => {
+        if (!acc[prompt.name]) {
+          acc[prompt.name] = [];
+        }
+        acc[prompt.name].push({ version: prompt.version, id: prompt.id });
+        return acc;
+      }, {}),
+    [promptMeta.data],
+  );
+
+  if (!hasExperimentWriteAccess) {
+    return null;
+  }
+
+  if (
+    !promptMeta.data ||
+    !datasets.data ||
+    (hasEvalReadAccess && !!datasetId && !evaluators.data)
+  ) {
+    return <Skeleton className="min-h-[70dvh] w-full" />;
+  }
+
+  return (
+    <Form {...form}>
+      <form className="space-y-6" onSubmit={form.handleSubmit(onSubmit)}>
+        <FormField
+          control={form.control}
+          name="name"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Experiment name (optional)</FormLabel>
+              <FormControl>
+                <Input {...field} type="string" />
+              </FormControl>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+        <FormField
+          control={form.control}
+          name="description"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Description (optional)</FormLabel>
+              <FormControl>
+                <Textarea
+                  {...field}
+                  placeholder="Add description..."
+                  className="focus:outline-none focus:ring-0 focus-visible:ring-0 focus-visible:ring-offset-0 active:ring-0"
+                />
+              </FormControl>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+        <FormField
+          control={form.control}
+          name="promptId"
+          render={() => (
+            <FormItem>
+              <FormLabel>Prompt</FormLabel>
+              {/* FIX: I need the command list in the popover to be scrollable, currently it's not */}
+              <div className="mb-2 flex gap-2">
+                <Popover open={open} onOpenChange={setOpen}>
+                  <PopoverTrigger asChild>
+                    <Button
+                      variant="outline"
+                      role="combobox"
+                      aria-expanded={open}
+                      className="w-2/3 justify-between px-2 font-normal"
+                    >
+                      {selectedPromptName || "Select a prompt"}
+                      <ChevronDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+                    </Button>
+                  </PopoverTrigger>
+                  <PopoverContent
+                    className="w-[--radix-popover-trigger-width] overflow-auto p-0"
+                    align="start"
+                  >
+                    <Command>
+                      <CommandInput
+                        placeholder="Search prompts..."
+                        className="h-9"
+                      />
+                      <CommandList>
+                        <CommandEmpty>No prompt found.</CommandEmpty>
+                        <CommandGroup>
+                          {promptsByName &&
+                            Object.entries(promptsByName).map(
+                              ([name, promptData]) => (
+                                <CommandItem
+                                  key={name}
+                                  onSelect={() => {
+                                    setSelectedPromptName(name);
+                                    const latestVersion = promptData[0];
+                                    setSelectedPromptVersion(
+                                      latestVersion.version,
+                                    );
+                                    form.setValue("promptId", latestVersion.id);
+                                    form.clearErrors("promptId");
+                                  }}
+                                >
+                                  {name}
+                                  <CheckIcon
+                                    className={cn(
+                                      "ml-auto h-4 w-4",
+                                      name === selectedPromptName
+                                        ? "opacity-100"
+                                        : "opacity-0",
+                                    )}
+                                  />
+                                </CommandItem>
+                              ),
+                            )}
+                        </CommandGroup>
+                      </CommandList>
+                    </Command>
+                  </PopoverContent>
+                </Popover>
+
+                <Popover>
+                  <PopoverTrigger asChild>
+                    <Button
+                      disabled={!selectedPromptName}
+                      variant="outline"
+                      role="combobox"
+                      className="w-1/3 justify-between px-2 font-normal"
+                    >
+                      {selectedPromptVersion
+                        ? `Version ${selectedPromptVersion}`
+                        : "Version"}
+                      <ChevronDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+                    </Button>
+                  </PopoverTrigger>
+                  <PopoverContent
+                    className="w-[--radix-popover-trigger-width] p-0"
+                    align="start"
+                  >
+                    <Command>
+                      <CommandList>
+                        <CommandEmpty>No version found.</CommandEmpty>
+                        <CommandGroup className="overflow-y-auto">
+                          {promptsByName &&
+                          selectedPromptName &&
+                          promptsByName[selectedPromptName] ? (
+                            promptsByName[selectedPromptName].map((prompt) => (
+                              <CommandItem
+                                key={prompt.id}
+                                onSelect={() => {
+                                  setSelectedPromptVersion(prompt.version);
+                                  form.setValue("promptId", prompt.id);
+                                  form.clearErrors("promptId");
+                                }}
+                              >
+                                Version {prompt.version}
+                                <CheckIcon
+                                  className={cn(
+                                    "ml-auto h-4 w-4",
+                                    prompt.version === selectedPromptVersion
+                                      ? "opacity-100"
+                                      : "opacity-0",
+                                  )}
+                                />
+                              </CommandItem>
+                            ))
+                          ) : (
+                            <CommandItem disabled>
+                              No versions available
+                            </CommandItem>
+                          )}
+                        </CommandGroup>
+                      </CommandList>
+                    </Command>
+                  </PopoverContent>
+                </Popover>
+              </div>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+
+        <FormField
+          control={form.control}
+          name="modelConfig"
+          render={() => (
+            <FormItem>
+              <Card className="p-4">
+                <ModelParameters
+                  {...{
+                    modelParams,
+                    availableModels,
+                    availableProviders,
+                    updateModelParamValue: updateModelParamValue,
+                    setModelParamEnabled,
+                    modelParamsDescription:
+                      "Select a model which supports function calling.",
+                  }}
+                  evalModelsOnly
+                />
+              </Card>
+              {form.formState.errors.modelConfig && (
+                <p
+                  id="modelConfig"
+                  className={cn("text-sm font-medium text-destructive")}
+                >
+                  {[
+                    form.formState.errors.modelConfig?.model?.message,
+                    form.formState.errors.modelConfig?.provider?.message,
+                  ].join(", ")}
+                </p>
+              )}
+            </FormItem>
+          )}
+        />
+
+        <FormField
+          control={form.control}
+          name="datasetId"
+          render={({ field }) => (
+            <FormItem>
+              <div className="flex items-center gap-2">
+                <FormLabel>Dataset</FormLabel>
+                <Popover>
+                  <PopoverTrigger asChild>
+                    <span className="cursor-pointer text-xs text-muted-foreground">
+                      (expected columns)
+                    </span>
+                  </PopoverTrigger>
+                  <PopoverContent className="w-80">
+                    <div className="flex flex-col space-y-2">
+                      <h4 className="text-sm font-medium leading-none">
+                        Expected columns
+                      </h4>
+                      <span className="text-sm text-muted-foreground">
+                        {promptId ? (
+                          <div>
+                            <span>
+                              Given current prompt, dataset item input must
+                              contain at least one of these first-level JSON
+                              keys, mapped to a string value:
+                            </span>
+                            <ul className="my-2 ml-2 list-inside list-disc">
+                              {expectedColumns.map((col) => (
+                                <li key={col}>{col}</li>
+                              ))}
+                            </ul>
+                            <span>
+                              These will be used as the input to your prompt.
+                            </span>
+                          </div>
+                        ) : (
+                          "Please select a prompt first"
+                        )}
+                      </span>
+                    </div>
+                  </PopoverContent>
+                </Popover>
+              </div>
+              <Select onValueChange={field.onChange} defaultValue={field.value}>
+                <FormControl>
+                  <SelectTrigger>
+                    <SelectValue placeholder="Select a dataset" />
+                  </SelectTrigger>
+                </FormControl>
+                <SelectContent>
+                  {datasets.data?.map((dataset) => (
+                    <SelectItem value={dataset.id} key={dataset.id}>
+                      {dataset.name}
+                    </SelectItem>
+                  ))}
+                </SelectContent>
+              </Select>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+
+        {evaluators.data && !!datasetId ? (
+          <FormItem>
+            <FormLabel>Evaluators</FormLabel>
+            <FormDescription>
+              Will run against your experiment results.
+            </FormDescription>
+            <MultiSelectKeyValues
+              key={datasetId}
+              placeholder="Value"
+              align="end"
+              className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
+              disabled={!hasEvalWriteAccess}
+              onValueChange={handleOnValueChange}
+              options={evaluatorOptions}
+              values={
+                selectedEvaluators as {
+                  value: string;
+                  key: string;
+                }[]
+              }
+              hideClearButton
+              controlButtons={
+                <CommandItem
+                  onSelect={() => {
+                    window.open(`/project/${projectId}/evals`, "_blank");
+                  }}
+                >
+                  Manage evaluators
+                </CommandItem>
+              }
+            />
+          </FormItem>
+        ) : (
+          <FormItem>
+            <FormLabel>Evaluators</FormLabel>
+            {hasEvalReadAccess ? (
+              <FormDescription>
+                Select a dataset first to set up evaluators.
+              </FormDescription>
+            ) : (
+              <FormDescription>
+                ⓘ You do not have access to view evaluators. Please contact your
+                admin to upgrade your role.
+              </FormDescription>
+            )}
+          </FormItem>
+        )}
+
+        <div className="mt-4 flex flex-col gap-4">
+          {validationResult.isLoading && Boolean(promptId && datasetId) && (
+            <Card className="relative overflow-hidden rounded-md shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm">
+                  <span>Validating configuration...</span>
+                  <Loader2 className="h-3 w-3 animate-spin" />
+                </CardTitle>
+                <CardDescription className="text-foreground">
+                  Checking dataset items against prompt variables
+                </CardDescription>
+              </CardHeader>
+            </Card>
+          )}
+          {validationResult.data?.isValid === false && (
+            <Card className="relative overflow-hidden rounded-md border-dark-yellow bg-light-yellow shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm text-dark-yellow">
+                  <span>Invalid configuration</span>
+                  {/* TODO: add link to docs explaining error cases */}
+                  <Info className="h-4 w-4" />
+                </CardTitle>
+                <CardDescription className="text-foreground">
+                  {validationResult.data?.message}
+                </CardDescription>
+              </CardHeader>
+            </Card>
+          )}
+          {validationResult.data?.isValid === true && (
+            <Card className="relative overflow-hidden rounded-md border-dark-green bg-light-green shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm text-dark-green">
+                  <span>Valid configuration</span>
+                  <CircleCheck className="h-4 w-4" />
+                </CardTitle>
+                <div className="text-sm">
+                  Matches between dataset items and prompt variables
+                  <ul className="my-2 ml-2 list-inside list-disc">
+                    {Object.entries(
+                      validationResult.data.variablesMap ?? {},
+                    ).map(([variable, count]) => (
+                      <li key={variable}>
+                        <strong>{variable}:</strong> {count} /{" "}
+                        {validationResult.data?.isValid
+                          ? validationResult.data.totalItems
+                          : "unknown"}
+                      </li>
+                    ))}
+                  </ul>
+                  Items missing all prompt variables will be excluded from the
+                  experiment.
+                </div>
+              </CardHeader>
+            </Card>
+          )}
+
+          <div className="flex justify-end">
+            <Button
+              type="submit"
+              disabled={
+                Boolean(promptId && datasetId) &&
+                !validationResult.data?.isValid
+              }
+              loading={form.formState.isSubmitting}
+            >
+              Create
+            </Button>
+          </div>
+        </div>
+      </form>
+    </Form>
+  );
+};

web/src/ee/features/experiments/server/router.ts

@@ -0,0 +1,220 @@
+import { z } from "zod";
+import { randomUUID } from "crypto";
+import {
+  type ExperimentMetadata,
+  QueueJobs,
+  QueueName,
+  redis,
+  ZodModelConfig,
+  ExperimentCreateQueue,
+} from "@langfuse/shared/src/server";
+import { env } from "@/src/env.mjs";
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
+import {
+  type DatasetItem,
+  extractVariables,
+  UnauthorizedError,
+} from "@langfuse/shared";
+import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
+import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+
+const ValidConfigResponse = z.object({
+  isValid: z.literal(true),
+  totalItems: z.number(),
+  variablesMap: z.record(z.string(), z.number()),
+});
+
+const InvalidConfigResponse = z.object({
+  isValid: z.literal(false),
+  message: z.string(),
+});
+
+const ConfigResponse = z.discriminatedUnion("isValid", [
+  ValidConfigResponse,
+  InvalidConfigResponse,
+]);
+
+const validateDatasetItems = (
+  datasetItems: DatasetItem[],
+  variables: string[],
+): Record<string, number> => {
+  const variableMap: Record<string, number> = {};
+
+  for (const { input } of datasetItems) {
+    if (!input) {
+      continue;
+    }
+
+    const inputKeys = Object.keys(input);
+
+    // For each variable, increment its count if it exists in this item
+    for (const variable of variables) {
+      if (inputKeys.includes(variable)) {
+        variableMap[variable] = (variableMap[variable] || 0) + 1;
+      }
+    }
+  }
+
+  return variableMap;
+};
+
+export const experimentsRouter = createTRPCRouter({
+  validateConfig: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        datasetId: z.string(),
+        promptId: z.string(),
+      }),
+    )
+    .output(ConfigResponse)
+    .query(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "experiments",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "experiments:CUD",
+      });
+
+      const prompt = await ctx.prisma.prompt.findFirst({
+        where: {
+          id: input.promptId,
+          projectId: input.projectId,
+        },
+      });
+
+      if (!prompt) {
+        return {
+          isValid: false,
+          message: "Selected prompt not found.",
+        };
+      }
+
+      const extractedVariables = extractVariables(
+        prompt?.type === PromptType.Text
+          ? (prompt.prompt?.toString() ?? "")
+          : JSON.stringify(prompt.prompt),
+      );
+
+      if (!Boolean(extractedVariables.length)) {
+        return {
+          isValid: false,
+          message: "Selected prompt has no variables.",
+        };
+      }
+
+      const datasetItems = await ctx.prisma.datasetItem.findMany({
+        where: {
+          datasetId: input.datasetId,
+          projectId: input.projectId,
+        },
+      });
+
+      if (!Boolean(datasetItems.length)) {
+        return {
+          isValid: false,
+          message: "Selected dataset is empty.",
+        };
+      }
+
+      const variablesMap = validateDatasetItems(
+        datasetItems,
+        extractedVariables,
+      );
+
+      if (!Boolean(Object.keys(variablesMap).length)) {
+        return {
+          isValid: false,
+          message: "No dataset item contains any variables.",
+        };
+      }
+
+      return {
+        isValid: true,
+        totalItems: datasetItems.length,
+        variablesMap: variablesMap,
+      };
+    }),
+
+  createExperiment: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string().optional(),
+        promptId: z.string().min(1, "Please select a prompt"),
+        datasetId: z.string().min(1, "Please select a dataset"),
+        description: z.string().max(1000).optional(),
+        modelConfig: z.object({
+          provider: z.string().min(1, "Please select a provider"),
+          model: z.string().min(1, "Please select a model"),
+          modelParams: ZodModelConfig,
+        }),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "experiments",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "experiments:CUD",
+      });
+
+      if (!redis || !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
+        throw new UnauthorizedError("Experiment creation failed");
+      }
+
+      const metadata: ExperimentMetadata = {
+        prompt_id: input.promptId,
+        provider: input.modelConfig.provider,
+        model: input.modelConfig.model,
+        model_params: input.modelConfig.modelParams,
+      };
+      const name =
+        input.name ?? `${input.promptId}-${new Date().toISOString()}`;
+
+      const datasetRun = await ctx.prisma.datasetRuns.create({
+        data: {
+          name: name,
+          description: input.description,
+          datasetId: input.datasetId,
+          metadata: metadata,
+          projectId: input.projectId,
+        },
+      });
+
+      const queue = ExperimentCreateQueue.getInstance();
+
+      if (queue) {
+        await queue.add(QueueName.ExperimentCreate, {
+          name: QueueJobs.ExperimentCreateJob,
+          id: randomUUID(),
+          timestamp: new Date(),
+          payload: {
+            projectId: input.projectId,
+            datasetId: input.datasetId,
+            runId: datasetRun.id,
+            description: input.description,
+          },
+        });
+      }
+
+      return {
+        success: true,
+        datasetId: input.datasetId,
+        runId: datasetRun.id,
+        runName: name,
+      };
+    }),
+});

web/src/ee/features/multi-tenant-sso/types.ts

@@ -28,6 +28,20 @@ export const GithubProviderSchema = base.extend({
     .nullish(),
 });
 
+export const GithubEnterpriseProviderSchema = base.extend({
+  authProvider: z.literal("github-enterprise"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      enterprise: z.object({
+        baseUrl: z.string().url(),
+      }),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const GitlabProviderSchema = base.extend({
   authProvider: z.literal("gitlab"),
   authConfig: z
@@ -88,6 +102,18 @@ export const CognitoProviderSchema = base.extend({
     .nullish(),
 });
 
+export const KeycloakProviderSchema = base.extend({
+  authProvider: z.literal("keycloak"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const CustomProviderSchema = base.extend({
   authProvider: z.literal("custom"),
   authConfig: z
@@ -97,6 +123,7 @@ export const CustomProviderSchema = base.extend({
       clientSecret: z.string(),
       issuer: z.string(),
       scope: z.string().nullish(),
+      idToken: z.boolean().optional().default(true),
       allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
     })
     .nullish(),
@@ -104,21 +131,27 @@ export const CustomProviderSchema = base.extend({
 
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type GithubEnterpriseProviderSchema = z.infer<
+  typeof GithubEnterpriseProviderSchema
+>;
 export type GitlabProviderSchema = z.infer<typeof GitlabProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
 export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type KeycloakProviderSchema = z.infer<typeof KeycloakProviderSchema>;
 export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
   GithubProviderSchema,
+  GithubEnterpriseProviderSchema,
   GitlabProviderSchema,
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
   CognitoProviderSchema,
+  KeycloakProviderSchema,
   CustomProviderSchema,
 ]);
 

web/src/ee/features/multi-tenant-sso/utils.ts

@@ -4,6 +4,7 @@ import GitHubProvider from "next-auth/providers/github";
 import GitLabProvider from "next-auth/providers/gitlab";
 import OktaProvider from "next-auth/providers/okta";
 import CognitoProvider from "next-auth/providers/cognito";
+import KeycloakProvider from "next-auth/providers/keycloak";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeEnabled } from "@/src/ee/utils/isEeEnabled";
@@ -12,6 +13,7 @@ import { decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import {
   CustomSSOProvider,
+  GitHubEnterpriseProvider,
   logger,
   traceException,
 } from "@langfuse/shared/src/server";
@@ -188,6 +190,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "keycloak")
+    return KeycloakProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else if (provider.authProvider === "custom")
     return CustomSSOProvider({
       id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
@@ -197,6 +205,15 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
         params: { scope: provider.authConfig.scope ?? "openid email profile" },
       },
     });
+  else if (provider.authProvider === "github-enterprise")
+    return GitHubEnterpriseProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+      enterprise: {
+        baseUrl: provider.authConfig.enterprise.baseUrl,
+      },
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line @typescript-eslint/no-unused-vars

web/src/ee/features/playground/page/context/index.tsx

@@ -16,9 +16,9 @@ import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlayg
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
-import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
+  extractVariables,
   type ChatMessageWithId,
   type PromptVariable,
   type UIModelParams,

web/src/ee/features/playground/server/chatCompletionHandler.ts

@@ -45,7 +45,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
       );
     }
 
-    const stream = await fetchLLMCompletion({
+    const { completion } = await fetchLLMCompletion({
       messages,
       modelParams,
       streaming: true,
@@ -55,7 +55,7 @@ export default async function chatCompletionHandler(req: NextRequest) {
       config: parsedKey.data.config,
     });
 
-    return new StreamingTextResponse(stream);
+    return new StreamingTextResponse(completion);
   } catch (err) {
     logger.error("Failed to handle chat completion", err);
 

web/src/env.mjs

@@ -1,6 +1,26 @@
 import { z } from "zod";
 import { createEnv } from "@t3-oss/env-nextjs";
 
+const zAuthMethod = z
+  .enum([
+    "client_secret_basic",
+    "client_secret_post",
+    "client_secret_jwt",
+    "private_key_jwt",
+    "tls_client_auth",
+    "self_signed_tls_client_auth",
+    "none",
+  ])
+  .optional()
+  .default("client_secret_basic");
+
+
+const zAuthChecks = z
+  .string()
+  .optional()
+  .transform((s) => s?.split(",").map((s) => s.trim()))
+  .pipe(z.array(z.enum(["nonce", "none", "pkce", "state"])).optional());
+
 export const env = createEnv({
   /**
    * Specify your server-side environment variables schema here. This way you can ensure the app
@@ -46,7 +66,7 @@ export const env = createEnv({
     LANGFUSE_DEFAULT_PROJECT_ROLE: z
       .enum(["OWNER", "ADMIN", "MEMBER", "VIEWER"])
       .optional(),
-    LANGFUSE_CSP_ENFORCE_HTTPS: z.enum(["true", "false"]).optional(),
+    LANGFUSE_CSP_ENFORCE_HTTPS: z.enum(["true", "false"]).optional().default("false"),
     // Telemetry
     TELEMETRY_ENABLED: z.enum(["true", "false"]).optional(),
     // AUTH
@@ -54,36 +74,68 @@ export const env = createEnv({
     AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
     AUTH_GOOGLE_ALLOWED_DOMAINS: z.string().optional(),
     AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GOOGLE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GOOGLE_CHECKS: zAuthChecks,
     AUTH_GITHUB_CLIENT_ID: z.string().optional(),
     AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GITHUB_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITHUB_CHECKS: zAuthChecks,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_ID: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING: z
+      .enum(["true", "false"])
+      .optional(),
+    AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITHUB_ENTERPRISE_CHECKS: zAuthChecks,
     AUTH_GITLAB_CLIENT_ID: z.string().optional(),
     AUTH_GITLAB_CLIENT_SECRET: z.string().optional(),
     AUTH_GITLAB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_GITLAB_ISSUER: z.string().optional(),
+    AUTH_GITLAB_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITLAB_CHECKS: zAuthChecks,
     AUTH_AZURE_AD_CLIENT_ID: z.string().optional(),
     AUTH_AZURE_AD_CLIENT_SECRET: z.string().optional(),
     AUTH_AZURE_AD_TENANT_ID: z.string().optional(),
     AUTH_AZURE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_AZURE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_AZURE_CHECKS: zAuthChecks,
     AUTH_OKTA_CLIENT_ID: z.string().optional(),
     AUTH_OKTA_CLIENT_SECRET: z.string().optional(),
     AUTH_OKTA_ISSUER: z.string().optional(),
     AUTH_OKTA_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_OKTA_CHECKS: zAuthChecks,
+    AUTH_OKTA_CLIENT_AUTH_METHOD: zAuthMethod,
     AUTH_AUTH0_CLIENT_ID: z.string().optional(),
     AUTH_AUTH0_CLIENT_SECRET: z.string().optional(),
     AUTH_AUTH0_ISSUER: z.string().url().optional(),
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_AUTH0_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_AUTH0_CHECKS: zAuthChecks,
     AUTH_COGNITO_CLIENT_ID: z.string().optional(),
     AUTH_COGNITO_CLIENT_SECRET: z.string().optional(),
     AUTH_COGNITO_ISSUER: z.string().url().optional(),
     AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_COGNITO_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_COGNITO_CHECKS: zAuthChecks,
+    AUTH_KEYCLOAK_CLIENT_ID: z.string().optional(),
+    AUTH_KEYCLOAK_CLIENT_SECRET: z.string().optional(),
+    AUTH_KEYCLOAK_ISSUER: z.string().optional(),
+    AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_KEYCLOAK_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_KEYCLOAK_CHECKS: zAuthChecks,
     AUTH_CUSTOM_CLIENT_ID: z.string().optional(),
     AUTH_CUSTOM_CLIENT_SECRET: z.string().optional(),
     AUTH_CUSTOM_ISSUER: z.string().url().optional(),
     AUTH_CUSTOM_NAME: z.string().optional(),
     AUTH_CUSTOM_SCOPE: z.string().optional(),
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_CUSTOM_CHECKS: zAuthChecks,
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_CUSTOM_ID_TOKEN: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
+    AUTH_IGNORE_ACCOUNT_FIELDS: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
     AUTH_SESSION_MAX_AGE: z.coerce
@@ -95,6 +147,8 @@ export const env = createEnv({
       )
       .optional()
       .default(30 * 24 * 60), // default to 30 days
+    AUTH_HTTP_PROXY: z.string().url().optional(),
+    AUTH_HTTPS_PROXY: z.string().url().optional(),
     // EMAIL
     EMAIL_FROM_ADDRESS: z.string().optional(),
     SMTP_CONNECTION_URL: z.string().optional(),
@@ -119,6 +173,7 @@ export const env = createEnv({
     CLICKHOUSE_URL: z.string().optional(),
     CLICKHOUSE_USER: z.string().optional(),
     CLICKHOUSE_PASSWORD: z.string().optional(),
+    CLICKHOUSE_CLUSTER_ENABLED: z.enum(["true", "false"]).default("false"),
     // EE ui customization
     LANGFUSE_UI_API_HOST: z.string().optional(),
     LANGFUSE_UI_DOCUMENTATION_HREF: z.string().url().optional(),
@@ -182,26 +237,6 @@ export const env = createEnv({
       .nonnegative()
       .default(3600),
 
-    // Ingestion event upload to S3
-    LANGFUSE_S3_EVENT_UPLOAD_ENABLED: z
-      .enum(["true", "false"])
-      .default("false"),
-
-    LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string().optional(),
-    LANGFUSE_S3_EVENT_UPLOAD_PREFIX: z.string().default(""),
-    LANGFUSE_S3_EVENT_UPLOAD_REGION: z.string().optional(),
-    LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: z.string().optional(),
-    LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: z.string().optional(),
-    LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY: z.string().optional(),
-    LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
-      .enum(["true", "false"])
-      .default("false"),
-    LANGFUSE_ASYNC_INGESTION_PROCESSING: z
-      .enum(["true", "false"])
-      .default("false"),
-    LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING: z
-      .enum(["true", "false"])
-      .default("false"),
     LANGFUSE_ALLOWED_ORGANIZATION_CREATORS: z
       .string()
       .optional()
@@ -213,13 +248,10 @@ export const env = createEnv({
           (creator) => emailSchema.safeParse(creator).success,
         );
       }, "LANGFUSE_ALLOWED_ORGANIZATION_CREATORS must be a comma separated list of valid email addresses"),
-    LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
-      .number()
-      .nonnegative()
-      .default(15_000),
     LANGFUSE_READ_FROM_POSTGRES_ONLY: z.enum(["true", "false"]).default("true"),
     LANGFUSE_RETURN_FROM_CLICKHOUSE: z.enum(["true", "false"]).default("false"),
     LANGFUSE_EXPERIMENT_EXCLUDED_PROJECT_IDS: z.string().optional(),
+    LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS: z.string().optional(),
     LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: z
       .enum(["true", "false"])
       .default("false"),
@@ -288,6 +320,8 @@ export const env = createEnv({
     NEXT_PUBLIC_SIGN_UP_DISABLED: process.env.NEXT_PUBLIC_SIGN_UP_DISABLED,
     LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:
       process.env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES,
+    LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS:
+      process.env.LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS,
     LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES:
       process.env.LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES,
     LANGFUSE_TEAM_SLACK_WEBHOOK: process.env.LANGFUSE_TEAM_SLACK_WEBHOOK,
@@ -307,47 +341,85 @@ export const env = createEnv({
     AUTH_GOOGLE_ALLOWED_DOMAINS: process.env.AUTH_GOOGLE_ALLOWED_DOMAINS,
     AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING,
+    AUTH_GOOGLE_CLIENT_AUTH_METHOD: process.env.AUTH_GOOGLE_CLIENT_AUTH_METHOD,
+    AUTH_GOOGLE_CHECKS: process.env.AUTH_GOOGLE_CHECKS,
     AUTH_GITHUB_CLIENT_ID: process.env.AUTH_GITHUB_CLIENT_ID,
     AUTH_GITHUB_CLIENT_SECRET: process.env.AUTH_GITHUB_CLIENT_SECRET,
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GITHUB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITHUB_CLIENT_AUTH_METHOD: process.env.AUTH_GITHUB_CLIENT_AUTH_METHOD,
+    AUTH_GITHUB_CHECKS: process.env.AUTH_GITHUB_CHECKS,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_ID:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_ID,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET,
+    AUTH_GITHUB_ENTERPRISE_BASE_URL:
+      process.env.AUTH_GITHUB_ENTERPRISE_BASE_URL,
+    AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD,
+    AUTH_GITHUB_ENTERPRISE_CHECKS: process.env.AUTH_GITHUB_ENTERPRISE_CHECKS,
     AUTH_GITLAB_ISSUER: process.env.AUTH_GITLAB_ISSUER,
     AUTH_GITLAB_CLIENT_ID: process.env.AUTH_GITLAB_CLIENT_ID,
     AUTH_GITLAB_CLIENT_SECRET: process.env.AUTH_GITLAB_CLIENT_SECRET,
     AUTH_GITLAB_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GITLAB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITLAB_CLIENT_AUTH_METHOD: process.env.AUTH_GITLAB_CLIENT_AUTH_METHOD,
+    AUTH_GITLAB_CHECKS: process.env.AUTH_GITLAB_CHECKS,
     AUTH_AZURE_AD_CLIENT_ID: process.env.AUTH_AZURE_AD_CLIENT_ID,
     AUTH_AZURE_AD_CLIENT_SECRET: process.env.AUTH_AZURE_AD_CLIENT_SECRET,
     AUTH_AZURE_AD_TENANT_ID: process.env.AUTH_AZURE_AD_TENANT_ID,
     AUTH_AZURE_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AZURE_ALLOW_ACCOUNT_LINKING,
+    AUTH_AZURE_CLIENT_AUTH_METHOD: process.env.AUTH_AZURE_CLIENT_AUTH_METHOD,
+    AUTH_AZURE_CHECKS: process.env.AUTH_AZURE_CHECKS,
     AUTH_OKTA_CLIENT_ID: process.env.AUTH_OKTA_CLIENT_ID,
     AUTH_OKTA_CLIENT_SECRET: process.env.AUTH_OKTA_CLIENT_SECRET,
     AUTH_OKTA_ISSUER: process.env.AUTH_OKTA_ISSUER,
     AUTH_OKTA_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
+    AUTH_OKTA_CLIENT_AUTH_METHOD: process.env.AUTH_OKTA_CLIENT_AUTH_METHOD,
+    AUTH_OKTA_CHECKS: process.env.AUTH_OKTA_CHECKS,
     AUTH_AUTH0_CLIENT_ID: process.env.AUTH_AUTH0_CLIENT_ID,
     AUTH_AUTH0_CLIENT_SECRET: process.env.AUTH_AUTH0_CLIENT_SECRET,
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_AUTH0_CLIENT_AUTH_METHOD: process.env.AUTH_AUTH0_CLIENT_AUTH_METHOD,
+    AUTH_AUTH0_CHECKS: process.env.AUTH_AUTH0_CHECKS,
     AUTH_COGNITO_CLIENT_ID: process.env.AUTH_COGNITO_CLIENT_ID,
     AUTH_COGNITO_CLIENT_SECRET: process.env.AUTH_COGNITO_CLIENT_SECRET,
     AUTH_COGNITO_ISSUER: process.env.AUTH_COGNITO_ISSUER,
     AUTH_COGNITO_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
+    AUTH_COGNITO_CLIENT_AUTH_METHOD: process.env.AUTH_COGNITO_CLIENT_AUTH_METHOD,
+    AUTH_COGNITO_CHECKS: process.env.AUTH_COGNITO_CHECKS,
+    AUTH_KEYCLOAK_CLIENT_ID: process.env.AUTH_KEYCLOAK_CLIENT_ID,
+    AUTH_KEYCLOAK_CLIENT_SECRET: process.env.AUTH_KEYCLOAK_CLIENT_SECRET,
+    AUTH_KEYCLOAK_ISSUER: process.env.AUTH_KEYCLOAK_ISSUER,
+    AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING,
+    AUTH_KEYCLOAK_CLIENT_AUTH_METHOD: process.env.AUTH_KEYCLOAK_CLIENT_AUTH_METHOD,
+    AUTH_KEYCLOAK_CHECKS: process.env.AUTH_KEYCLOAK_CHECKS,
     AUTH_CUSTOM_CLIENT_ID: process.env.AUTH_CUSTOM_CLIENT_ID,
     AUTH_CUSTOM_CLIENT_SECRET: process.env.AUTH_CUSTOM_CLIENT_SECRET,
     AUTH_CUSTOM_ISSUER: process.env.AUTH_CUSTOM_ISSUER,
     AUTH_CUSTOM_NAME: process.env.AUTH_CUSTOM_NAME,
     AUTH_CUSTOM_SCOPE: process.env.AUTH_CUSTOM_SCOPE,
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD: process.env.AUTH_CUSTOM_CLIENT_AUTH_METHOD,
+    AUTH_CUSTOM_CHECKS: process.env.AUTH_CUSTOM_CHECKS,
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING,
+    AUTH_CUSTOM_ID_TOKEN: process.env.AUTH_CUSTOM_ID_TOKEN,
+    AUTH_IGNORE_ACCOUNT_FIELDS: process.env.AUTH_IGNORE_ACCOUNT_FIELDS,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
     AUTH_DISABLE_SIGNUP: process.env.AUTH_DISABLE_SIGNUP,
     AUTH_SESSION_MAX_AGE: process.env.AUTH_SESSION_MAX_AGE,
+    AUTH_HTTP_PROXY: process.env.AUTH_HTTP_PROXY,
+    AUTH_HTTPS_PROXY: process.env.AUTH_HTTPS_PROXY,
     // Email
     EMAIL_FROM_ADDRESS: process.env.EMAIL_FROM_ADDRESS,
     SMTP_CONNECTION_URL: process.env.SMTP_CONNECTION_URL,
@@ -384,23 +456,6 @@ export const env = createEnv({
       process.env.LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE,
     LANGFUSE_S3_MEDIA_DOWNLOAD_URL_EXPIRY_SECONDS:
       process.env.LANGFUSE_S3_MEDIA_DOWNLOAD_URL_EXPIRY_SECONDS,
-    // S3 event upload
-    LANGFUSE_S3_EVENT_UPLOAD_ENABLED:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED,
-    LANGFUSE_S3_EVENT_UPLOAD_BUCKET:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-    LANGFUSE_S3_EVENT_UPLOAD_PREFIX:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX,
-    LANGFUSE_S3_EVENT_UPLOAD_REGION:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
-    LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
-    LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
-    LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
-    LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE:
-      process.env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE,
     // Database exports
     DB_EXPORT_PAGE_SIZE: process.env.DB_EXPORT_PAGE_SIZE,
     // Worker
@@ -414,6 +469,7 @@ export const env = createEnv({
     CLICKHOUSE_URL: process.env.CLICKHOUSE_URL,
     CLICKHOUSE_USER: process.env.CLICKHOUSE_USER,
     CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD,
+    CLICKHOUSE_CLUSTER_ENABLED: process.env.CLICKHOUSE_CLUSTER_ENABLED,
     // EE ui customization
     LANGFUSE_UI_API_HOST: process.env.LANGFUSE_UI_API_HOST,
     LANGFUSE_UI_DOCUMENTATION_HREF: process.env.LANGFUSE_UI_DOCUMENTATION_HREF,
@@ -444,14 +500,8 @@ export const env = createEnv({
     LANGFUSE_CACHE_API_KEY_ENABLED: process.env.LANGFUSE_CACHE_API_KEY_ENABLED,
     LANGFUSE_CACHE_API_KEY_TTL_SECONDS:
       process.env.LANGFUSE_CACHE_API_KEY_TTL_SECONDS,
-    LANGFUSE_ASYNC_INGESTION_PROCESSING:
-      process.env.LANGFUSE_ASYNC_INGESTION_PROCESSING,
-    LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING:
-      process.env.LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING,
     LANGFUSE_ALLOWED_ORGANIZATION_CREATORS:
       process.env.LANGFUSE_ALLOWED_ORGANIZATION_CREATORS,
-    LANGFUSE_INGESTION_QUEUE_DELAY_MS:
-      process.env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
     LANGFUSE_READ_FROM_POSTGRES_ONLY:
       process.env.LANGFUSE_READ_FROM_POSTGRES_ONLY,
     LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:

web/src/features/datasets/components/DatasetAggregateTableCell.tsx

@@ -47,6 +47,7 @@ const DatasetAggregateCell = ({
       refetchOnWindowFocus: false,
       refetchOnReconnect: false,
       staleTime: Infinity,
+      onError: () => {},
     },
   );
   const observation = api.observations.byId.useQuery(
@@ -66,6 +67,7 @@ const DatasetAggregateCell = ({
       refetchOnWindowFocus: false,
       refetchOnReconnect: false,
       staleTime: Infinity,
+      onError: () => {},
     },
   );
 
@@ -81,7 +83,10 @@ const DatasetAggregateCell = ({
       {variant === "peek" && actionButtons}
       <div className="flex flex-row items-center justify-center gap-1">
         <IOTableCell
-          isLoading={!!!observationId ? trace.isLoading : observation.isLoading}
+          isLoading={
+            (!!!observationId ? trace.isLoading : observation.isLoading) ||
+            !data
+          }
           data={data?.output}
           className={"bg-accent-light-green"}
           singleLine={singleLine}

web/src/features/datasets/components/DatasetCompareRunsTable.tsx

@@ -10,7 +10,7 @@ import { type ScoreAggregate } from "@langfuse/shared";
 import { type Prisma } from "@langfuse/shared";
 import { NumberParam } from "use-query-params";
 import { useQueryParams, withDefault } from "use-query-params";
-import { useMemo, useState } from "react";
+import { useMemo, useState, useCallback } from "react";
 import { usdFormatter } from "@/src/utils/numbers";
 import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { api, type RouterOutputs } from "@/src/utils/api";
@@ -24,6 +24,8 @@ import {
 } from "@/src/components/ui/dropdown-menu";
 import { DatasetCompareRunPeekView } from "@/src/features/datasets/components/DatasetCompareRunPeekView";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
+import { getQueryKey } from "@trpc/react-query";
+import { useQueryClient } from "@tanstack/react-query";
 
 export type RunMetrics = {
   id: string;
@@ -47,6 +49,16 @@ export type DatasetCompareRunRowData = {
   runs?: RunAggregate;
 };
 
+const getRefetchInterval = (
+  runId: string,
+  localExperiments: { key: string; value: string }[],
+  unchangedCounts: Record<string, number>,
+) => {
+  if (unchangedCounts[runId] < 2) return 5000;
+  if (localExperiments.some((run) => run.key === runId)) return 3000;
+  return false;
+};
+
 const DATASET_RUN_METRICS = ["scores", "resourceMetrics"] as const;
 export type DatasetRunMetric = (typeof DATASET_RUN_METRICS)[number];
 
@@ -55,6 +67,7 @@ export function DatasetCompareRunsTable(props: {
   datasetId: string;
   runIds: string[];
   runsData?: RouterOutputs["datasets"]["baseRunDataByDatasetId"];
+  localExperiments: { key: string; value: string }[];
 }) {
   const [selectedMetrics, setSelectedMetrics] = useState<DatasetRunMetric[]>([
     "scores",
@@ -69,6 +82,10 @@ export function DatasetCompareRunsTable(props: {
     traceId: string;
     observationId?: string;
   } | null>(null);
+  const [unchangedCounts, setUnchangedCounts] = useState<
+    Record<string, number>
+  >({});
+  const queryClient = useQueryClient();
 
   const rowHeight = "l";
 
@@ -84,8 +101,51 @@ export function DatasetCompareRunsTable(props: {
     limit: paginationState.pageSize,
   });
   const queryClickhouse = useClickhouse();
-  // Individual queries for each run
-  const runs = (props.runIds ?? []).map((runId) => ({
+
+  // 1. First, separate the run definitions
+  const runQueries = useMemo(
+    () =>
+      (props.runIds ?? []).map((runId) => ({
+        runId,
+        queryKey: getQueryKey(api.datasets.runitemsByRunIdOrItemId, {
+          projectId: props.projectId,
+          datasetRunId: runId,
+          page: paginationState.pageIndex,
+          limit: paginationState.pageSize,
+          queryClickhouse,
+        }),
+      })),
+    [
+      props.runIds,
+      props.projectId,
+      paginationState.pageIndex,
+      paginationState.pageSize,
+      queryClickhouse,
+    ],
+  );
+
+  // 2. Track changes using onSuccess callback in the queries instead of useEffect
+  const handleQuerySuccess = useCallback(
+    (runId: string, newData: any) => {
+      setUnchangedCounts((prev) => {
+        const prevCount = prev[runId] || 0;
+        const queryKey = runQueries.find((r) => r.runId === runId)?.queryKey;
+        const prevData = queryClient.getQueryData(queryKey || []);
+
+        // Only increment if we have previous data and it matches the new data
+        if (prevData && JSON.stringify(prevData) === JSON.stringify(newData)) {
+          const newCount = prevCount + 1;
+          return { ...prev, [runId]: newCount };
+        }
+
+        return { ...prev, [runId]: 0 };
+      });
+    },
+    [queryClient, runQueries],
+  );
+
+  // 3. Use the queries with success callback
+  const runs = runQueries.map(({ runId }) => ({
     runId,
     items: api.datasets.runitemsByRunIdOrItemId.useQuery(
       {
@@ -99,8 +159,14 @@ export function DatasetCompareRunsTable(props: {
         refetchOnWindowFocus: false,
         refetchOnMount: false,
         refetchOnReconnect: false,
-        staleTime: 5 * 60 * 1000, // 5 minutes
+        staleTime: 5 * 60 * 1000,
         enabled: baseDatasetItems.isSuccess,
+        refetchInterval: getRefetchInterval(
+          runId,
+          props.localExperiments,
+          unchangedCounts,
+        ),
+        onSuccess: (data) => handleQuerySuccess(runId, data),
       },
     ),
   }));
@@ -139,7 +205,7 @@ export function DatasetCompareRunsTable(props: {
       {},
     );
 
-    return baseDatasetItems.data?.map(
+    return baseDatasetItems.data?.datasetItems.map(
       (item): DatasetCompareRunRowData => ({
         id: item.id,
         input: item.input ?? "null",
@@ -356,7 +422,7 @@ export function DatasetCompareRunsTable(props: {
                 }
         }
         pagination={{
-          totalCount: baseDatasetItems.data?.length ?? null,
+          totalCount: baseDatasetItems.data?.totalCount ?? null,
           onChange: setPaginationState,
           state: paginationState,
         }}