chore: release v2.95.2

security: upgrade dompurify

.env.dev-azure.example

@@ -0,0 +1,92 @@
+# When adding additional environment variables, the schema in "/src/env.mjs"
+# should be updated accordingly.
+
+# Prisma
+# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
+
+# Clickhouse
+CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+CLICKHOUSE_URL="http://localhost:8123"
+CLICKHOUSE_USER="clickhouse"
+CLICKHOUSE_PASSWORD="clickhouse"
+CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+
+# Next Auth
+# You can generate a new secret on the command line with:
+# openssl rand -base64 32
+# https://next-auth.js.org/configuration/options#secret
+# NEXTAUTH_SECRET=""
+NEXTAUTH_URL="http://localhost:3000"
+NEXTAUTH_SECRET="secret"
+
+# Langfuse Cloud Environment
+NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
+
+# Langfuse experimental features
+LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
+
+# Salt for API key hashing
+SALT="salt"
+
+# Email
+EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
+SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+
+# DON'T PANIC: The Azurite Secrets are well-known and meant to be hard-coded
+# S3 storage
+S3_ENDPOINT=http://localhost:10000/devstoreaccount1
+S3_ACCESS_KEY_ID=devstoreaccount1
+S3_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+S3_BUCKET_NAME=langfuse
+S3_REGION=auto
+## Necessary for minio compatibility
+S3_FORCE_PATH_STYLE=true
+
+# S3 Media Upload LOCAL
+LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
+LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_MEDIA_UPLOAD_REGION=auto
+LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
+
+# S3 Event Bucket Upload
+## Set to true to test uploading all events to S3
+LANGFUSE_S3_EVENT_UPLOAD_ENABLED=true
+LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
+LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=devstoreaccount1
+LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+LANGFUSE_S3_EVENT_UPLOAD_REGION=auto
+LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:10000/devstoreaccount1
+## Necessary for minio compatibility
+LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
+
+LANGFUSE_USE_AZURE_BLOB=true
+
+# Set during docker build of application
+# Used to disable environment verification at build time
+# DOCKER_BUILD=1
+
+REDIS_HOST="127.0.0.1"
+REDIS_PORT=6379
+REDIS_AUTH="myredissecret"
+
+# openssl rand -hex 32 used only here
+ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
+
+# speeds up local development by not executing init scripts on server startup
+NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+LANGFUSE_READ_FROM_POSTGRES_ONLY=false
+LANGFUSE_RETURN_FROM_CLICKHOUSE=true
+LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=true
+LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=true
+
+LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
+LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING="true"

.env.dev.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_DISABLED="true"
 
 # Next Auth
 # You can generate a new secret on the command line with:
@@ -43,7 +43,7 @@ S3_REGION=us-east-1
 ## Necessary for minio compatibility
 S3_FORCE_PATH_STYLE=true
 
-# # S3 Media Upload LOCAL
+# S3 Media Upload LOCAL
 LANGFUSE_S3_MEDIA_UPLOAD_ENABLED=true
 LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
 LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio

.env.dev.legacy.example

@@ -11,7 +11,7 @@ CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
 CLICKHOUSE_URL="http://localhost:8123"
 CLICKHOUSE_USER="clickhouse"
 CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_MIGRATION_CLUSTER_DISABLED="true"
+CLICKHOUSE_CLUSTER_ENABLED="false"
 
 # Next Auth
 # You can generate a new secret on the command line with:

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -67,6 +68,10 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_GITHUB_CLIENT_ID=
 # AUTH_GITHUB_CLIENT_SECRET=
 # AUTH_GITHUB_ALLOW_ACCOUNT_LINKING=false
+# AUTH_GITHUB_ENTERPRISE_CLIENT_ID=
+# AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET=
+# AUTH_GITHUB_ENTERPRISE_BASE_URL=
+# AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING=false
 # AUTH_GITLAB_CLIENT_ID=
 # AUTH_GITLAB_CLIENT_SECRET=
 # AUTH_GITLAB_ALLOW_ACCOUNT_LINKING=false
@@ -87,12 +92,17 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_COGNITO_CLIENT_SECRET=
 # AUTH_COGNITO_ISSUER=
 # AUTH_COGNITO_ALLOW_ACCOUNT_LINKING=false
+# AUTH_KEYCLOAK_CLIENT_ID=
+# AUTH_KEYCLOAK_CLIENT_SECRET=
+# AUTH_KEYCLOAK_ISSUER=
+# AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING=false
 # AUTH_CUSTOM_CLIENT_ID=
 # AUTH_CUSTOM_CLIENT_SECRET=
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -245,4 +255,4 @@ OTEL_SERVICE_NAME="langfuse"
 # LANGFUSE_ASYNC_INGESTION_PROCESSING="true"
 # QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion

.github/workflows/codespell.yml

@@ -3,9 +3,14 @@ name: Codespell
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
   pull_request:
-    branches: [main]
+    branches:
+      - "**"
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/pipeline.yml

@@ -82,7 +82,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -137,11 +137,12 @@ jobs:
   tests-web-async:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -150,7 +151,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -172,11 +173,11 @@ jobs:
           pnpm install
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          grep -v -e '^S3_BUCKET_NAME=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
         env:
@@ -205,11 +206,12 @@ jobs:
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
+    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
         node-version: [20]
         postgres-version: [12, 15]
+        blob-provider: ["", "-azure"]
     steps:
       - name: Set Swap Space
         uses: pierotofy/set-swap-space@master
@@ -235,17 +237,17 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
         run: |
-          cp .env.dev.example .env
-          cp .env.dev.example web/.env
-          cp .env.dev.example worker/.env
+          cp .env.dev${{ matrix.blob-provider }}.example .env
+          cp .env.dev${{ matrix.blob-provider }}.example web/.env
+          cp .env.dev${{ matrix.blob-provider }}.example worker/.env
       - name: Run + migrate
         run: |
-          docker compose -f docker-compose.dev.yml up -d
+          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
           docker compose ps
       - name: Ensure no unhealthy status
@@ -326,18 +328,18 @@ jobs:
       - name: install dependencies
         run: |
           pnpm install
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
+          echo "LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_ASYNC_INGESTION_PROCESSING=true" >> .env
           echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
           echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
-      - name: Install golang-migrate for Clickhouse migrations
-        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
-          sudo mv migrate /usr/bin/migrate
-          which migrate
-
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -439,6 +441,8 @@ jobs:
           images: |
             ghcr.io/langfuse/langfuse # GitHub
             langfuse/langfuse # Docker Hub
+          flavor: |
+            latest=false
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -446,6 +450,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:

.github/workflows/release.yml

@@ -3,7 +3,7 @@ on:
   push:
     # Pattern matched against refs/tags
     tags:
-      - "v[0-9]+.[0-9]+.[0-9]+" # Semantic version tags
+      - "v3.[0-9]+.[0-9]+" # Semantic version tags
 
 jobs:
   release:

.gitignore

@@ -37,6 +37,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env*
 !.env.dev.example
+!.env.dev-azure.example
 !.env.local.example
 !.env.prod.example
 !.env.dev.legacy.example

docker-compose.dev-azure.yml

@@ -0,0 +1,63 @@
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    user: "101:101"
+    container_name: clickhouse
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: default
+      CLICKHOUSE_USER: clickhouse
+      CLICKHOUSE_PASSWORD: clickhouse
+    volumes:
+      - langfuse_clickhouse_data:/var/lib/clickhouse
+      - langfuse_clickhouse_logs:/var/log/clickhouse-server
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    depends_on:
+      - postgres
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    container_name: azurite
+    command: azurite-blob --blobHost 0.0.0.0
+    ports:
+      - "10000:10000"
+    volumes:
+      - langfuse_azurite_data:/data
+
+  redis:
+    image: redis:7.2.4
+    restart: always
+    command: >
+      --requirepass ${REDIS_AUTH:-myredissecret}
+    ports:
+      - 6379:6379
+
+  postgres:
+    image: postgres:${POSTGRES_VERSION:-latest}
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 3s
+      timeout: 3s
+      retries: 10
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=postgres
+    ports:
+      - 5432:5432
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_clickhouse_data:
+    driver: local
+  langfuse_clickhouse_logs:
+    driver: local
+  langfuse_azurite_data:
+    driver: local

docker-compose.v3preview.yml

@@ -25,7 +25,7 @@ services:
       CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://clickhouse:8123}
       CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
       CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
-      CLICKHOUSE_MIGRATION_CLUSTER_DISABLED: ${CLICKHOUSE_MIGRATION_CLUSTER_DISABLED:-true}
+      CLICKHOUSE_CLUSTER_ENABLED: ${CLICKHOUSE_CLUSTER_ENABLED:-false}
       LANGFUSE_S3_EVENT_UPLOAD_ENABLED: ${LANGFUSE_S3_EVENT_UPLOAD_ENABLED:-true}
       LANGFUSE_S3_EVENT_UPLOAD_BUCKET: ${LANGFUSE_S3_EVENT_UPLOAD_BUCKET:-langfuse}
       LANGFUSE_S3_EVENT_UPLOAD_REGION: ${LANGFUSE_S3_EVENT_UPLOAD_REGION:-us-east-1}

ee/package.json

@@ -27,8 +27,9 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "https-proxy-agent": "^7.0.6",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {
@@ -47,7 +48,7 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0"
     }
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "2.92.0",
+  "version": "2.95.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -83,7 +83,12 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
     }
   },
   "packageManager": "pnpm@9.5.0"

packages/shared/clickhouse/scripts/down.sh

@@ -19,7 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
   else

packages/shared/clickhouse/scripts/up.sh

@@ -19,7 +19,7 @@ then
 fi
 
 # Construct the database URL
-if [ "$CLICKHOUSE_MIGRATION_CLUSTER_DISABLED" != "true" ] ; then
+if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "true" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
       DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
   else

packages/shared/package.json

@@ -59,6 +59,7 @@
     "@aws-sdk/client-s3": "^3.675.0",
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
     "@langchain/anthropic": "^0.3.8",
     "@langchain/aws": "^0.1.2",
@@ -75,12 +76,13 @@
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
     "langchain": "^0.3.6",
-    "langfuse-langchain": "3.30.1",
+    "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
-    "next-auth": "^4.24.7",
+    "next-auth": "^4.24.11",
     "nodemailer": "^6.9.15",
     "prisma-extension-kysely": "^2.1.0",
     "uuid": "^9.0.1",

packages/shared/prisma/migrations/20241125124029_add_chatgpt_4o_prices/migration.sql

@@ -0,0 +1,25 @@
+INSERT INTO models (
+  id,
+  project_id,
+  model_name,
+  match_pattern,
+  start_date,
+  input_price,
+  output_price,
+  total_price,
+  unit,
+  tokenizer_id,
+  tokenizer_config
+)
+VALUES
+  ('cm3x0p8ev000008kyd96800c8', NULL, 'chatgpt-4o-latest', '(?i)^(chatgpt-4o-latest)$', NULL, 0.000005, 0.000015, NULL, 'TOKENS', 'openai', '{ "tokensPerMessage": 3, "tokensPerName": 1, "tokenizerModel": "gpt-4o" }');
+
+INSERT INTO prices (
+  id, 
+  model_id, 
+  usage_type,
+  price
+)
+VALUES
+  ('cm3x0psrz000108kydpxg9o2k', 'cm3x0p8ev000008kyd96800c8', 'input', 0.000005),
+  ('cm3x0pyt7000208ky8737gdla', 'cm3x0p8ev000008kyd96800c8', 'output', 0.000015);

packages/shared/src/env.ts

@@ -58,6 +58,8 @@ const EnvSchema = z.object({
   LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  STRIPE_SECRET_KEY: z.string().optional(),
 });
 
 export const env = EnvSchema.parse(removeEmptyEnvVariables(process.env));

packages/shared/src/index.ts

@@ -11,6 +11,7 @@ export * from "./server/auth/apiKeys";
 export * from "./observationsTable";
 export * from "./utils/zod";
 export * from "./utils/json";
+export * from "./utils/stringChecks";
 export * from "./utils/objects";
 export * from "./utils/typeChecks";
 export * from "./features/entitlements/plans";

packages/shared/src/server/auth/customSsoProvider.ts

@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
     authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
-    idToken: true,
     profile(profile) {
       return {
         id: profile.sub,

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -0,0 +1,65 @@
+import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
+import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
+
+export function GitHubEnterpriseProvider<P extends GithubProfile>(
+    options: OAuthUserConfig<P> & {
+        enterprise?: {
+            baseUrl?: string;
+        };
+    }
+): OAuthConfig<P> {
+    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
+    const apiBaseUrl = options?.enterprise?.baseUrl
+        ? `${options?.enterprise?.baseUrl}/api/v3`
+        : "https://api.github.com"
+
+    return {
+        id: "github-enterprise",
+        name: "GitHub Enterprise",
+        type: "oauth",
+        authorization: {
+            url: `${baseUrl}/login/oauth/authorize`,
+            params: { scope: "read:user user:email" },
+        },
+        token: `${baseUrl}/login/oauth/access_token`,
+        userinfo: {
+            url: `${apiBaseUrl}/user`,
+            async request({ client, tokens }) {
+                const profile = await client.userinfo(tokens.access_token!)
+
+                if (!profile.email) {
+                    // If the user does not have a public email, get another via the GitHub API
+                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
+                        headers: { Authorization: `token ${tokens.access_token}` },
+                    })
+
+                    if (res.ok) {
+                        const emails: GithubEmail[] = await res.json()
+                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
+                    }
+                }
+
+                return profile
+            },
+        },
+        profile(profile) {
+            return {
+                id: profile.id.toString(),
+                name: profile.name ?? profile.login,
+                email: profile.email,
+                image: profile.avatar_url,
+            }
+        },
+        style: {
+            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+            logoDark:
+                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+            bg: "#fff",
+            bgDark: "#000",
+            text: "#000",
+            textDark: "#fff",
+        },
+        options,
+    }
+}

packages/shared/src/server/index.ts

@@ -1,4 +1,4 @@
-export * from "./services/S3StorageService";
+export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
@@ -6,6 +6,7 @@ export * from "./services/PromptService";
 export * from "./services/traces-ui-table-service";
 export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
+export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
@@ -21,10 +22,13 @@ export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
+export * from "./redis/CloudUsageMeteringQueue";
+export * from "./redis/getQueue";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/legacyIngestion";
 export * from "./redis/ingestionQueue";
+export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
 export * from "./ingestion/legacy/index";
 export * from "./queues";

packages/shared/src/server/ingestion/legacy/index.ts

@@ -3,13 +3,7 @@ import z from "zod";
 import { ForbiddenError, UnauthorizedError } from "../../../errors";
 import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
 import { getProcessorForEvent } from "./EventProcessor";
-import { TraceUpsertEventType } from "../../queues";
-import {
-  convertTraceUpsertEventsToRedisEvents,
-  TraceUpsertQueue,
-} from "../../redis/traceUpsert";
 import { ApiAccessScope } from "../../auth/types";
-import { redis } from "../../redis/redis";
 import { backOff } from "exponential-backoff";
 import { Model } from "../../..";
 import { logger } from "../../logger";
@@ -163,45 +157,5 @@ export function cleanEvent(obj: unknown): unknown {
   }
 }
 
-export const isNotNullOrUndefined = <T>(
-  val?: T | null,
-): val is Exclude<T, null | undefined> => !isUndefinedOrNull(val);
-
 export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
   val === undefined || val === null;
-
-export const addTracesToTraceUpsertQueue = async (
-  batchResults: BatchResult[],
-  projectId: string,
-): Promise<void> => {
-  const traceEvents: TraceUpsertEventType[] = batchResults
-    .filter((result) => result.type === eventTypes.TRACE_CREATE) // we only have create, no update.
-    .map((result) =>
-      result.result &&
-      typeof result.result === "object" &&
-      "id" in result.result
-        ? // ingestion API only gets traces for one projectId
-          {
-            traceId: result.result.id as string,
-            projectId,
-          }
-        : null,
-    )
-    .filter(isNotNullOrUndefined);
-
-  try {
-    if (env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION && redis) {
-      logger.debug(`Sending ${traceEvents.length} events to worker via Redis`);
-
-      const queue = TraceUpsertQueue.getInstance();
-      if (!queue) {
-        logger.error("TraceUpsertQueue not initialized");
-        return;
-      }
-
-      await queue.addBulk(convertTraceUpsertEventsToRedisEvents(traceEvents));
-    }
-  } catch (error) {
-    logger.error("Error sending events to worker", error);
-  }
-};

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -22,8 +22,11 @@ import { LegacyIngestionEventType, QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
 import { LegacyIngestionQueue } from "../redis/legacyIngestion";
 import { redis } from "../redis/redis";
-import { S3StorageService } from "../services/S3StorageService";
-import { addTracesToTraceUpsertQueue, handleBatch } from "./legacy";
+import { handleBatch } from "./legacy";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { getProcessorForEvent } from "./legacy/EventProcessor";
 import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
 
@@ -32,11 +35,11 @@ export type TokenCountDelegate = (p: {
   text: unknown;
 }) => number | undefined;
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -286,8 +289,6 @@ export const processEventBatch = async (
    *******************/
   const result = await handleBatch(sortedBatch, authCheck, tokenCountDelegate);
 
-  await addTracesToTraceUpsertQueue(result.results, authCheck.scope.projectId);
-
   //  in case we did not return early, we return the result here
   return aggregateBatchResult(
     [...validationErrors, ...authenticationErrors, ...result.errors],

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -31,7 +31,7 @@ import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
 type ProcessTracedEvents = () => Promise<void>;
 
-type TraceParams = {
+export type TraceParams = {
   traceName: string;
   traceId: string;
   projectId: string;
@@ -106,6 +106,7 @@ export async function fetchLLMCompletion(
   if (traceParams) {
     const handler = new CallbackHandler({
       _projectId: traceParams.projectId,
+      _isLocalEventExportEnabled: true,
       tags: traceParams.tags,
     });
 
@@ -196,15 +197,17 @@ export async function fetchLLMCompletion(
     throw new Error("This model provider is not supported.");
   }
 
+  const runConfig = {
+    callbacks: finalCallbacks,
+    runId: traceParams?.traceId,
+    runName: traceParams?.traceName,
+  };
+
   if (params.structuredOutputSchema) {
     return {
       completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
         .withStructuredOutput(params.structuredOutputSchema)
-        .invoke(finalMessages, {
-          callbacks: finalCallbacks,
-          runId: traceParams?.traceId,
-          runName: traceParams?.traceName,
-        }),
+        .invoke(finalMessages, runConfig),
       processTracedEvents,
     };
   }
@@ -238,6 +241,7 @@ export async function fetchLLMCompletion(
         .pipe(new StringOutputParser())
         .invoke(
           finalMessages.filter((message) => message._getType() !== "system"),
+          runConfig,
         ),
       processTracedEvents,
     };
@@ -247,7 +251,7 @@ export async function fetchLLMCompletion(
     return {
       completion: await chatModel
         .pipe(new BytesOutputParser())
-        .stream(finalMessages),
+        .stream(finalMessages, runConfig),
       processTracedEvents,
     };
   }
@@ -255,7 +259,7 @@ export async function fetchLLMCompletion(
   return {
     completion: await chatModel
       .pipe(new StringOutputParser())
-      .invoke(finalMessages),
+      .invoke(finalMessages, runConfig),
     processTracedEvents,
   };
 }

packages/shared/src/server/llm/types.ts

@@ -26,6 +26,20 @@ export enum ChatMessageRole {
 
 export const ChatMessageDefaultRoleSchema = z.nativeEnum(ChatMessageRole);
 
+const ChatMessageSchema = z.object({
+  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
+  content: z.string(),
+});
+
+export const ChatMessageListSchema = z.array(ChatMessageSchema);
+export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+
+export const PromptContentSchema = z.union([
+  ChatMessageListSchema,
+  TextPromptSchema,
+]);
+export type PromptContent = z.infer<typeof PromptContentSchema>;
+
 export type ModelParams = {
   provider: string;
   adapter: LLMAdapter;
@@ -49,6 +63,17 @@ export const ZodModelConfig = z.object({
   top_p: z.coerce.number().optional(),
 });
 
+// Experiment config
+export const ExperimentMetadataSchema = z
+  .object({
+    prompt_id: z.string(),
+    provider: z.string(),
+    model: z.string(),
+    model_params: ZodModelConfig,
+  })
+  .strict();
+export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
+
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/playground#openai-playground--anthropic-playground
 export const openAIModels = [
   "gpt-4o",

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -1,10 +1,13 @@
 import { filterOperators } from "../../../interfaces/filters";
 import { clickhouseCompliantRandomCharacters } from "../../repositories";
 
+export type ClickhouseOperator =
+  | (typeof filterOperators)[keyof typeof filterOperators][number]
+  | "!=";
 export interface Filter {
   apply(): ClickhouseFilter;
   clickhouseTable: string;
-  operator: (typeof filterOperators)[keyof typeof filterOperators][number];
+  operator: ClickhouseOperator;
   field: string;
 }
 type ClickhouseFilter = {
@@ -69,28 +72,32 @@ export class NumberFilter implements Filter {
   public clickhouseTable: string;
   public field: string;
   public value: number;
-  public operator: (typeof filterOperators)["number"][number];
+  public operator: (typeof filterOperators)["number"][number] | "!=";
+  public clickhouseTypeOverwrite?: string;
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["number"][number];
+    operator: (typeof filterOperators)["number"][number] | "!=";
     value: number;
     tablePrefix?: string;
+    clickhouseTypeOverwrite?: string;
   }) {
     this.clickhouseTable = opts.clickhouseTable;
     this.field = opts.field;
     this.value = opts.value;
     this.operator = opts.operator;
     this.tablePrefix = opts.tablePrefix;
+    this.clickhouseTypeOverwrite = opts.clickhouseTypeOverwrite;
   }
 
   apply(): ClickhouseFilter {
     const uid = clickhouseCompliantRandomCharacters();
     const varName = `numberFilter${uid}`;
+    const type = this.clickhouseTypeOverwrite ?? "Decimal64(12)";
     return {
-      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: Decimal64(12)}`,
+      query: `${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field} ${this.operator} {${varName}: ${type}}`,
       params: { [varName]: this.value.toString() },
     };
   }
@@ -256,7 +263,7 @@ export class ArrayOptionsFilter implements Filter {
         query = `hasAny({${varName}: Array(String)}, ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = False`;
         break;
       case "all of":
-        query = `arrayAll(x -> has({${varName}: Array(String)}, x), ${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}) = True`;
+        query = `hasAll(${this.tablePrefix ? this.tablePrefix + "." : ""}${this.field}, {${varName}: Array(String)}) = True`;
         break;
       default:
         throw new Error(`Unsupported operator: ${this.operator}`);
@@ -300,13 +307,13 @@ export class NumberObjectFilter implements Filter {
   public field: string;
   public key: string;
   public value: number;
-  public operator: (typeof filterOperators)["numberObject"][number];
+  public operator: (typeof filterOperators)["numberObject"][number] | "!=";
   protected tablePrefix?: string;
 
   constructor(opts: {
     clickhouseTable: string;
     field: string;
-    operator: (typeof filterOperators)["numberObject"][number];
+    operator: (typeof filterOperators)["numberObject"][number] | "!=";
     key: string;
     value: number;
     tablePrefix?: string;
@@ -364,7 +371,7 @@ export class BooleanFilter implements Filter {
 export class FilterList {
   private filters: Filter[];
 
-  constructor(filters: Filter[]) {
+  constructor(filters: Filter[] = []) {
     this.filters = filters;
   }
 
@@ -376,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  length() {
+    return this.filters.length;
+  }
+
   public apply(): ClickhouseFilter {
     if (this.filters.length === 0) {
       return {

packages/shared/src/server/queries/clickhouse-sql/factory.ts

@@ -67,6 +67,7 @@ export const createFilterFromFilterState = (
           operator: frontEndFilter.operator,
           value: frontEndFilter.value,
           tablePrefix: column.queryPrefix,
+          clickhouseTypeOverwrite: column.clickhouseTypeOverwrite,
         });
       case "arrayOptions":
         return new ArrayOptionsFilter({

packages/shared/src/server/queries/index.ts

@@ -7,3 +7,16 @@ export {
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,
 } from "./createGenerationsQuery";
+export {
+  FilterList,
+  StringFilter,
+  DateTimeFilter,
+  StringOptionsFilter,
+  NumberFilter,
+  ArrayOptionsFilter,
+  BooleanFilter,
+  NumberObjectFilter,
+  StringObjectFilter,
+  NullFilter,
+  type ClickhouseOperator,
+} from "./clickhouse-sql/clickhouse-filter";

packages/shared/src/server/queues.ts

@@ -7,6 +7,7 @@ export enum EventName {
   EvaluationExecution = "EvaluationExecution",
   LegacyIngestion = "LegacyIngestion",
   CloudUsageMetering = "CloudUsageMetering",
+  ExperimentCreate = "ExperimentCreate",
 }
 
 export const LegacyIngestionEventFull = z.object({
@@ -78,6 +79,13 @@ export const EvalExecutionEvent = z.object({
   delay: z.number().nullish(),
 });
 
+export const ExperimentCreateEventSchema = z.object({
+  projectId: z.string(),
+  datasetId: z.string(),
+  runId: z.string(),
+  description: z.string().optional(),
+});
+
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceUpsertEventType = z.infer<typeof TraceUpsertEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
@@ -86,6 +94,9 @@ export type DatasetRunItemUpsertEventType = z.infer<
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
 export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
+export type ExperimentCreateEventType = z.infer<
+  typeof ExperimentCreateEventSchema
+>;
 
 export const EventBodySchema = z.union([
   z.object({
@@ -100,6 +111,10 @@ export const EventBodySchema = z.union([
     name: z.literal(EventName.BatchExport),
     payload: BatchExportJobSchema,
   }),
+  z.object({
+    name: z.literal(EventName.ExperimentCreate),
+    payload: ExperimentCreateEventSchema,
+  }),
 ]);
 export type EventBodyType = z.infer<typeof EventBodySchema>;
 
@@ -111,6 +126,7 @@ export enum QueueName {
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
   LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  ExperimentCreate = "experiment-create-queue",
 }
 
 export enum QueueJobs {
@@ -122,6 +138,7 @@ export enum QueueJobs {
   LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  ExperimentCreateJob = "experiment-create-job",
 }
 
 export type TQueueJobTypes = {
@@ -161,4 +178,10 @@ export type TQueueJobTypes = {
     payload: IngestionEventQueueType;
     name: QueueJobs.IngestionJob;
   };
+  [QueueName.ExperimentCreate]: {
+    timestamp: Date;
+    id: string;
+    payload: ExperimentCreateEventType;
+    name: QueueJobs.ExperimentCreateJob;
+  };
 };

packages/shared/src/server/redis/CloudUsageMeteringQueue.ts

@@ -0,0 +1,61 @@
+import { Queue } from "bullmq";
+import { env } from "../..";
+import { logger } from "@azure/storage-blob";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class CloudUsageMeteringQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudUsageMeteringQueue.instance) {
+      return CloudUsageMeteringQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudUsageMeteringQueue.instance = newRedis
+      ? new Queue(QueueName.CloudUsageMeteringQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudUsageMeteringQueue.instance?.on("error", (err) => {
+      logger.error("CloudUsageMeteringQueue error", err);
+    });
+
+    if (CloudUsageMeteringQueue.instance) {
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {
+          repeat: { pattern: "5 * * * *" },
+        },
+      );
+
+      CloudUsageMeteringQueue.instance.add(
+        QueueJobs.CloudUsageMeteringJob,
+        {},
+        {},
+      );
+    }
+
+    return CloudUsageMeteringQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -27,7 +27,8 @@ export class DatasetRunItemUpsertQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 5,
+              delay: 30_000, // 30 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/redis/evalExecutionQueue.ts

@@ -26,7 +26,7 @@ export class EvalExecutionQueue {
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 10_000,
-              attempts: 2,
+              attempts: 10,
               backoff: {
                 type: "exponential",
                 delay: 5000,

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -0,0 +1,45 @@
+import { Queue } from "bullmq";
+import { logger } from "../logger";
+import { TQueueJobTypes, QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+
+export class ExperimentCreateQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.ExperimentCreate]
+  > | null {
+    if (ExperimentCreateQueue.instance) return ExperimentCreateQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    ExperimentCreateQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.ExperimentCreate]>(
+          QueueName.ExperimentCreate,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 10_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    ExperimentCreateQueue.instance?.on("error", (err) => {
+      logger.error("ExperimentCreateQueue error", err);
+    });
+
+    return ExperimentCreateQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -0,0 +1,34 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { BatchExportQueue } from "./batchExport";
+import { CloudUsageMeteringQueue } from "./CloudUsageMeteringQueue";
+import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
+import { EvalExecutionQueue } from "./evalExecutionQueue";
+import { ExperimentCreateQueue } from "./experimentCreateQueue";
+import { IngestionQueue } from "./ingestionQueue";
+import { LegacyIngestionQueue } from "./legacyIngestion";
+import { TraceUpsertQueue } from "./traceUpsert";
+
+export function getQueue(queueName: QueueName): Queue | null {
+  switch (queueName) {
+    case QueueName.LegacyIngestionQueue:
+      return LegacyIngestionQueue.getInstance();
+    case QueueName.BatchExport:
+      return BatchExportQueue.getInstance();
+    case QueueName.CloudUsageMeteringQueue:
+      return CloudUsageMeteringQueue.getInstance();
+    case QueueName.DatasetRunItemUpsert:
+      return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.EvaluationExecution:
+      return EvalExecutionQueue.getInstance();
+    case QueueName.ExperimentCreate:
+      return ExperimentCreateQueue.getInstance();
+    case QueueName.TraceUpsert:
+      return TraceUpsertQueue.getInstance();
+    case QueueName.IngestionQueue:
+      return IngestionQueue.getInstance();
+    default:
+      const exhaustiveCheckDefault: never = queueName;
+      throw new Error(`Queue ${queueName} not found`);
+  }
+}

packages/shared/src/server/redis/traceUpsert.ts

@@ -32,6 +32,7 @@ export class TraceUpsertQueue {
               removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
               removeOnFail: 100_000,
               attempts: 5,
+              delay: 10_000, // 10 seconds
               backoff: {
                 type: "exponential",
                 delay: 5000,
@@ -48,43 +49,3 @@ export class TraceUpsertQueue {
     return TraceUpsertQueue.instance;
   }
 }
-
-export function convertTraceUpsertEventsToRedisEvents(
-  events: TraceUpsertEventType[],
-) {
-  const uniqueTracesPerProject = events.reduce((acc, event) => {
-    if (!acc.get(event.projectId)) {
-      acc.set(event.projectId, new Set());
-    }
-    acc.get(event.projectId)?.add(event.traceId);
-    return acc;
-  }, new Map<string, Set<string>>());
-
-  return [...uniqueTracesPerProject.entries()]
-    .map((tracesPerProject) => {
-      const [projectId, traceIds] = tracesPerProject;
-
-      return [...traceIds].map((traceId) => ({
-        name: QueueJobs.TraceUpsert,
-        data: {
-          payload: {
-            projectId,
-            traceId,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.TraceUpsert as const,
-        },
-        opts: {
-          removeOnFail: 1_000,
-          removeOnComplete: true,
-          attempts: 5,
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-        },
-      }));
-    })
-    .flat();
-}

packages/shared/src/server/repositories/clickhouse.ts

@@ -5,16 +5,19 @@ import {
 } from "../clickhouse/client";
 import { logger } from "../logger";
 import { instrumentAsync } from "../instrumentation";
-import { S3StorageService } from "../services/S3StorageService";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
-const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
@@ -29,7 +32,7 @@ const getS3StorageServiceClient = (bucketName: string): S3StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces"; // TODO: Modify eventType logic to support more tables going forward
+  table: "scores" | "traces" | "observations";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>;
 }): Promise<void> {
@@ -37,10 +40,6 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    // drop trailing s and pretend it's always a create.
-    // Only applicable to scores and traces.
-    const eventType = `${opts.table.slice(0, -1)}-create`;
-
     // If event upload is enabled, we store all rows in S3 to have a backup
     if (env.LANGFUSE_S3_EVENT_UPLOAD_ENABLED === "true") {
       if (env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET === undefined) {
@@ -51,6 +50,13 @@ export async function upsertClickhouse<
       );
       await Promise.all(
         opts.records.map((record) => {
+          // drop trailing s and pretend it's always a create.
+          // Only applicable to scores and traces.
+          let eventType = `${opts.table.slice(0, -1)}-create`;
+          if (opts.table === "observations") {
+            // @ts-ignore - If it's an observation we now that `type` is a string
+            eventType = `${record["type"].toLowerCase()}-create`;
+          }
           s3Client.uploadJson(
             `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
             [

packages/shared/src/server/repositories/definitions.ts

@@ -129,8 +129,8 @@ export const scoreRecordBaseSchema = z.object({
   project_id: z.string(),
   trace_id: z.string(),
   observation_id: z.string().nullish(),
-  name: z.string().nullish(),
-  value: z.union([z.number(), z.string()]).nullish(),
+  name: z.string(),
+  value: z.number().nullish(),
   source: z.string(),
   comment: z.string().nullish(),
   author_user_id: z.string().nullish(),

packages/shared/src/server/repositories/observations.ts

@@ -2,6 +2,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  upsertClickhouse,
 } from "./clickhouse";
 import { ObservationLevel } from "@prisma/client";
 import { logger } from "../logger";
@@ -38,6 +39,58 @@ import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
 } from "./constants";
 
+export const checkObservationExists = async (
+  projectId: string,
+  id: string,
+  startTime: Date | undefined,
+): Promise<boolean> => {
+  const query = `
+    SELECT id, project_id
+    FROM observations o
+    WHERE project_id = {projectId: String}
+    AND id = {id: String}
+    ${startTime ? `AND start_time >= {startTime: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    ORDER BY event_ts DESC
+    LIMIT 1 BY id, project_id
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      id,
+      projectId,
+      ...(startTime
+        ? { startTime: convertDateToClickhouseDateTime(startTime) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
+
+/**
+ * Accepts a trace in a Clickhouse-ready format.
+ * id, project_id, and timestamp must always be provided.
+ */
+export const upsertObservation = async (
+  observation: Partial<ObservationRecordReadType>,
+) => {
+  if (
+    !["id", "project_id", "start_time", "type"].every(
+      (key) => key in observation,
+    )
+  ) {
+    throw new Error(
+      "Identifier fields must be provided to upsert Observation.",
+    );
+  }
+  await upsertClickhouse({
+    table: "observations",
+    records: [observation as ObservationRecordReadType],
+    eventBodyMapper: convertObservation,
+  });
+};
+
 export const getObservationsViewForTrace = async (
   traceId: string,
   projectId: string,
@@ -94,6 +147,65 @@ export const getObservationsViewForTrace = async (
   return records.map(convertObservationToView);
 };
 
+export const getObservationForTraceIdByName = async (
+  traceId: string,
+  projectId: string,
+  name: string,
+  timestamp?: Date,
+  fetchWithInputOutput: boolean = false,
+) => {
+  const query = `
+  SELECT
+    id,
+    trace_id,
+    project_id,
+    type,
+    parent_observation_id,
+    start_time,
+    end_time,
+    name,
+    metadata,
+    level,
+    status_message,
+    version,
+    ${fetchWithInputOutput ? "input, output," : ""}
+    provided_model_name,
+    internal_model_id,
+    model_parameters,
+    provided_usage_details,
+    usage_details,
+    provided_cost_details,
+    cost_details,
+    total_cost,
+    completion_start_time,
+    prompt_id,
+    prompt_name,
+    prompt_version,
+    created_at,
+    updated_at,
+    event_ts
+  FROM observations 
+  WHERE trace_id = {traceId: String}
+  AND project_id = {projectId: String}
+  AND name = {name: String}
+   ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  ORDER BY event_ts DESC
+  LIMIT 1 BY id, project_id`;
+  const records = await queryClickhouse<ObservationRecordReadType>({
+    query,
+    params: {
+      traceId,
+      projectId,
+      name,
+      ...(timestamp
+        ? { traceTimestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return records.map(convertObservationToView);
+};
+
 export const getObservationById = async (
   id: string,
   projectId: string,

packages/shared/src/server/repositories/observations_converters.ts

@@ -3,11 +3,13 @@ import {
   ObservationView,
   ObservationType,
   ObservationLevel,
+  Prisma,
 } from "@prisma/client";
 import Decimal from "decimal.js";
-import { jsonSchema } from "../../utils/zod";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
+import { parseJsonPrioritised } from "../../utils/json";
+import { jsonSchema } from "../../utils/zod";
 
 export const convertObservationToView = (
   record: ObservationRecordReadType,
@@ -56,8 +58,12 @@ export const convertObservation = (
     level: record.level as ObservationLevel,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
-    input: jsonSchema.nullish().parse(record.input) ?? null,
-    output: jsonSchema.nullish().parse(record.output) ?? null,
+    input: (record.input
+      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      : null) as Prisma.JsonValue | null,
     modelParameters: record.model_parameters
       ? JSON.parse(record.model_parameters)
       : null,

packages/shared/src/server/repositories/scores.ts

@@ -24,28 +24,7 @@ import {
 } from "./scores_converters";
 import { SCORE_TO_TRACE_OBSERVATIONS_INTERVAL } from "./constants";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
-
-export type FetchScoresReturnType = {
-  id: string;
-  timestamp: string;
-  project_id: string;
-  trace_id: string;
-  observation_id: string | null;
-  name: string;
-  value: number;
-  source: string;
-  comment: string | null;
-  author_user_id: string | null;
-  config_id: string | null;
-  data_type: string;
-  string_value: string | null;
-  queue_id: string | null;
-  created_at: string;
-  updated_at: string;
-  event_ts: string;
-  is_deleted: number;
-  projectId: string;
-};
+import { ScoreRecordReadType } from "./definitions";
 
 export const searchExistingAnnotationScore = async (
   projectId: string,
@@ -74,7 +53,7 @@ export const searchExistingAnnotationScore = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -103,7 +82,7 @@ export const getScoreById = async (
     LIMIT 1
   `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query,
     params: {
       projectId,
@@ -118,13 +97,13 @@ export const getScoreById = async (
  * Accepts a score in a Clickhouse-ready format.
  * id, project_id, name, and timestamp must always be provided.
  */
-export const upsertScore = async (score: Partial<FetchScoresReturnType>) => {
+export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   if (!["id", "project_id", "name", "timestamp"].every((key) => key in score)) {
     throw new Error("Identifier fields must be provided to upsert Score.");
   }
   await upsertClickhouse({
     table: "scores",
-    records: [score as FetchScoresReturnType],
+    records: [score as ScoreRecordReadType],
     eventBodyMapper: convertToScore,
   });
 };
@@ -148,7 +127,7 @@ export const getScoresForTraces = async (
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId,
@@ -181,7 +160,7 @@ export const getScoresForObservations = async (
       ${limit !== undefined && offset !== undefined ? `limit {limit: Int32} offset {offset: Int32}` : ""}
     `;
 
-  const rows = await queryClickhouse<FetchScoresReturnType>({
+  const rows = await queryClickhouse<ScoreRecordReadType>({
     query: query,
     params: {
       projectId: projectId,

packages/shared/src/server/repositories/scores_converters.ts

@@ -1,5 +1,5 @@
 import { ScoreSource, ScoreDataType } from "@prisma/client";
-import { FetchScoresReturnType } from "./scores";
+import { ScoreRecordReadType } from "./definitions";
 
 export type ScoreAggregation = {
   id: string;
@@ -11,22 +11,22 @@ export type ScoreAggregation = {
   comment: string | null;
 };
 
-export const convertToScore = (row: FetchScoresReturnType) => {
+export const convertToScore = (row: ScoreRecordReadType) => {
   return {
     id: row.id,
     timestamp: new Date(row.timestamp),
     projectId: row.project_id,
     traceId: row.trace_id,
-    observationId: row.observation_id,
+    observationId: row.observation_id ?? null,
     name: row.name,
-    value: row.value,
+    value: row.value ?? null,
     source: row.source as ScoreSource,
-    comment: row.comment,
-    authorUserId: row.author_user_id,
-    configId: row.config_id,
+    comment: row.comment ?? null,
+    authorUserId: row.author_user_id ?? null,
+    configId: row.config_id ?? null,
     dataType: row.data_type as ScoreDataType,
-    stringValue: row.string_value,
-    queueId: row.queue_id,
+    stringValue: row.string_value ?? null,
+    queueId: row.queue_id ?? null,
     createdAt: new Date(row.created_at),
     updatedAt: new Date(row.updated_at),
   };

packages/shared/src/server/repositories/traces.ts

@@ -12,6 +12,7 @@ import { FilterState } from "../../types";
 import {
   DateTimeFilter,
   FilterList,
+  StringFilter,
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
@@ -25,6 +26,48 @@ import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
 import { TRACE_TO_OBSERVATIONS_INTERVAL } from "./constants";
 import { FetchTracesTableProps } from "../services/traces-ui-table-service";
 
+export const checkTraceExists = async (
+  projectId: string,
+  traceId: string,
+  timestamp: Date | undefined,
+  filter: FilterState,
+): Promise<boolean> => {
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "t",
+  });
+
+  tracesFilter.push(
+    ...createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+    new StringFilter({
+      clickhouseTable: "t",
+      field: "id",
+      operator: "=",
+      value: traceId,
+    }),
+  );
+
+  const tracesFilterRes = tracesFilter.apply();
+
+  const query = `
+    SELECT id, project_id
+    FROM traces t FINAL
+    WHERE ${tracesFilterRes.query}
+    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+  `;
+
+  const rows = await queryClickhouse<{ id: string; project_id: string }>({
+    query,
+    params: {
+      ...tracesFilterRes.params,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
+    },
+  });
+
+  return rows.length > 0;
+};
+
 /**
  * Accepts a trace in a Clickhouse-ready format.
  * id, project_id, and timestamp must always be provided.

packages/shared/src/server/repositories/traces_converters.ts

@@ -81,7 +81,7 @@ export const convertToDomain = (row: TracesTableReturnType) => {
     version: row.version ?? null,
     userId: row.user_id ?? null,
     sessionId: row.session_id ?? null,
-    latencyMilliseconds: Number(row.latency_milliseconds),
+    latency: Number(row.latency),
     usageDetails: row.usage_details,
     costDetails: row.cost_details,
     level: row.level,

packages/shared/src/server/services/S3StorageService.ts

@@ -1,172 +0,0 @@
-import type { Readable } from "stream";
-import {
-  GetObjectCommand,
-  ListObjectsV2Command,
-  PutObjectCommand,
-  S3Client,
-} from "@aws-sdk/client-s3";
-import { Upload } from "@aws-sdk/lib-storage";
-import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { logger } from "../logger";
-
-type UploadFile = {
-  fileName: string;
-  fileType: string;
-  data: Readable | string;
-  expiresInSeconds: number;
-};
-
-export class S3StorageService {
-  private client: S3Client;
-  private bucketName: string;
-
-  constructor(params: {
-    accessKeyId: string | undefined;
-    secretAccessKey: string | undefined;
-    bucketName: string;
-    endpoint: string | undefined;
-    region: string | undefined;
-    forcePathStyle: boolean;
-  }) {
-    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
-    const { accessKeyId, secretAccessKey } = params;
-    const credentials =
-      accessKeyId !== undefined && secretAccessKey !== undefined
-        ? {
-            accessKeyId,
-            secretAccessKey,
-          }
-        : undefined;
-
-    this.client = new S3Client({
-      credentials,
-      endpoint: params.endpoint,
-      region: params.region,
-      forcePathStyle: params.forcePathStyle,
-    });
-    this.bucketName = params.bucketName;
-  }
-
-  public async uploadFile({
-    fileName,
-    fileType,
-    data,
-    expiresInSeconds,
-  }: UploadFile): Promise<{ signedUrl: string }> {
-    try {
-      await new Upload({
-        client: this.client,
-        params: {
-          Bucket: this.bucketName,
-          Key: fileName,
-          Body: data,
-          ContentType: fileType,
-        },
-      }).done();
-
-      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
-
-      return { signedUrl };
-    } catch (err) {
-      logger.error(`Failed to upload file to ${fileName}`, err);
-      throw new Error("Failed to upload to S3 or generate signed URL");
-    }
-  }
-
-  public async uploadJson(path: string, body: Record<string, unknown>[]) {
-    const putCommand = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-      Body: JSON.stringify(body),
-      ContentType: "application/json",
-    });
-
-    try {
-      await this.client.send(putCommand);
-    } catch (err) {
-      logger.error(`Failed to upload JSON to S3 ${path}`, err);
-      throw Error("Failed to upload JSON to S3");
-    }
-  }
-
-  public async download(path: string): Promise<string> {
-    const getCommand = new GetObjectCommand({
-      Bucket: this.bucketName,
-      Key: path,
-    });
-
-    try {
-      const response = await this.client.send(getCommand);
-      return (await response.Body?.transformToString()) ?? "";
-    } catch (err) {
-      logger.error(`Failed to download file from S3 ${path}`, err);
-      throw Error("Failed to download file from S3");
-    }
-  }
-
-  public async listFiles(prefix: string): Promise<string[]> {
-    const listCommand = new ListObjectsV2Command({
-      Bucket: this.bucketName,
-      Prefix: prefix,
-    });
-
-    try {
-      const response = await this.client.send(listCommand);
-      return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
-      );
-    } catch (err) {
-      logger.error(`Failed to list files from S3 ${prefix}`, err);
-      throw Error("Failed to list files from S3");
-    }
-  }
-
-  public async getSignedUrl(
-    fileName: string,
-    ttlSeconds: number,
-    asAttachment: boolean = true,
-  ): Promise<string> {
-    try {
-      return await getSignedUrl(
-        this.client,
-        new GetObjectCommand({
-          Bucket: this.bucketName,
-          Key: fileName,
-          ResponseContentDisposition: asAttachment
-            ? `attachment; filename="${fileName}"`
-            : undefined,
-        }),
-        { expiresIn: ttlSeconds },
-      );
-    } catch (err) {
-      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
-      throw Error("Failed to generate signed URL");
-    }
-  }
-
-  public async getSignedUploadUrl(params: {
-    path: string;
-    ttlSeconds: number;
-    sha256Hash: string;
-    contentType: string;
-    contentLength: number;
-  }): Promise<string> {
-    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
-
-    return await getSignedUrl(
-      this.client,
-      new PutObjectCommand({
-        Bucket: this.bucketName,
-        Key: path,
-        ContentType: contentType,
-        ChecksumSHA256: sha256Hash,
-        ContentLength: contentLength,
-      }),
-      {
-        expiresIn: ttlSeconds,
-        signableHeaders: new Set(["content-type", "content-length"]),
-        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
-      },
-    );
-  }
-}

packages/shared/src/server/services/StorageService.ts

@@ -0,0 +1,432 @@
+import { Readable } from "stream";
+import {
+  GetObjectCommand,
+  ListObjectsV2Command,
+  PutObjectCommand,
+  S3Client,
+} from "@aws-sdk/client-s3";
+import { Upload } from "@aws-sdk/lib-storage";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from "@azure/storage-blob";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+type UploadFile = {
+  fileName: string;
+  fileType: string;
+  data: Readable | string;
+  expiresInSeconds: number;
+};
+
+export interface StorageService {
+  uploadFile(params: UploadFile): Promise<{ signedUrl: string }>;
+
+  uploadJson(path: string, body: Record<string, unknown>[]): Promise<void>;
+
+  download(path: string): Promise<string>;
+
+  listFiles(prefix: string): Promise<string[]>;
+
+  getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string>;
+
+  getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string>;
+}
+
+export class StorageServiceFactory {
+  public static getInstance(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }): StorageService {
+    if (env.LANGFUSE_USE_AZURE_BLOB === "true") {
+      return new AzureBlobStorageService(params);
+    }
+    return new S3StorageService(params);
+  }
+}
+
+class AzureBlobStorageService implements StorageService {
+  private client: ContainerClient;
+  private container: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    const { accessKeyId, secretAccessKey, endpoint } = params;
+    if (!accessKeyId || !secretAccessKey || !endpoint) {
+      throw new Error(
+        `Endpoint, account and account key must be configured to use Azure Blob Storage`,
+      );
+    }
+
+    const sharedKeyCredential = new StorageSharedKeyCredential(
+      accessKeyId,
+      secretAccessKey,
+    );
+    const blobServiceClient = new BlobServiceClient(
+      endpoint,
+      sharedKeyCredential,
+    );
+    this.container = params.bucketName;
+    this.client = blobServiceClient.getContainerClient(this.container);
+  }
+
+  private async createContainerIfNotExists(): Promise<void> {
+    try {
+      await this.client.createIfNotExists();
+    } catch (err) {
+      logger.error(
+        `Failed to create Azure Blob Storage container ${this.container}`,
+        err,
+      );
+      throw Error("Failed to create Azure Blob Storage container ");
+    }
+  }
+
+  public async uploadFile(params: UploadFile): Promise<{ signedUrl: string }> {
+    const { fileName, data, expiresInSeconds } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+
+      if (typeof data === "string") {
+        await blockBlobClient.upload(data, data.length);
+      } else if (data instanceof Readable) {
+        let offset = 0;
+        const blockIds = [];
+        for await (const chunk of data) {
+          const blockId = Buffer.from(`block-${offset}`).toString("base64");
+          const bufferChunk = Buffer.isBuffer(chunk)
+            ? chunk
+            : Buffer.from(chunk);
+
+          await blockBlobClient.stageBlock(
+            blockId,
+            bufferChunk,
+            bufferChunk.length,
+          );
+          blockIds.push(blockId);
+
+          offset += bufferChunk.length;
+        }
+
+        await blockBlobClient.commitBlockList(blockIds);
+      } else {
+        throw new Error("Unsupported data type. Must be Readable or string.");
+      }
+
+      return {
+        signedUrl: await this.getSignedUrl(fileName, expiresInSeconds, false),
+      };
+    } catch (err) {
+      logger.error(
+        `Failed to upload file to Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to upload file to Azure Blob Storage");
+    }
+  }
+
+  public async uploadJson(
+    path: string,
+    body: Record<string, unknown>[],
+  ): Promise<void> {
+    await this.createContainerIfNotExists();
+
+    const blockBlobClient = this.client.getBlockBlobClient(path);
+    const content = JSON.stringify(body);
+    try {
+      await blockBlobClient.upload(content, content.length);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to Azure Blob Storage ${path}`, err);
+      throw Error("Failed to upload JSON to Azure Blob Storage");
+    }
+  }
+
+  private async streamToString(
+    readableStream: NodeJS.ReadableStream,
+  ): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const chunks: string[] = [];
+      readableStream.on("data", (data) => {
+        chunks.push(data.toString());
+      });
+      readableStream.on("end", () => {
+        resolve(chunks.join(""));
+      });
+      readableStream.on("error", reject);
+    });
+  }
+
+  public async download(path: string): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blobClient = this.client.getBlobClient(path);
+      const downloadResponse = await blobClient.download();
+      if (!downloadResponse.readableStreamBody) {
+        throw Error("No stream body available");
+      }
+      return this.streamToString(downloadResponse.readableStreamBody);
+    } catch (err) {
+      logger.error(
+        `Failed to download file from Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error("Failed to download file from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const result = await this.client.listBlobsFlat({ prefix });
+      const files = [];
+      for await (const blob of result) {
+        if (blob.name.startsWith(prefix)) {
+          files.push(blob.name);
+        }
+      }
+      return files;
+    } catch (err) {
+      logger.error(
+        `Failed to list files from Azure Blob Storage ${prefix}`,
+        err,
+      );
+      throw Error("Failed to list files from Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment?: boolean,
+  ): Promise<string> {
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(fileName);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("r"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentDisposition: asAttachment
+          ? `attachment; filename="${fileName}"`
+          : undefined,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned URL for Azure Blob Storage ${fileName}`,
+        err,
+      );
+      throw Error("Failed to generate presigned URL for Azure Blob Storage");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType } = params;
+    try {
+      await this.createContainerIfNotExists();
+
+      const blockBlobClient = this.client.getBlockBlobClient(path);
+      return blockBlobClient.generateSasUrl({
+        permissions: BlobSASPermissions.parse("w"),
+        expiresOn: new Date(Date.now() + ttlSeconds * 1000),
+        contentType: contentType,
+      });
+    } catch (err) {
+      logger.error(
+        `Failed to generate presigned upload URL for Azure Blob Storage ${path}`,
+        err,
+      );
+      throw Error(
+        "Failed to generate presigned upload URL for Azure Blob Storage",
+      );
+    }
+  }
+}
+
+class S3StorageService implements StorageService {
+  private client: S3Client;
+  private bucketName: string;
+
+  constructor(params: {
+    accessKeyId: string | undefined;
+    secretAccessKey: string | undefined;
+    bucketName: string;
+    endpoint: string | undefined;
+    region: string | undefined;
+    forcePathStyle: boolean;
+  }) {
+    // Use accessKeyId and secretAccessKey if provided or fallback to default credentials
+    const { accessKeyId, secretAccessKey } = params;
+    const credentials =
+      accessKeyId !== undefined && secretAccessKey !== undefined
+        ? {
+            accessKeyId,
+            secretAccessKey,
+          }
+        : undefined;
+
+    this.client = new S3Client({
+      credentials,
+      endpoint: params.endpoint,
+      region: params.region,
+      forcePathStyle: params.forcePathStyle,
+    });
+    this.bucketName = params.bucketName;
+  }
+
+  public async uploadFile({
+    fileName,
+    fileType,
+    data,
+    expiresInSeconds,
+  }: UploadFile): Promise<{ signedUrl: string }> {
+    try {
+      await new Upload({
+        client: this.client,
+        params: {
+          Bucket: this.bucketName,
+          Key: fileName,
+          Body: data,
+          ContentType: fileType,
+        },
+      }).done();
+
+      const signedUrl = await this.getSignedUrl(fileName, expiresInSeconds);
+
+      return { signedUrl };
+    } catch (err) {
+      logger.error(`Failed to upload file to ${fileName}`, err);
+      throw new Error("Failed to upload to S3 or generate signed URL");
+    }
+  }
+
+  public async uploadJson(path: string, body: Record<string, unknown>[]) {
+    const putCommand = new PutObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+      Body: JSON.stringify(body),
+      ContentType: "application/json",
+    });
+
+    try {
+      await this.client.send(putCommand);
+    } catch (err) {
+      logger.error(`Failed to upload JSON to S3 ${path}`, err);
+      throw Error("Failed to upload JSON to S3");
+    }
+  }
+
+  public async download(path: string): Promise<string> {
+    const getCommand = new GetObjectCommand({
+      Bucket: this.bucketName,
+      Key: path,
+    });
+
+    try {
+      const response = await this.client.send(getCommand);
+      return (await response.Body?.transformToString()) ?? "";
+    } catch (err) {
+      logger.error(`Failed to download file from S3 ${path}`, err);
+      throw Error("Failed to download file from S3");
+    }
+  }
+
+  public async listFiles(prefix: string): Promise<string[]> {
+    const listCommand = new ListObjectsV2Command({
+      Bucket: this.bucketName,
+      Prefix: prefix,
+    });
+
+    try {
+      const response = await this.client.send(listCommand);
+      return (
+        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
+      );
+    } catch (err) {
+      logger.error(`Failed to list files from S3 ${prefix}`, err);
+      throw Error("Failed to list files from S3");
+    }
+  }
+
+  public async getSignedUrl(
+    fileName: string,
+    ttlSeconds: number,
+    asAttachment: boolean = true,
+  ): Promise<string> {
+    try {
+      return await getSignedUrl(
+        this.client,
+        new GetObjectCommand({
+          Bucket: this.bucketName,
+          Key: fileName,
+          ResponseContentDisposition: asAttachment
+            ? `attachment; filename="${fileName}"`
+            : undefined,
+        }),
+        { expiresIn: ttlSeconds },
+      );
+    } catch (err) {
+      logger.error(`Failed to generate presigned URL for ${fileName}`, err);
+      throw Error("Failed to generate signed URL");
+    }
+  }
+
+  public async getSignedUploadUrl(params: {
+    path: string;
+    ttlSeconds: number;
+    sha256Hash: string;
+    contentType: string;
+    contentLength: number;
+  }): Promise<string> {
+    const { path, ttlSeconds, contentType, contentLength, sha256Hash } = params;
+
+    return await getSignedUrl(
+      this.client,
+      new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: path,
+        ContentType: contentType,
+        ChecksumSHA256: sha256Hash,
+        ContentLength: contentLength,
+      }),
+      {
+        expiresIn: ttlSeconds,
+        signableHeaders: new Set(["content-type", "content-length"]),
+        unhoistableHeaders: new Set(["x-amz-checksum-sha256"]),
+      },
+    );
+  }
+}

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -33,7 +33,7 @@ export type TracesTableReturnType = Pick<
 > & {
   level: ObservationLevel;
   observation_count: number | null;
-  latency_milliseconds: string | null;
+  latency: string | null;
   usage_details: Record<string, number>;
   cost_details: Record<string, number>;
   scores_avg: Array<{ name: string; avg_value: number }>;
@@ -89,7 +89,7 @@ export const getTracesTable = async (
     t.version, 
     t.user_id, 
     t.session_id,
-    os.latency_milliseconds,
+    os.latency_milliseconds / 1000 as latency,
     os.cost_details as cost_details,
     os.usage_details as usage_details,
     os.level as level,

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -69,6 +69,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(completion_start_time), NULL,  date_diff('seconds', start_time, completion_start_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Latency (s)",
@@ -76,6 +78,8 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect:
       "if(isNull(end_time), NULL, date_diff('seconds', start_time, end_time))",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Tokens per second",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -106,7 +106,9 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableName: "Latency (s)",
     uiTableId: "latency",
     clickhouseTableName: "traces",
-    clickhouseSelect: "latency_milliseconds",
+    clickhouseSelect: "latency_milliseconds / 1000",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Input Cost ($)",

packages/shared/src/tableDefinitions/tracesTable.ts

@@ -138,16 +138,16 @@ export const tracesTableCols: ColumnDefinition[] = [
   },
 ];
 
-// Used only for dataset evaluator form, not on any table
-export const datasetOnlyCols: ColumnDefinition[] = [
-  {
-    name: "Dataset",
-    id: "datasetId",
-    type: "stringOptions",
-    internal: 'di."dataset_id"',
-    options: [], // to be filled in at runtime
-  },
-];
+export const datasetCol: ColumnDefinition = {
+  name: "Dataset",
+  id: "datasetId",
+  type: "stringOptions",
+  internal: 'di."dataset_id"',
+  options: [], // to be filled in at runtime
+};
+
+// Used only for dataset evaluator, not on dataset table
+export const datasetOnlyCols: ColumnDefinition[] = [datasetCol];
 
 export const evalTraceTableCols: ColumnDefinition[] = tracesOnlyCols;
 export const evalDatasetFormFilterCols: ColumnDefinition[] = datasetOnlyCols;

packages/shared/src/tableDefinitions/types.ts

@@ -3,6 +3,7 @@ export type UiColumnMapping = {
   uiTableId: string;
   clickhouseTableName: string;
   clickhouseSelect: string;
+  clickhouseTypeOverwrite?: string;
   queryPrefix?: string;
 };
 

packages/shared/src/utils/stringChecks.ts

@@ -0,0 +1,23 @@
+export function getIsCharOrUnderscore(value: string): boolean {
+  const charOrUnderscore = /^[A-Za-z_]+$/;
+
+  return charOrUnderscore.test(value);
+}
+
+export function extractVariables(mustacheString: string): string[] {
+  const mustacheRegex = /\{\{(.*?)\}\}/g;
+  const uniqueVariables = new Set<string>();
+
+  for (const match of mustacheString.matchAll(mustacheRegex)) {
+    uniqueVariables.add(match[1]);
+  }
+
+  for (const variable of uniqueVariables) {
+    // if validated fails, remove from set
+    if (!getIsCharOrUnderscore(variable)) {
+      uniqueVariables.delete(variable);
+    }
+  }
+
+  return Array.from(uniqueVariables);
+}

patches/next-auth@4.24.11.patch

@@ -0,0 +1,26 @@
+diff --git a/core/lib/oauth/client.js b/core/lib/oauth/client.js
+index 52c51eb6ff422dc0899ccec31baf3fa39e42eeae..bc50c35bb617d0e86b68ca42f64d44b475ba4abb 100644
+--- a/core/lib/oauth/client.js
++++ b/core/lib/oauth/client.js
+@@ -1,5 +1,7 @@
+ "use strict";
+ 
++var HttpsProxyAgent = require('https-proxy-agent').HttpsProxyAgent;
++
+ Object.defineProperty(exports, "__esModule", {
+   value: true
+ });
+@@ -7,7 +9,12 @@ exports.openidClient = openidClient;
+ var _openidClient = require("openid-client");
+ async function openidClient(options) {
+   const provider = options.provider;
+-  if (provider.httpOptions) _openidClient.custom.setHttpOptionsDefaults(provider.httpOptions);
++  let httpOptions = {};
++  if (provider.httpOptions) httpOptions = { ...provider.httpOptions };
++  if (process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY) {
++    httpOptions.agent = new HttpsProxyAgent(process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY);
++  }
++  _openidClient.custom.setHttpOptionsDefaults(httpOptions);
+   let issuer;
+   if (provider.wellKnown) {
+     issuer = await _openidClient.Issuer.discover(provider.wellKnown);

web/Dockerfile

@@ -109,7 +109,7 @@ RUN adduser --system --uid 1001 nextjs
 RUN npm install -g --no-package-lock --no-save prisma@5.22.0
 
 RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
-    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.0/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
+    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
     mv migrate /usr/bin/migrate
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .

web/entrypoint.sh

@@ -12,11 +12,16 @@ if [ -z "$DATABASE_URL" ]; then
         echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
         exit 1
     fi
+    if [ -n "$DATABASE_ARGS" ]; then
+        # Append ARGS to DATABASE_URL
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
+        export DATABASE_URL
+    fi
 fi
 
 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
-    export DIRECT_URL=$DATABASE_URL
+    export DIRECT_URL="${DATABASE_URL}"
 fi
 
 # Always execute the postgres migration, except when disabled.

web/next.config.mjs

@@ -28,6 +28,9 @@ const cspHeader = `
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
 `;
 
+// Match rules for Hugging Face
+const huggingFaceHosts = ["huggingface.co", ".*\\.hf\\.space$"];
+
 const reportToHeader = {
   key: "Report-To",
   value: JSON.stringify({
@@ -78,10 +81,6 @@ const nextConfig = {
       {
         source: "/:path*",
         headers: [
-          {
-            key: "x-frame-options",
-            value: "SAMEORIGIN",
-          },
           {
             key: "X-Content-Type-Options",
             value: "nosniff",
@@ -97,6 +96,21 @@ const nextConfig = {
           ...(env.SENTRY_CSP_REPORT_URI ? [reportToHeader] : []),
         ],
       },
+      {
+        source: "/:path*",
+        headers: [
+          {
+            key: "x-frame-options",
+            value: "SAMEORIGIN",
+          },
+        ],
+        // Disable x-frame-options on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
+      },
+      // CSP header
       {
         source: "/:path((?!api).*)*",
         headers: [
@@ -105,26 +119,31 @@ const nextConfig = {
             value: cspHeader.replace(/\n/g, ""),
           },
         ],
+        // Disable CSP on Hugging Face to allow for embedded use of Langfuse
+        missing: huggingFaceHosts.map((host) => ({
+          type: "host",
+          value: host,
+        })),
       },
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-            {
-              source: "/api/auth/session",
-              headers: [
-                {
-                  key: "Access-Control-Allow-Origin",
-                  value: "https://langfuse.com",
-                },
-                { key: "Access-Control-Allow-Credentials", value: "true" },
-                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-                {
-                  key: "Access-Control-Allow-Headers",
-                  value: "Content-Type, Authorization",
-                },
-              ],
-            },
-          ]
+          {
+            source: "/api/auth/session",
+            headers: [
+              {
+                key: "Access-Control-Allow-Origin",
+                value: "https://langfuse.com",
+              },
+              { key: "Access-Control-Allow-Credentials", value: "true" },
+              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+              {
+                key: "Access-Control-Allow-Headers",
+                value: "Content-Type, Authorization",
+              },
+            ],
+          },
+        ]
         : []),
       // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
       {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "2.92.0",
+  "version": "2.95.2",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -83,7 +83,7 @@
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.39.0",
+    "@sentry/nextjs": "^8.52.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -107,8 +107,9 @@
     "date-fns": "^3.3.1",
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
-    "dompurify": "^3.1.5",
+    "dompurify": "^3.2.4",
     "graphql": "^16.9.0",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "ip-address": "^9.0.5",
     "js-tiktoken": "^1.0.15",
@@ -116,8 +117,8 @@
     "langchain": "^0.3.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.447.0",
-    "next": "^14.2.15",
-    "next-auth": "^4.24.7",
+    "next": "^14.2.21",
+    "next-auth": "^4.24.11",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.176.0",

web/src/__e2e__/api.servertest.ts

@@ -163,12 +163,12 @@ describe("Ingestion Pipeline", () => {
         // failure due to missing openai key in the pipeline. Expected
         expect(evalExecution.status).toBe(JobExecutionStatus.ERROR);
       },
-      40000,
-      1000,
+      50000,
+      10000,
     );
 
     expect(response.status).toBe(207);
-  }, 50000);
+  }, 60000);
 
   it("rate limit ingestion", async () => {
     // update the org in the database and set the rate limit to 1 for ingestion

web/src/__tests__/async/dataset-service.servertest.ts

@@ -13,6 +13,7 @@ import {
 import {
   createDatasetRunsTable,
   fetchDatasetItems,
+  getRunItemsByRunIdOrItemId,
 } from "@/src/features/datasets/server/service";
 
 const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
@@ -71,6 +72,8 @@ describe("Fetch datasets for UI presentation", () => {
     const traceId3 = v4();
     const traceId4 = v4();
     const scoreId = v4();
+    const scoreId2 = v4();
+    const scoreId3 = v4();
     const scoreName = v4();
 
     await prisma.datasetRunItems.create({
@@ -163,7 +166,28 @@ describe("Fetch datasets for UI presentation", () => {
       project_id: projectId,
       name: scoreName,
     });
-    await createScores([score]);
+    const score2 = createScore({
+      id: scoreId2,
+      observation_id: null,
+      trace_id: traceId,
+      project_id: projectId,
+      name: scoreName,
+      value: 1,
+      comment: "some other comment",
+    });
+    const observationId2 = v4(); // this one is not related to a run
+    const anotherScoreName = v4();
+
+    const score3 = createScore({
+      id: scoreId3,
+      observation_id: observationId2,
+      trace_id: traceId,
+      project_id: projectId,
+      name: anotherScoreName,
+      value: 1,
+      comment: "some other comment for non run related score",
+    });
+    await createScores([score, score2, score3]);
 
     const runs = await createDatasetRunsTable({
       projectId,
@@ -190,16 +214,21 @@ describe("Fetch datasets for UI presentation", () => {
     expect(firstRun.avgLatency).toBeGreaterThanOrEqual(10800);
     expect(firstRun.avgTotalCost.toString()).toStrictEqual("275");
 
-    const expectedObject = JSON.stringify({
+    const expectedObject = {
       [`${scoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
         type: "NUMERIC",
-        values: [100.5],
-        average: 100.5,
-        comment: "comment",
+        values: expect.arrayContaining([1, 100.5]),
+        average: 50.75,
       },
-    });
+      [`${anotherScoreName.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([1]),
+        average: 1,
+        comment: "some other comment for non run related score",
+      },
+    };
 
-    expect(JSON.stringify(firstRun.scores)).toEqual(expectedObject);
+    expect(firstRun.scores).toEqual(expectedObject);
 
     const secondRun = runs.find((run) => run.run_id === datasetRun2Id);
 
@@ -217,6 +246,245 @@ describe("Fetch datasets for UI presentation", () => {
     expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
   });
 
+  it("should fetch dataset run items for UI", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const traceId = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const traceId2 = v4();
+    const observationId = v4();
+    const datasetRunItemId2 = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRunId,
+        traceId: traceId2,
+        projectId,
+        datasetItemId,
+        observationId,
+      },
+    });
+
+    const trace1 = createTrace({
+      id: traceId,
+      project_id: projectId,
+    });
+
+    const trace2 = createTrace({
+      id: traceId2,
+      project_id: projectId,
+    });
+
+    await createTraces([trace1, trace2]);
+
+    const observation = createObservation({
+      id: observationId,
+      trace_id: traceId2,
+      project_id: projectId,
+      start_time: new Date().getTime() - 1000 * 60 * 60, // minus 1 min
+      end_time: new Date().getTime(),
+    });
+
+    const observation2 = createObservation({
+      trace_id: traceId,
+    });
+
+    await createObservations([observation]);
+
+    const score = createScore({
+      observation_id: observation2.id,
+      trace_id: traceId2,
+      project_id: projectId,
+    });
+
+    await createScores([score]);
+
+    const runs = await getRunItemsByRunIdOrItemId(
+      projectId,
+      // fetch directly from the db to have realistic data.
+      await prisma.datasetRunItems.findMany({
+        where: {
+          id: {
+            in: [datasetRunItemId, datasetRunItemId2],
+          },
+        },
+      }),
+    );
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.id === datasetRunItemId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+
+    expect(firstRun.id).toEqual(datasetRunItemId);
+    expect(firstRun.datasetItemId).toEqual(datasetItemId);
+    expect(firstRun.observation).toBeUndefined();
+    expect(firstRun.trace).toBeDefined();
+    expect(firstRun.trace?.id).toEqual(traceId);
+
+    const secondRun = runs.find((run) => run.id === datasetRunItemId2);
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("secondRun is not defined");
+    }
+
+    expect(secondRun.id).toEqual(datasetRunItemId2);
+    expect(secondRun.datasetItemId).toEqual(datasetItemId);
+    expect(secondRun.trace?.id).toEqual(traceId2);
+    expect(secondRun.observation?.id).toEqual(observationId);
+
+    const expectedObject = {
+      [`${score.name.replaceAll("-", "_")}-API-NUMERIC`]: {
+        type: "NUMERIC",
+        values: expect.arrayContaining([100.5]),
+        average: 100.5,
+        comment: "comment",
+      },
+    };
+
+    expect(secondRun.scores).toEqual(expectedObject);
+  });
+
+  it("should fetch dataset run items for UI with missing tracing data", async () => {
+    const datasetId = v4();
+
+    await prisma.dataset.create({
+      data: {
+        id: datasetId,
+        name: v4(),
+        projectId: projectId,
+      },
+    });
+    const datasetRunId = v4();
+    await prisma.datasetRuns.create({
+      data: {
+        id: datasetRunId,
+        name: v4(),
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetItemId = v4();
+    await prisma.datasetItem.create({
+      data: {
+        id: datasetItemId,
+        datasetId,
+        metadata: {},
+        projectId,
+      },
+    });
+
+    const datasetRunItemId = v4();
+    const traceId = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId,
+        datasetRunId: datasetRunId,
+        traceId: traceId,
+        projectId,
+        datasetItemId,
+      },
+    });
+
+    const traceId2 = v4();
+    const observationId = v4();
+    const datasetRunItemId2 = v4();
+
+    await prisma.datasetRunItems.create({
+      data: {
+        id: datasetRunItemId2,
+        datasetRunId: datasetRunId,
+        traceId: traceId2,
+        projectId,
+        datasetItemId,
+        observationId,
+      },
+    });
+
+    const runs = await getRunItemsByRunIdOrItemId(
+      projectId,
+      // fetch directly from the db to have realistic data.
+      await prisma.datasetRunItems.findMany({
+        where: {
+          id: {
+            in: [datasetRunItemId, datasetRunItemId2],
+          },
+        },
+      }),
+    );
+
+    expect(runs).toHaveLength(2);
+
+    const firstRun = runs.find((run) => run.id === datasetRunItemId);
+    expect(firstRun).toBeDefined();
+    if (!firstRun) {
+      throw new Error("first run is not defined");
+    }
+
+    expect(firstRun.id).toEqual(datasetRunItemId);
+    expect(firstRun.datasetItemId).toEqual(datasetItemId);
+    expect(firstRun.observation).toBeUndefined();
+    expect(firstRun.trace).toBeDefined();
+    expect(firstRun.trace?.id).toEqual(traceId);
+
+    const secondRun = runs.find((run) => run.id === datasetRunItemId2);
+    expect(secondRun).toBeDefined();
+    if (!secondRun) {
+      throw new Error("secondRun is not defined");
+    }
+
+    expect(secondRun.id).toEqual(datasetRunItemId2);
+    expect(secondRun.datasetItemId).toEqual(datasetItemId);
+    expect(secondRun.trace?.id).toEqual(traceId2);
+    expect(secondRun.observation?.id).toEqual(observationId);
+  });
+
   it("should fetch dataset items correctly", async () => {
     // Create test data in the database
 

web/src/__tests__/async/observation-api.servertest.ts

@@ -1,8 +1,10 @@
-import { createObservation } from "@/src/__tests__/fixtures/tracing-factory";
-import { createObservations } from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import { createObservation as createObservationObject } from "@/src/__tests__/fixtures/tracing-factory";
+import { createObservations as createObservationsInClickhouse } from "@/src/__tests__/server/repositories/clickhouse-helpers";
 import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
 import { GetObservationV1Response } from "@/src/features/public-api/types/observations";
 import { v4 } from "uuid";
+import { v4 as uuidv4 } from "uuid";
+import { GetObservationsV1Response } from "@/src/features/public-api/types/observations";
 
 const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
 
@@ -12,15 +14,17 @@ describe("/api/public/observations API Endpoint", () => {
       const observationId = v4();
       const traceId = v4();
 
-      const observation = createObservation({
+      const observation = createObservationObject({
         id: observationId,
         project_id: projectId,
         trace_id: traceId,
         internal_model_id: "b9854a5c92dc496b997d99d21",
         provided_model_name: "gpt-4o-2024-05-13",
+        input: "input",
+        output: "output",
       });
 
-      await createObservations([observation]);
+      await createObservationsInClickhouse([observation]);
 
       const getEventRes = await makeZodVerifiedAPICall(
         GetObservationV1Response,
@@ -33,7 +37,217 @@ describe("/api/public/observations API Endpoint", () => {
         type: observation.type,
         modelId: observation.internal_model_id,
         inputPrice: 0.000005,
+        input: observation.input,
+        output: observation.output,
       });
     });
   });
 });
+
+describe("/api/public/observations API Endpoint", () => {
+  it("should fetch all observations", async () => {
+    const traceId = uuidv4();
+
+    const observation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      internal_model_id: "clrkwk4cb000408l576jl7koo",
+      provided_model_name: "gpt-3.5-turbo",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      usage_details: {
+        input: 10,
+        output: 20,
+        total: 30,
+      },
+      version: "2.0.0",
+      type: "GENERATION",
+    });
+
+    await createObservationsInClickhouse([observation]);
+
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      "/api/public/observations?traceId=" + traceId,
+      undefined,
+    );
+
+    expect(fetchedObservations.status).toBe(200);
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
+    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
+    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
+    expect(fetchedObservations.body.data[0]?.model).toEqual("gpt-3.5-turbo");
+    expect(fetchedObservations.body.data[0]?.modelId).toEqual(
+      "clrkwk4cb000408l576jl7koo",
+    );
+    expect(
+      fetchedObservations.body.data[0]?.calculatedInputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedOutputCost,
+    ).toBeGreaterThan(0);
+    expect(
+      fetchedObservations.body.data[0]?.calculatedTotalCost,
+    ).toBeGreaterThan(0);
+  });
+
+  it("should fetch all observations, filtered by generations", async () => {
+    const traceId = uuidv4();
+
+    const generationObservation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      internal_model_id: "model-1",
+      provided_model_name: "gpt-3.5-turbo",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      usage_details: {
+        input: 10,
+        output: 20,
+        total: 30,
+      },
+      version: "2.0.0",
+      type: "GENERATION",
+    });
+
+    const spanObservation = createObservationObject({
+      id: uuidv4(),
+      trace_id: traceId,
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      input: JSON.stringify({ key: "input" }),
+      output: JSON.stringify({ key: "output" }),
+      version: "2.0.0",
+      type: "SPAN",
+    });
+
+    await createObservationsInClickhouse([
+      generationObservation,
+      spanObservation,
+    ]);
+
+    const fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      "/api/public/observations?type=GENERATION&traceId=" + traceId,
+      undefined,
+    );
+
+    console.log(fetchedObservations.body);
+
+    expect(fetchedObservations.status).toBe(200);
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
+    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
+    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
+    expect(fetchedObservations.body.data[0]?.type).toEqual("GENERATION");
+  });
+
+  it("GET /observations with timestamp filters and pagination", async () => {
+    const traceId = v4();
+    const obs1 = createObservationObject({
+      id: "observation-2021-01-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "GENERATION",
+    });
+
+    const obs2 = createObservationObject({
+      id: "observation-2021-02-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "SPAN",
+    });
+    const obs3 = createObservationObject({
+      id: "observation-2021-03-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-03-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "EVENT",
+    });
+    const obs4 = createObservationObject({
+      id: "observation-2021-04-01",
+      trace_id: traceId,
+      name: "generation-name",
+      start_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
+      end_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      type: "GENERATION",
+    });
+
+    await createObservationsInClickhouse([obs1, obs2, obs3, obs4]);
+
+    const fromTimestamp = "2021-02-01T00:00:00.000Z";
+    const toTimestamp = "2021-04-01T00:00:00.000Z";
+
+    // Test with both fromTimestamp and toTimestamp
+    let fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(2);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(2);
+
+    // Test with only fromTimestamp
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?fromStartTime=${fromTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(3);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+    // Test with only toTimestamp
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?toStartTime=${toTimestamp}&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(3);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
+    expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
+    expect(fetchedObservations.body.meta.totalItems).toBe(3);
+
+    // test pagination only
+    fetchedObservations = await makeZodVerifiedAPICall(
+      GetObservationsV1Response,
+      "GET",
+      `/api/public/observations?limit=1&page=2&traceId=${traceId}`,
+      undefined,
+    );
+
+    expect(fetchedObservations.body.data.length).toBe(1);
+    expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
+    expect(fetchedObservations.body.meta).toMatchObject({
+      totalItems: 4,
+      totalPages: 4,
+      page: 2,
+      limit: 1,
+    });
+  });
+});

web/src/__tests__/async/scores-api.servertest.ts

@@ -1,13 +1,27 @@
-import { createScores } from "@/src/__tests__/server/repositories/clickhouse-helpers";
-import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
-import { GetScoreResponse } from "@langfuse/shared";
+import {
+  createObservation,
+  createScore,
+  createTrace,
+} from "@/src/__tests__/fixtures/tracing-factory";
+import {
+  createObservations,
+  createScores,
+  createTraces,
+} from "@/src/__tests__/server/repositories/clickhouse-helpers";
+import {
+  createOrgProjectAndApiKey,
+  makeZodVerifiedAPICall,
+} from "@/src/__tests__/test-utils";
+import { GetScoreResponse, GetScoresResponse } from "@langfuse/shared";
+import { prisma } from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
-
-const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+import { z } from "zod";
 
 describe("/api/public/scores API Endpoint", () => {
   describe("GET /api/public/scores/:scoreId", () => {
     it("should GET a score", async () => {
+      const { projectId: projectId, auth } = await createOrgProjectAndApiKey();
+
       const scoreId = v4();
       const traceId = v4();
       const score = {
@@ -33,6 +47,8 @@ describe("/api/public/scores API Endpoint", () => {
         GetScoreResponse,
         "GET",
         `/api/public/scores/${scoreId}`,
+        undefined,
+        auth,
       );
 
       expect(getScore.status).toBe(200);
@@ -49,3 +65,653 @@ describe("/api/public/scores API Endpoint", () => {
     });
   });
 });
+
+describe("/api/public/scores API Endpoint", () => {
+  it("should create score for a trace", async () => {
+    const traceId = v4();
+
+    const { projectId: projectId, auth } = await createOrgProjectAndApiKey();
+
+    const trace = createTrace({
+      id: traceId,
+      project_id: projectId,
+    });
+    await createTraces([trace]);
+
+    const scoreId = v4();
+
+    const score = createScore({
+      id: scoreId,
+      project_id: projectId,
+      trace_id: traceId,
+      name: "score-name",
+      value: 100.5,
+      source: "API",
+      comment: "comment",
+      observation_id: null,
+    });
+    await createScores([score]);
+
+    const fetchedScore = await makeZodVerifiedAPICall(
+      GetScoreResponse,
+      "GET",
+      `/api/public/scores/${scoreId}`,
+      undefined,
+      auth,
+    );
+
+    expect(fetchedScore.body?.id).toBe(scoreId);
+    expect(fetchedScore.body?.traceId).toBe(traceId);
+    expect(fetchedScore.body?.name).toBe("score-name");
+    expect(fetchedScore.body?.value).toBe(100.5);
+    expect(fetchedScore.body?.observationId).toBeNull();
+    expect(fetchedScore.body?.comment).toBe("comment");
+    expect(fetchedScore.body?.source).toBe("API");
+    expect(fetchedScore.body?.projectId).toBe(projectId);
+  });
+  it("should GET score with minimal score data and minimal trace data", async () => {
+    const { projectId, auth } = await createOrgProjectAndApiKey();
+
+    const minimalTraceId = v4();
+
+    const trace = createTrace({
+      id: minimalTraceId,
+      project_id: projectId,
+    });
+    await createTraces([trace]);
+
+    const minimalScoreId = v4();
+
+    const score = createScore({
+      id: minimalScoreId,
+      project_id: projectId,
+      trace_id: minimalTraceId,
+      name: "score-name",
+      value: 100.5,
+      source: "API",
+      comment: null,
+      observation_id: null,
+    });
+    await createScores([score]);
+
+    const fetchedScore = await makeZodVerifiedAPICall(
+      GetScoreResponse,
+      "GET",
+      `/api/public/scores/${minimalScoreId}`,
+      undefined,
+      auth,
+    );
+
+    expect(fetchedScore.status).toBe(200);
+  });
+  describe("should Filter scores", () => {
+    let configId = "";
+    const userId = "user-name";
+    const traceTags = ["prod", "test"];
+    const traceTags_2 = ["staging", "dev"];
+    const scoreName = "score-name";
+    const queryUserName = `userId=${userId}&name=${scoreName}`;
+    const traceId = v4();
+    const traceId_2 = v4();
+    const traceId_3 = v4();
+    const generationId = v4();
+    const scoreId_1 = v4();
+    const scoreId_2 = v4();
+    const scoreId_3 = v4();
+    const scoreId_4 = v4();
+    const scoreId_5 = v4();
+    let authentication: string;
+    let newProjectId: string;
+
+    beforeEach(async () => {
+      const { projectId, auth } = await createOrgProjectAndApiKey();
+      authentication = auth;
+      newProjectId = projectId;
+
+      const trace = createTrace({
+        id: traceId,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: traceTags,
+      });
+
+      const trace_2 = createTrace({
+        id: traceId_2,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: traceTags_2,
+      });
+
+      const trace_3 = createTrace({
+        id: traceId_3,
+        project_id: newProjectId,
+        user_id: userId,
+        tags: ["staging"],
+      });
+
+      await createTraces([trace, trace_2, trace_3]);
+
+      const generation = createObservation({
+        id: generationId,
+        project_id: newProjectId,
+        type: "GENERATION",
+      });
+
+      await createObservations([generation]);
+
+      const config = await prisma.scoreConfig.create({
+        data: {
+          name: scoreName,
+          dataType: "NUMERIC",
+          maxValue: 100,
+          projectId: newProjectId,
+        },
+      });
+
+      configId = config.id;
+
+      const score1 = createScore({
+        id: scoreId_1,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 10.5,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        config_id: config.id,
+        comment: "comment",
+      });
+
+      const score2 = createScore({
+        id: scoreId_2,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 50.5,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        comment: "comment",
+      });
+
+      const score3 = createScore({
+        id: scoreId_3,
+        project_id: newProjectId,
+        trace_id: traceId,
+        name: scoreName,
+        value: 100.8,
+        data_type: "NUMERIC",
+        observation_id: generationId,
+        comment: "comment",
+      });
+
+      const score4 = createScore({
+        id: scoreId_4,
+        project_id: newProjectId,
+        trace_id: traceId_2,
+        name: "other-score-name",
+        value: 0,
+        string_value: "best",
+        data_type: "CATEGORICAL",
+        comment: "comment",
+      });
+
+      const score5 = createScore({
+        id: scoreId_5,
+        project_id: newProjectId,
+        trace_id: traceId_3,
+        name: "other-score-name",
+        value: 0,
+        data_type: "CATEGORICAL",
+        string_value: "test",
+        comment: "comment",
+      });
+
+      await createScores([score1, score2, score3, score4, score5]);
+    });
+
+    it("get all scores", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}`,
+        undefined,
+        authentication,
+      );
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+        });
+      }
+    });
+
+    it("get all scores for config", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?configId=${configId}`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+          configId: configId,
+        });
+      }
+    });
+
+    it("get all scores for numeric data type", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?dataType=NUMERIC`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          observationId: generationId,
+          dataType: "NUMERIC",
+        });
+      }
+    });
+
+    it("get all scores for trace tag 'prod'", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?traceTags=prod`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId,
+          trace: { tags: ["prod", "test"], userId: "user-name" },
+        });
+      }
+    });
+
+    it("get all scores for trace tags 'staging' and 'dev'", async () => {
+      const getAllScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?traceTags=${["staging", "dev"]}`,
+        undefined,
+        authentication,
+      );
+
+      expect(getAllScore.status).toBe(200);
+      expect(getAllScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      for (const val of getAllScore.body.data) {
+        expect(val).toMatchObject({
+          traceId: traceId_2,
+          trace: {
+            tags: expect.arrayContaining(["dev", "staging"]),
+            userId: "user-name",
+          },
+        });
+      }
+    });
+
+    describe("should Filter scores by queueId", () => {
+      describe("queueId filtering", () => {
+        let queueId: string;
+
+        beforeEach(async () => {
+          queueId = v4();
+          const score = createScore({
+            id: v4(),
+            project_id: newProjectId,
+            trace_id: traceId,
+            name: "score-name",
+            value: 100.5,
+            source: "ANNOTATION",
+            comment: "comment",
+            observation_id: generationId,
+            queue_id: queueId,
+          });
+          const score2 = createScore({
+            id: v4(),
+            project_id: newProjectId,
+            trace_id: traceId,
+            name: "score-name",
+            value: 75.0,
+            source: "ANNOTATION",
+            comment: "comment",
+            observation_id: generationId,
+            queue_id: queueId,
+          });
+
+          await createScores([score, score2]);
+        });
+
+        it("get all scores for queueId", async () => {
+          const getAllScore = await makeZodVerifiedAPICall(
+            GetScoresResponse,
+            "GET",
+            `/api/public/scores?queueId=${queueId}`,
+            undefined,
+            authentication,
+          );
+          expect(getAllScore.status).toBe(200);
+          expect(getAllScore.body.meta).toMatchObject({
+            page: 1,
+            limit: 50,
+            totalItems: 2,
+            totalPages: 1,
+          });
+          for (const val of getAllScore.body.data) {
+            expect(val).toMatchObject({
+              traceId: traceId,
+              observationId: generationId,
+              queueId: queueId,
+              source: "ANNOTATION",
+            });
+          }
+        });
+      });
+    });
+
+    it("test only operator", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test only value", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&value=0.8`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 3,
+        totalPages: 1,
+      });
+    });
+
+    it("test operator <", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<&value=50`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_1,
+          name: scoreName,
+          value: 10.5,
+        },
+      ]);
+    });
+    it("test operator >", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>&value=100`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_3,
+          name: scoreName,
+          value: 100.8,
+        },
+      ]);
+    });
+    it("test operator <=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=<=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator >=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=>=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_3,
+            name: scoreName,
+            value: 100.8,
+          }),
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator !=", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator=!=&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_3,
+            name: scoreName,
+            value: 100.8,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+    it("test operator =", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?${queryUserName}&operator==&value=50.5`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 1,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toMatchObject([
+        {
+          id: scoreId_2,
+          name: scoreName,
+          value: 50.5,
+        },
+      ]);
+    });
+
+    it("test invalid operator", async () => {
+      try {
+        await makeZodVerifiedAPICall(
+          z.object({
+            message: z.string(),
+            error: z.array(z.object({})),
+          }),
+          "GET",
+          `/api/public/scores?${queryUserName}&operator=op&value=50.5`,
+          undefined,
+          authentication,
+        );
+      } catch (error) {
+        expect((error as Error).message).toBe(
+          `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"received\":\"op\",\"code\":\"invalid_enum_value\",\"options\":[\"<\",\">\",\"<=\",\">=\",\"!=\",\"=\"],\"path\":[\"operator\"],\"message\":\"Invalid enum value. Expected '<' | '>' | '<=' | '>=' | '!=' | '=', received 'op'\"}]}`,
+        );
+      }
+    });
+    it("test invalid value", async () => {
+      try {
+        await makeZodVerifiedAPICall(
+          z.object({
+            message: z.string(),
+            error: z.array(z.object({})),
+          }),
+          "GET",
+          `/api/public/scores?${queryUserName}&operator=<&value=myvalue`,
+          undefined,
+          authentication,
+        );
+      } catch (error) {
+        expect((error as Error).message).toBe(
+          'API call did not return 200, returned status 400, body {"message":"Invalid request data","error":[{"code":"invalid_type","expected":"number","received":"nan","path":["value"],"message":"Expected number, received nan"}]}',
+        );
+      }
+    });
+
+    it("should filter scores by score IDs", async () => {
+      const getScore = await makeZodVerifiedAPICall(
+        GetScoresResponse,
+        "GET",
+        `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`,
+        undefined,
+        authentication,
+      );
+      expect(getScore.status).toBe(200);
+      expect(getScore.body.meta).toMatchObject({
+        page: 1,
+        limit: 50,
+        totalItems: 2,
+        totalPages: 1,
+      });
+      expect(getScore.body.data).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: scoreId_2,
+            name: scoreName,
+            value: 50.5,
+          }),
+          expect.objectContaining({
+            id: scoreId_1,
+            name: scoreName,
+            value: 10.5,
+          }),
+        ]),
+      );
+    });
+  });
+});

web/src/__tests__/async/traces-api.servertest.ts

@@ -27,13 +27,17 @@ describe("/api/public/traces API Endpoint", () => {
         name: "observation-name",
         end_time: new Date().getTime(),
         start_time: new Date().getTime() - 1000,
+        input: "input",
+        output: "output",
       }),
       createObservation({
         trace_id: createdTrace.id,
         project_id: createdTrace.project_id,
-        name: "observation-name",
+        name: "observation-name-2",
         end_time: new Date().getTime(),
         start_time: new Date().getTime() - 100000,
+        input: "input-2",
+        output: "output-2",
       }),
     ];
 
@@ -54,5 +58,19 @@ describe("/api/public/traces API Endpoint", () => {
     expect(trace.body.latency).toBe(100);
     expect(trace.body.observations.length).toBe(2);
     expect(trace.body.scores.length).toBe(0);
+    expect(trace.body.observations).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          name: "observation-name-2",
+          input: "input-2",
+          output: "output-2",
+        }),
+        expect.objectContaining({
+          name: "observation-name",
+          input: "input",
+          output: "output",
+        }),
+      ]),
+    );
   });
 });

web/src/__tests__/async/traces-ui-table.servertest.ts

@@ -43,7 +43,7 @@ describe("Traces table API test", () => {
     expect(tableRows[0].userId).toEqual(trace.user_id);
     expect(tableRows[0].sessionId).toEqual(trace.session_id);
     expect(tableRows[0].public).toEqual(trace.public);
-    expect(tableRows[0].latencyMilliseconds).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
     expect(tableRows[0].usageDetails).toEqual({});
     expect(tableRows[0].costDetails).toEqual({});
     expect(tableRows[0].level).toBeDefined();
@@ -82,7 +82,7 @@ describe("Traces table API test", () => {
     expect(tableRows[0].userId).toEqual(trace.user_id);
     expect(tableRows[0].sessionId).toEqual(trace.session_id);
     expect(tableRows[0].public).toEqual(trace.public);
-    expect(tableRows[0].latencyMilliseconds).toBeGreaterThanOrEqual(0);
+    expect(tableRows[0].latency).toBeGreaterThanOrEqual(0);
     expect(tableRows[0].usageDetails).toEqual({
       input: (obs1.usage_details.input + obs2.usage_details.input).toString(),
       output: (
@@ -163,6 +163,19 @@ describe("Traces table API test", () => {
       ],
       expected: [],
     },
+    {
+      traceInput: {},
+      observationInput: [],
+      filterstate: [
+        {
+          column: "Latency (s)",
+          operator: ">" as const,
+          value: 5_000_000, // Verify that we can pass large values
+          type: "number" as const,
+        },
+      ],
+      expected: [],
+    },
   ].forEach(async (testConfig: TestCase) => {
     it(`should get a correct trace with filters ${JSON.stringify(testConfig)}`, async () => {
       const project_id = v4();
@@ -234,10 +247,8 @@ describe("Traces table API test", () => {
         if (expectedTrace.public !== undefined) {
           expect(tableRows[index].public).toEqual(expectedTrace.public);
         }
-        if (expectedTrace.latency_milliseconds !== undefined) {
-          expect(tableRows[index].latencyMilliseconds).toEqual(
-            expectedTrace.latency_milliseconds,
-          );
+        if (expectedTrace.latency !== undefined) {
+          expect(tableRows[index].latency).toEqual(expectedTrace.latency);
         }
         if (expectedTrace.usage_details !== undefined) {
           expect(tableRows[index].usageDetails).toEqual(

web/src/__tests__/media.servertest.ts

@@ -167,7 +167,12 @@ describe("Media Upload API", () => {
       );
       result.getDownloadUrlResponse = getDownloadUrlResponse;
 
-      if (getDownloadUrlResponse.status !== 200) {
+      if (
+        !(
+          getDownloadUrlResponse.status === 200 ||
+          getDownloadUrlResponse.status === 201
+        )
+      ) {
         return result;
       }
 

web/src/__tests__/observations.servertest.ts

@@ -1,313 +0,0 @@
-/** @jest-environment node */
-
-import { v4 as uuidv4 } from "uuid";
-import {
-  makeZodVerifiedAPICall,
-  pruneDatabase,
-} from "@/src/__tests__/test-utils";
-import { ModelUsageUnit } from "@langfuse/shared";
-import { prisma } from "@langfuse/shared/src/db";
-import { GetObservationsV1Response } from "@/src/features/public-api/types/observations";
-
-describe("/api/public/observations API Endpoint", () => {
-  beforeEach(async () => await pruneDatabase());
-  afterEach(async () => await pruneDatabase());
-
-  it("should fetch all observations", async () => {
-    await pruneDatabase();
-
-    const traceId = uuidv4();
-
-    await prisma.trace.create({
-      data: {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    });
-
-    const model = await prisma.model.create({
-      data: {
-        id: "model-1",
-        modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0000010",
-        outputPrice: "0.0000020",
-        totalPrice: "0.1",
-        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        projectId: null,
-        unit: ModelUsageUnit.Tokens,
-      },
-    });
-
-    const prompt = await prisma.prompt.create({
-      data: {
-        name: "prompt-name",
-        prompt: "prompt-one",
-        isActive: false,
-        version: 1,
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        createdBy: "user-1",
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "gpt-3.5-turbo",
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        promptTokens: 10,
-        completionTokens: 20,
-        totalTokens: 30,
-        version: "2.0.0",
-        type: "GENERATION",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-        internalModel: "gpt-3.5-turbo",
-        unit: ModelUsageUnit.Tokens,
-        promptId: prompt.id,
-      },
-    });
-
-    const fetchedObservations = await makeZodVerifiedAPICall(
-      GetObservationsV1Response,
-      "GET",
-      "/api/public/observations",
-      undefined,
-    );
-
-    expect(fetchedObservations.status).toBe(200);
-
-    expect(fetchedObservations.body.data.length).toBe(1);
-    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
-    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
-    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
-    expect(fetchedObservations.body.data[0]?.model).toEqual("gpt-3.5-turbo");
-    expect(fetchedObservations.body.data[0]?.modelId).toEqual(model.id);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedInputCost,
-    ).toBeGreaterThan(0);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedOutputCost,
-    ).toBeGreaterThan(0);
-    expect(
-      fetchedObservations.body.data[0]?.calculatedTotalCost,
-    ).toBeGreaterThan(0);
-    expect(fetchedObservations.body.data[0]?.promptId).toBe(prompt.id);
-    expect(fetchedObservations.body.data[0]?.promptName).toBe(prompt.name);
-    expect(fetchedObservations.body.data[0]?.promptVersion).toBe(
-      prompt.version,
-    );
-  });
-  it("should fetch all observations, filtered by generations", async () => {
-    await pruneDatabase();
-
-    const traceId = uuidv4();
-
-    await prisma.trace.create({
-      data: {
-        id: traceId,
-        name: "trace-name",
-        userId: "user-1",
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        metadata: { key: "value" },
-        release: "1.0.0",
-        version: "2.0.0",
-      },
-    });
-
-    await prisma.model.create({
-      data: {
-        id: "model-1",
-        modelName: "gpt-3.5-turbo",
-        inputPrice: "0.0000010",
-        outputPrice: "0.0000020",
-        totalPrice: "0.1",
-        matchPattern: "(.*)(gpt-)(35|3.5)(-turbo)?(.*)",
-        projectId: null,
-        unit: ModelUsageUnit.Tokens,
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        model: "gpt-3.5-turbo",
-        internalModel: "gpt-3.5-turbo",
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        promptTokens: 10,
-        completionTokens: 20,
-        totalTokens: 30,
-        version: "2.0.0",
-        unit: ModelUsageUnit.Tokens,
-        type: "GENERATION",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-      },
-    });
-
-    await prisma.observation.create({
-      data: {
-        id: uuidv4(),
-        traceId: traceId,
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        modelParameters: { key: "value" },
-        input: { key: "input" },
-        output: { key: "output" },
-        version: "2.0.0",
-        type: "SPAN",
-        project: {
-          connect: { id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" },
-        },
-      },
-    });
-
-    const fetchedObservations = await makeZodVerifiedAPICall(
-      GetObservationsV1Response,
-      "GET",
-      "/api/public/observations?type=GENERATION",
-      undefined,
-    );
-
-    console.log(fetchedObservations.body);
-
-    expect(fetchedObservations.status).toBe(200);
-
-    expect(fetchedObservations.body.data.length).toBe(1);
-    expect(fetchedObservations.body.data[0]?.traceId).toBe(traceId);
-    expect(fetchedObservations.body.data[0]?.input).toEqual({ key: "input" });
-    expect(fetchedObservations.body.data[0]?.output).toEqual({ key: "output" });
-    expect(fetchedObservations.body.data[0]?.type).toEqual("GENERATION");
-  });
-});
-
-it("GET /observations with timestamp filters and pagination", async () => {
-  await prisma.trace.create({
-    data: {
-      id: "trace-id",
-      name: "trace-name",
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    },
-  });
-  await prisma.observation.createMany({
-    data: [
-      {
-        id: "observation-2021-01-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-01-01T00:00:00.000Z"),
-        endTime: new Date("2021-01-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "GENERATION",
-      },
-      {
-        id: "observation-2021-02-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-02-01T00:00:00.000Z"),
-        endTime: new Date("2021-02-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "SPAN",
-      },
-      {
-        id: "observation-2021-03-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-03-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "EVENT",
-      },
-      {
-        id: "observation-2021-04-01",
-        traceId: "trace-id",
-        name: "generation-name",
-        startTime: new Date("2021-04-01T00:00:00.000Z"),
-        endTime: new Date("2021-04-01T00:00:00.000Z"),
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-        type: "GENERATION",
-      },
-    ],
-  });
-
-  const fromTimestamp = "2021-02-01T00:00:00.000Z";
-  const toTimestamp = "2021-04-01T00:00:00.000Z";
-
-  // Test with both fromTimestamp and toTimestamp
-  let fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?fromStartTime=${fromTimestamp}&toStartTime=${toTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(2);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(2);
-
-  // Test with only fromTimestamp
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?fromStartTime=${fromTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(3);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-04-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(3);
-
-  // Test with only toTimestamp
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?toStartTime=${toTimestamp}`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(3);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.data[1]?.id).toBe("observation-2021-02-01");
-  expect(fetchedObservations.body.data[2]?.id).toBe("observation-2021-01-01");
-  expect(fetchedObservations.body.meta.totalItems).toBe(3);
-
-  // test pagination only
-  fetchedObservations = await makeZodVerifiedAPICall(
-    GetObservationsV1Response,
-    "GET",
-    `/api/public/observations?limit=1&page=2`,
-    undefined,
-  );
-
-  expect(fetchedObservations.body.data.length).toBe(1);
-  expect(fetchedObservations.body.data[0]?.id).toBe("observation-2021-03-01");
-  expect(fetchedObservations.body.meta).toMatchObject({
-    totalItems: 4,
-    totalPages: 4,
-    page: 2,
-    limit: 1,
-  });
-});

web/src/__tests__/scores.servertest.ts

@@ -7,12 +7,7 @@ import {
   pruneDatabase,
 } from "@/src/__tests__/test-utils";
 import { v4 as uuidv4 } from "uuid";
-import {
-  DeleteScoreResponse,
-  GetScoreResponse,
-  GetScoresResponse,
-} from "@langfuse/shared";
-import { z } from "zod";
+import { DeleteScoreResponse, GetScoreResponse } from "@langfuse/shared";
 import { PostTracesV1Response } from "@/src/features/public-api/types/traces";
 
 const traceId = "de98afa2-89dc-47e9-9924-33f1490fdaf4";
@@ -111,39 +106,6 @@ describe("/api/public/scores API Endpoint", () => {
     expect(fetchedScore.body?.observationId).toBeNull();
   });
 
-  it("should GET score with minimal score data and minimal trace data", async () => {
-    const minimalTraceId = uuidv4();
-    await makeAPICall("POST", "/api/public/traces", {
-      id: minimalTraceId,
-      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-    });
-    const dbTrace = await prisma.trace.findMany({
-      where: {
-        id: minimalTraceId,
-      },
-    });
-
-    expect(dbTrace.length).toBeGreaterThan(0);
-    expect(dbTrace[0]?.id).toBe(minimalTraceId);
-
-    const minimalScoreId = uuidv4();
-
-    await makeAPICall("POST", "/api/public/scores", {
-      id: minimalScoreId,
-      name: "score-name",
-      value: 100,
-      traceId: minimalTraceId,
-    });
-
-    const fetchedScore = await makeZodVerifiedAPICall(
-      GetScoreResponse,
-      "GET",
-      `/api/public/scores/${minimalScoreId}`,
-    );
-
-    expect(fetchedScore.status).toBe(200);
-  });
-
   it("should create score for a generation", async () => {
     await pruneDatabase();
 
@@ -826,573 +788,4 @@ describe("/api/public/scores API Endpoint", () => {
     });
     expect(deletedScore).toBeNull();
   });
-
-  describe("should Filter scores", () => {
-    let configId = "";
-    const userId = "user-name";
-    const traceTags = ["prod", "test"];
-    const traceTags_2 = ["staging", "dev"];
-    const scoreName = "score-name";
-    const queryUserName = `userId=${userId}&name=${scoreName}`;
-    const traceId = uuidv4();
-    const traceId_2 = uuidv4();
-    const traceId_3 = uuidv4();
-    const generationId = uuidv4();
-    const scoreId_1 = uuidv4();
-    const scoreId_2 = uuidv4();
-    const scoreId_3 = uuidv4();
-    const scoreId_4 = uuidv4();
-    const scoreId_5 = uuidv4();
-
-    beforeAll(async () => {
-      should_prune_db = false;
-      await pruneDatabase();
-
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId,
-          userId: userId,
-          tags: traceTags,
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId_2,
-          userId: userId,
-          tags: traceTags_2,
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/traces",
-        {
-          id: traceId_3,
-          userId: userId,
-          tags: ["staging"],
-        },
-      );
-      await makeZodVerifiedAPICall(
-        PostTracesV1Response,
-        "POST",
-        "/api/public/generations",
-        {
-          id: generationId,
-        },
-      );
-
-      await makeAPICall("POST", "/api/public/score-configs", {
-        name: scoreName,
-        dataType: "NUMERIC",
-        maxValue: 100,
-      });
-
-      const config = await prisma.scoreConfig.findFirst({
-        where: {
-          name: scoreName,
-        },
-      });
-      configId = config?.id ?? "";
-
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_1,
-        observationId: generationId,
-        name: scoreName,
-        value: 10.5,
-        traceId: traceId,
-        comment: "comment",
-        configId,
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_2,
-        observationId: generationId,
-        name: scoreName,
-        value: 50.5,
-        traceId: traceId,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_3,
-        observationId: generationId,
-        name: scoreName,
-        value: 100.8,
-        traceId: traceId,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_4,
-        name: "other-score-name",
-        value: "best",
-        traceId: traceId_2,
-        comment: "comment",
-      });
-      await makeAPICall("POST", "/api/public/scores", {
-        id: scoreId_5,
-        name: "other-score-name",
-        value: "test",
-        traceId: traceId_3,
-        comment: "comment",
-      });
-    });
-    afterAll(async () => {
-      await pruneDatabase();
-    });
-
-    it("get all scores", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}`,
-      );
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-        });
-      }
-    });
-
-    it("get all scores for config", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?configId=${configId}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-          configId: configId,
-        });
-      }
-    });
-
-    it("get all scores for numeric data type", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?dataType=${"NUMERIC"}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          observationId: generationId,
-          dataType: "NUMERIC",
-        });
-      }
-    });
-
-    it("get all scores for trace tag 'prod'", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?traceTags=${"prod"}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId,
-          trace: { tags: ["prod", "test"], userId: "user-name" },
-        });
-      }
-    });
-
-    it("get all scores for trace tags 'staging' and 'dev'", async () => {
-      const getAllScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?traceTags=${["staging", "dev"]}`,
-      );
-
-      expect(getAllScore.status).toBe(200);
-      expect(getAllScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      for (const val of getAllScore.body.data) {
-        expect(val).toMatchObject({
-          traceId: traceId_2,
-          trace: { tags: ["dev", "staging"], userId: "user-name" },
-        });
-      }
-    });
-
-    describe("should Filter scores by queueId", () => {
-      describe("queueId filtering", () => {
-        let queueId: string;
-        const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
-
-        beforeEach(async () => {
-          queueId = uuidv4();
-
-          await Promise.all([
-            prisma.score.create({
-              data: {
-                observationId: generationId,
-                name: "annotation-score-1",
-                value: 100.5,
-                traceId: traceId,
-                comment: "comment 1",
-                queueId,
-                source: "ANNOTATION",
-                project: { connect: { id: projectId } },
-                dataType: "NUMERIC",
-              },
-            }),
-            prisma.score.create({
-              data: {
-                observationId: generationId,
-                name: "annotation-score-2",
-                value: 75.0,
-                traceId: traceId,
-                comment: "comment 2",
-                queueId,
-                source: "ANNOTATION",
-                project: { connect: { id: projectId } },
-                dataType: "NUMERIC",
-              },
-            }),
-          ]);
-        });
-
-        afterEach(async () => {
-          await prisma.score.deleteMany({
-            where: { queueId, projectId },
-          });
-        });
-
-        it("get all scores for queueId", async () => {
-          const getAllScore = await makeZodVerifiedAPICall(
-            GetScoresResponse,
-            "GET",
-            `/api/public/scores?queueId=${queueId}`,
-          );
-
-          expect(getAllScore.status).toBe(200);
-          expect(getAllScore.body.meta).toMatchObject({
-            page: 1,
-            limit: 50,
-            totalItems: 2,
-            totalPages: 1,
-          });
-          for (const val of getAllScore.body.data) {
-            expect(val).toMatchObject({
-              traceId: traceId,
-              observationId: generationId,
-              queueId: queueId,
-              source: "ANNOTATION",
-            });
-          }
-        });
-      });
-    });
-
-    it("test only operator", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-    });
-
-    it("test only value", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&value=0.8`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 3,
-        totalPages: 1,
-      });
-    });
-
-    it("test operator <", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<&value=50`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator >", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=>&value=100`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-      ]);
-    });
-    it("test operator <=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=<=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator >=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=>=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-      ]);
-    });
-    it("test operator !=", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator=!=&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_3,
-          name: scoreName,
-          value: 100.8,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-    it("test operator =", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?${queryUserName}&operator==&value=50.5`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 1,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-      ]);
-    });
-    it("should GET ALL scores with minimal score data and minimal trace data", async () => {
-      const minimalTraceId = uuidv4();
-      await makeAPICall("POST", "/api/public/traces", {
-        id: minimalTraceId,
-        projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
-      });
-      const dbTrace = await prisma.trace.findMany({
-        where: {
-          id: minimalTraceId,
-        },
-      });
-
-      expect(dbTrace.length).toBeGreaterThan(0);
-      expect(dbTrace[0]?.id).toBe(minimalTraceId);
-
-      const createScore = await makeAPICall("POST", "/api/public/scores", {
-        name: "score-name",
-        value: 100,
-        traceId: minimalTraceId,
-      });
-
-      expect(createScore.status).toBe(200);
-
-      const fetchedScores = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores`,
-      );
-
-      expect(fetchedScores.status).toBe(200);
-      expect(fetchedScores.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 6,
-        totalPages: 1,
-      });
-      expect(fetchedScores.body.data.length).toBe(6);
-    });
-
-    it("test invalid operator", async () => {
-      try {
-        await makeZodVerifiedAPICall(
-          z.object({
-            message: z.string(),
-            error: z.array(z.object({})),
-          }),
-          "GET",
-          `/api/public/scores?${queryUserName}&operator=op&value=50.5`,
-        );
-      } catch (error) {
-        expect((error as Error).message).toBe(
-          `API call did not return 200, returned status 400, body {\"message\":\"Invalid request data\",\"error\":[{\"received\":\"op\",\"code\":\"invalid_enum_value\",\"options\":[\"<\",\">\",\"<=\",\">=\",\"!=\",\"=\"],\"path\":[\"operator\"],\"message\":\"Invalid enum value. Expected '<' | '>' | '<=' | '>=' | '!=' | '=', received 'op'\"}]}`,
-        );
-      }
-    });
-    it("test invalid value", async () => {
-      try {
-        await makeZodVerifiedAPICall(
-          z.object({
-            message: z.string(),
-            error: z.array(z.object({})),
-          }),
-          "GET",
-          `/api/public/scores?${queryUserName}&operator=<&value=myvalue`,
-        );
-      } catch (error) {
-        expect((error as Error).message).toBe(
-          'API call did not return 200, returned status 400, body {"message":"Invalid request data","error":[{"code":"invalid_type","expected":"number","received":"nan","path":["value"],"message":"Expected number, received nan"}]}',
-        );
-      }
-    });
-
-    it("should filter scores by score IDs", async () => {
-      const getScore = await makeZodVerifiedAPICall(
-        GetScoresResponse,
-        "GET",
-        `/api/public/scores?scoreIds=${scoreId_1},${scoreId_2}`,
-      );
-      expect(getScore.status).toBe(200);
-      expect(getScore.body.meta).toMatchObject({
-        page: 1,
-        limit: 50,
-        totalItems: 2,
-        totalPages: 1,
-      });
-      expect(getScore.body.data).toMatchObject([
-        {
-          id: scoreId_2,
-          name: scoreName,
-          value: 50.5,
-        },
-        {
-          id: scoreId_1,
-          name: scoreName,
-          value: 10.5,
-        },
-      ]);
-    });
-  });
 });

web/src/__tests__/test-utils.ts

@@ -1,6 +1,11 @@
 import { env } from "@/src/env.mjs";
 import { prisma } from "@langfuse/shared/src/db";
-import { clickhouseClient } from "@langfuse/shared/src/server";
+import {
+  clickhouseClient,
+  getDisplaySecretKey,
+  hashSecretKey,
+} from "@langfuse/shared/src/server";
+import { v4 } from "uuid";
 import { type z } from "zod";
 
 export const pruneDatabase = async () => {
@@ -38,7 +43,10 @@ export const pruneDatabase = async () => {
   });
 };
 
-function createBasicAuthHeader(username: string, password: string): string {
+export function createBasicAuthHeader(
+  username: string,
+  password: string,
+): string {
   const base64Credentials = Buffer.from(`${username}:${password}`).toString(
     "base64",
   );
@@ -130,3 +138,35 @@ export async function makeZodVerifiedAPICallSilent<T extends z.ZodTypeAny>(
 
   return { body: resBody, status };
 }
+
+export const createOrgProjectAndApiKey = async () => {
+  const projectId = v4();
+  const org = await prisma.organization.create({
+    data: {
+      id: v4(),
+      name: v4(),
+    },
+  });
+  await prisma.project.create({
+    data: {
+      id: projectId,
+      name: v4(),
+      orgId: org.id,
+    },
+  });
+  const publicKey = v4();
+  const secretKey = v4();
+
+  const auth = createBasicAuthHeader(publicKey, secretKey);
+  await prisma.apiKey.create({
+    data: {
+      id: v4(),
+      projectId: projectId,
+      publicKey: publicKey,
+      hashedSecretKey: await hashSecretKey(secretKey),
+      displaySecretKey: getDisplaySecretKey(secretKey),
+    },
+  });
+
+  return { projectId, publicKey, secretKey, auth };
+};

web/src/__tests__/usage.servertest.ts

@@ -53,6 +53,15 @@ describe("Token Count Functions", () => {
       expect(result).toBeUndefined();
     });
 
+    it("should return token count for strings with special characters", () => {
+      const result = tokenCount({
+        model: generateModel("gpt-4-1106-preview", "openai"),
+        text: "Hello <|endoftext|> World!",
+      });
+      expect(result).toBeDefined();
+      expect(result).toBeGreaterThan(0);
+    });
+
     it("check extensive openai chat message", () => {
       const result = tokenCount({
         model: generateModel("gpt-3.5-turbo", "openai"),
@@ -130,6 +139,7 @@ describe("Token Count Functions", () => {
       });
       expect(result).toBeUndefined();
     });
+
     it("should return undefined for undefined text input", () => {
       const result = tokenCount({
         model: generateModel("gpt-4", "openai"),

web/src/components/layouts/layout.tsx

@@ -57,6 +57,7 @@ const unauthenticatedPaths: string[] = [
   "/auth/sign-in",
   "/auth/sign-up",
   "/auth/error",
+  "/auth/hf-spaces",
 ];
 // auth or unauthed
 const publishablePaths: string[] = [

web/src/components/sidebar-notifications.tsx

@@ -23,6 +23,13 @@ type SidebarNotification = {
 };
 
 const notifications: SidebarNotification[] = [
+  {
+    id: "lw2-5",
+    title: "Launch Week 2 – Day 5",
+    description: "Introducing Prompt Experiments to test prompts on datasets",
+    link: "https://langfuse.com/changelog/2024-11-22-prompt-experimentation",
+    linkTitle: "Changelog",
+  },
   {
     id: "lw2-4",
     title: "Launch Week 2 – Day 4",
@@ -105,7 +112,7 @@ export function SidebarNotifications() {
   const currentNotification = notifications[currentNotificationIndex];
 
   return (
-    <Card className="relative max-h-44 overflow-hidden rounded-md bg-opacity-50 shadow-none group-data-[collapsible=icon]:hidden">
+    <Card className="relative max-h-60 overflow-hidden rounded-md bg-opacity-50 shadow-none group-data-[collapsible=icon]:hidden">
       <Button
         variant="ghost"
         size="sm"
@@ -148,7 +155,7 @@ export function SidebarNotifications() {
                 });
               }}
             >
-              {currentNotification.linkTitle ?? "Learn more"}
+              {currentNotification.linkTitle ?? "Learn more"} &rarr;
             </Link>
           </Button>
         )}

web/src/constants/VERSION.ts

@@ -1 +1 @@
-export const VERSION = "v2.92.0";
+export const VERSION = "v2.95.2";

web/src/ee/features/evals/components/evaluator-detail.tsx

@@ -11,7 +11,6 @@ import {
   PopoverTrigger,
 } from "@/src/components/ui/popover";
 import { useState } from "react";
-import { Trash } from "lucide-react";
 import TableLink from "@/src/components/table/table-link";
 import EvalLogTable from "@/src/ee/features/evals/components/eval-log";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
@@ -20,12 +19,23 @@ import { TableWithMetadataWrapper } from "@/src/components/table/TableWithMetada
 import { StatusBadge } from "@/src/components/layouts/status-badge";
 import { DetailPageNav } from "@/src/features/navigate-detail-pages/DetailPageNav";
 import { CardDescription } from "@/src/components/ui/card";
+import { EvaluatorStatus } from "@/src/ee/features/evals/types";
+import { Switch } from "@/src/components/ui/switch";
+import { Edit } from "lucide-react";
+import {
+  Dialog,
+  DialogContent,
+  DialogTitle,
+  DialogTrigger,
+} from "@/src/components/ui/dialog";
 
 export const EvaluatorDetail = () => {
   const router = useRouter();
   const projectId = router.query.projectId as string;
   const evaluatorId = router.query.evaluatorId as string;
 
+  const [isEditOpen, setIsEditOpen] = useState(false);
+
   // get the current template by id
   const evaluator = api.evals.configById.useQuery({
     projectId: projectId,
@@ -63,79 +73,102 @@ export const EvaluatorDetail = () => {
 
   return (
     <FullScreenPage>
-      <>
-        <Header
-          title={evaluator.data?.id ?? "Loading..."}
-          breadcrumb={[
-            {
-              name: "Evaluators",
-              href: `/project/${router.query.projectId as string}/evals`,
-            },
-            { name: evaluator.data?.id },
-          ]}
-          actionButtons={
-            <>
-              <DeactivateEvaluator
-                projectId={projectId}
-                evaluator={evaluator.data ?? undefined}
-                isLoading={evaluator.isLoading}
+      <Header
+        title={evaluator.data ? `Evaluator ${evaluator.data.id}` : "Loading..."}
+        breadcrumb={[
+          {
+            name: "Evaluators",
+            href: `/project/${router.query.projectId as string}/evals`,
+          },
+          { name: evaluator.data?.id },
+        ]}
+        actionButtons={
+          <>
+            <StatusBadge
+              type={evaluator.data?.status.toLowerCase()}
+              isLive
+              className="max-h-8"
+            />
+            <DeactivateEvaluator
+              projectId={projectId}
+              evaluator={evaluator.data ?? undefined}
+              isLoading={evaluator.isLoading}
+            />
+            {evaluator.data && (
+              <DetailPageNav
+                key="nav"
+                currentId={encodeURIComponent(evaluator.data.id)}
+                path={(entry) =>
+                  `/project/${projectId}/evals/${encodeURIComponent(entry.id)}`
+                }
+                listKey="evals"
               />
-              {evaluator.data && (
-                <DetailPageNav
-                  key="nav"
-                  currentId={encodeURIComponent(evaluator.data.id)}
-                  path={(entry) =>
-                    `/project/${projectId}/evals/${encodeURIComponent(entry.id)}`
-                  }
-                  listKey="evals"
+            )}
+          </>
+        }
+      />
+      {existingEvaluator && (
+        <TableWithMetadataWrapper
+          tableComponent={
+            <EvalLogTable
+              projectId={projectId}
+              jobConfigurationId={existingEvaluator.id}
+            />
+          }
+          cardTitleChildren={
+            <div className="flex w-full flex-row items-center justify-between">
+              <span>Evaluator configuration</span>
+              <Dialog open={isEditOpen} onOpenChange={setIsEditOpen}>
+                <DialogTrigger asChild>
+                  <Button variant="outline" className="flex items-center gap-2">
+                    <Edit className="h-4 w-4" />
+                  </Button>
+                </DialogTrigger>
+                <DialogContent className="max-w-screen-xl">
+                  <DialogTitle>Edit Evaluator</DialogTitle>
+                  <div className="max-h-[80vh] overflow-y-auto">
+                    <EvaluatorForm
+                      key={existingEvaluator.id}
+                      projectId={projectId}
+                      evalTemplates={allTemplates.data?.templates}
+                      existingEvaluator={existingEvaluator}
+                      shouldWrapVariables={true}
+                      mode="edit"
+                      onFormSuccess={() => {
+                        setIsEditOpen(false);
+                        // Force a reload as the form state is not properly updated
+                        void router.reload();
+                      }}
+                    />
+                  </div>
+                </DialogContent>
+              </Dialog>
+            </div>
+          }
+          cardContentChildren={
+            <>
+              <CardDescription className="flex items-center justify-between text-sm">
+                <span className="text-sm font-medium">Eval Template</span>
+                <TableLink
+                  path={`/project/${projectId}/evals/templates/${existingEvaluator.evalTemplateId}`}
+                  value={`${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})`}
+                  className="flex min-h-6 items-center"
                 />
-              )}
+              </CardDescription>
+              <div className="flex w-full flex-col items-start justify-between space-y-2 pb-4">
+                <EvaluatorForm
+                  key={existingEvaluator.id}
+                  projectId={projectId}
+                  evalTemplates={allTemplates.data?.templates}
+                  existingEvaluator={existingEvaluator}
+                  disabled={true}
+                  shouldWrapVariables={true}
+                />
+              </div>
             </>
           }
         />
-        {existingEvaluator && (
-          <TableWithMetadataWrapper
-            tableComponent={
-              <EvalLogTable
-                projectId={projectId}
-                jobConfigurationId={existingEvaluator.id}
-              />
-            }
-            cardTitleChildren={
-              <div className="flex w-full flex-row items-center justify-between">
-                <span>Evaluator</span>
-                <StatusBadge
-                  type={evaluator.data?.status.toLowerCase()}
-                  isLive
-                  className="max-h-8"
-                />
-              </div>
-            }
-            cardContentChildren={
-              <>
-                <CardDescription className="flex items-center justify-between text-sm">
-                  <span className="text-sm font-medium">Eval Template</span>
-                  <TableLink
-                    path={`/project/${projectId}/evals/templates/${existingEvaluator.evalTemplateId}`}
-                    value={`${existingEvaluator.evalTemplate.name} (v${existingEvaluator.evalTemplate.version})`}
-                    className="flex min-h-6 items-center"
-                  />
-                </CardDescription>
-                <div className="flex w-full flex-col items-start justify-between space-y-2 pb-4">
-                  <EvaluatorForm
-                    key={existingEvaluator.id}
-                    projectId={projectId}
-                    evalTemplates={allTemplates.data?.templates}
-                    existingEvaluator={existingEvaluator}
-                    disabled={true}
-                    shouldWrapVariables={true}
-                  />
-                </div>
-              </>
-            }
-          />
-        )}
-      </>
+      )}
     </FullScreenPage>
   );
 };
@@ -143,7 +176,6 @@ export const EvaluatorDetail = () => {
 export function DeactivateEvaluator({
   projectId,
   evaluator,
-  isLoading,
 }: {
   projectId: string;
   evaluator?: RouterOutputs["evals"]["configById"];
@@ -153,6 +185,7 @@ export function DeactivateEvaluator({
   const hasAccess = useHasProjectAccess({ projectId, scope: "evalJob:CUD" });
   const [isOpen, setIsOpen] = useState(false);
   const capture = usePostHogClientCapture();
+  const isActive = evaluator?.status === EvaluatorStatus.ACTIVE;
 
   const mutEvaluator = api.evals.updateEvalJob.useMutation({
     onSuccess: () => {
@@ -165,41 +198,50 @@ export function DeactivateEvaluator({
       console.error("Project ID is missing");
       return;
     }
+
+    const prevStatus = evaluator?.status;
+
     mutEvaluator.mutateAsync({
       projectId,
       evalConfigId: evaluator?.id ?? "",
-      updatedStatus: "INACTIVE",
+      config: {
+        status: isActive ? EvaluatorStatus.INACTIVE : EvaluatorStatus.ACTIVE,
+      },
     });
-    capture("eval_config:delete");
+    capture(
+      prevStatus === EvaluatorStatus.ACTIVE
+        ? "eval_config:deactivate"
+        : "eval_config:activate",
+    );
     setIsOpen(false);
   };
 
   return (
     <Popover open={isOpen} onOpenChange={() => setIsOpen(!isOpen)}>
       <PopoverTrigger asChild>
-        <Button
-          variant="outline"
-          size={"icon"}
-          disabled={!hasAccess || evaluator?.status !== "ACTIVE"}
-          loading={isLoading}
-        >
-          <Trash className="h-5 w-5" />
-        </Button>
+        <div className="flex items-center">
+          <Switch
+            disabled={!hasAccess}
+            checked={isActive}
+            className={isActive ? "data-[state=checked]:bg-dark-green" : ""}
+          />
+        </div>
       </PopoverTrigger>
       <PopoverContent>
         <h2 className="text-md mb-3 font-semibold">Please confirm</h2>
         <p className="mb-3 text-sm">
-          This action permanently deactivates the evaluator. No more traces will
-          be evaluated based on this evaluator.
+          {evaluator?.status === "ACTIVE"
+            ? "This action will deactivate the evaluator. No more traces will be evaluated based on this evaluator."
+            : "This action will activate the evaluator. New traces will be evaluated based on this evaluator."}
         </p>
         <div className="flex justify-end space-x-4">
           <Button
             type="button"
-            variant="destructive"
+            variant={evaluator?.status === "ACTIVE" ? "destructive" : "default"}
             loading={mutEvaluator.isLoading}
             onClick={onClick}
           >
-            Deactivate evaluator
+            {evaluator?.status === "ACTIVE" ? "Deactivate" : "Activate"}
           </Button>
         </div>
       </PopoverContent>

web/src/ee/features/evals/components/evaluator-form.tsx

@@ -69,9 +69,9 @@ import { showSuccessToast } from "@/src/features/notifications/showSuccessToast"
 const formSchema = z.object({
   scoreName: z.string(),
   target: z.string(),
-  filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
   mapping: z.array(wipVariableMapping),
-  sampling: z.coerce.number().gte(0).lte(1),
+  sampling: z.coerce.number().gt(0).lte(1),
   delay: z.coerce.number().optional().default(10),
 });
 
@@ -87,6 +87,7 @@ export const EvaluatorForm = (props: {
   disabled?: boolean;
   existingEvaluator?: JobConfiguration & { evalTemplate: EvalTemplate };
   onFormSuccess?: () => void;
+  mode?: "create" | "edit";
   shouldWrapVariables?: boolean;
 }) => {
   const [open, setOpen] = useState(false);
@@ -131,7 +132,7 @@ export const EvaluatorForm = (props: {
           <Popover open={open} onOpenChange={setOpen}>
             <PopoverTrigger asChild>
               <Button
-                disabled={props.disabled}
+                disabled={props.disabled || props.mode === "edit"}
                 variant="outline"
                 role="combobox"
                 aria-expanded={open}
@@ -193,7 +194,11 @@ export const EvaluatorForm = (props: {
           <Popover>
             <PopoverTrigger asChild>
               <Button
-                disabled={props.disabled || !selectedTemplateName}
+                disabled={
+                  props.disabled ||
+                  !selectedTemplateName ||
+                  props.mode === "edit"
+                }
                 variant="outline"
                 role="combobox"
                 className="w-1/3 justify-between px-2 font-normal"
@@ -276,6 +281,7 @@ export const EvaluatorForm = (props: {
           }
           onFormSuccess={props.onFormSuccess}
           shouldWrapVariables={props.shouldWrapVariables}
+          mode={props.mode}
         />
       ) : null}
     </>
@@ -289,6 +295,7 @@ export const InnerEvalConfigForm = (props: {
   existingEvaluator?: JobConfiguration;
   onFormSuccess?: () => void;
   shouldWrapVariables?: boolean;
+  mode?: "create" | "edit";
 }) => {
   const [formError, setFormError] = useState<string | null>(null);
   const capture = usePostHogClientCapture();
@@ -393,6 +400,10 @@ export const InnerEvalConfigForm = (props: {
     onSuccess: () => utils.models.invalidate(),
     onError: (error) => setFormError(error.message),
   });
+  const updateJobMutation = api.evals.updateEvalJob.useMutation({
+    onSuccess: () => utils.evals.invalidate(),
+    onError: (error) => setFormError(error.message),
+  });
   const [availableVariables, setAvailableVariables] = useState<
     typeof availableTraceEvalVariables | typeof availableDatasetEvalVariables
   >(
@@ -402,7 +413,11 @@ export const InnerEvalConfigForm = (props: {
   );
 
   function onSubmit(values: z.infer<typeof formSchema>) {
-    capture("eval_config:new_form_submit");
+    capture(
+      props.mode === "edit"
+        ? "eval_config:update"
+        : "eval_config:new_form_submit",
+    );
 
     const validatedFilter = z.array(singleFilter).safeParse(values.filter);
 
@@ -427,21 +442,42 @@ export const InnerEvalConfigForm = (props: {
       return;
     }
 
-    createJobMutation
-      .mutateAsync({
-        projectId: props.projectId,
-        evalTemplateId: props.evalTemplate.id,
-        scoreName: values.scoreName,
-        target: values.target,
-        filter: validatedFilter.data,
-        mapping: validatedVarMapping.data,
-        sampling: values.sampling,
-        delay: values.delay * 1000, // multiply by 1k to convert to ms
-      })
+    const delay = values.delay * 1000; // convert to ms
+    const sampling = values.sampling;
+    const mapping = validatedVarMapping.data;
+    const filter = validatedFilter.data;
+    const scoreName = values.scoreName;
+
+    (props.mode === "edit" && props.existingEvaluator
+      ? updateJobMutation.mutateAsync({
+          projectId: props.projectId,
+          evalConfigId: props.existingEvaluator.id,
+          config: {
+            delay,
+            filter,
+            variableMapping: mapping,
+            sampling,
+            scoreName,
+          },
+        })
+      : createJobMutation.mutateAsync({
+          projectId: props.projectId,
+          target: values.target,
+          evalTemplateId: props.evalTemplate.id,
+          scoreName,
+          filter,
+          mapping,
+          sampling,
+          delay,
+        })
+    )
       .then(() => {
-        props.onFormSuccess?.();
         form.reset();
-        void router.push(`/project/${props.projectId}/evals`);
+        props.onFormSuccess?.();
+
+        if (props.mode !== "edit") {
+          void router.push(`/project/${props.projectId}/evals`);
+        }
       })
       .catch((error) => {
         // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
@@ -507,10 +543,16 @@ export const InnerEvalConfigForm = (props: {
                       }}
                     >
                       <TabsList>
-                        <TabsTrigger value="trace" disabled={props.disabled}>
+                        <TabsTrigger
+                          value="trace"
+                          disabled={props.disabled || props.mode === "edit"}
+                        >
                           Trace
                         </TabsTrigger>
-                        <TabsTrigger value="dataset" disabled={props.disabled}>
+                        <TabsTrigger
+                          value="dataset"
+                          disabled={props.disabled || props.mode === "edit"}
+                        >
                           Dataset
                         </TabsTrigger>
                       </TabsList>
@@ -825,7 +867,7 @@ export const InnerEvalConfigForm = (props: {
         {!props.disabled ? (
           <Button
             type="submit"
-            loading={createJobMutation.isLoading}
+            loading={createJobMutation.isLoading || updateJobMutation.isLoading}
             className="mt-3"
           >
             Save

web/src/ee/features/evals/components/template-form.tsx

@@ -15,7 +15,7 @@ import {
 import { Textarea } from "@/src/components/ui/textarea";
 import { api } from "@/src/utils/api";
 import { zodResolver } from "@hookform/resolvers/zod";
-import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
+import { extractVariables, getIsCharOrUnderscore } from "@langfuse/shared";
 import router from "next/router";
 import { type EvalTemplate } from "@langfuse/shared";
 import { ModelParameters } from "@/src/components/ModelParameters";

web/src/ee/features/evals/server/addDatasetRunItemsToEvalQueue.ts

@@ -1,7 +1,10 @@
 import { env } from "@/src/env.mjs";
-import { DatasetRunItemUpsertQueue } from "../../../../../../packages/shared/dist/src/server/redis/datasetRunItemUpsert";
 import { randomUUID } from "crypto";
-import { QueueJobs, redis } from "@langfuse/shared/src/server";
+import {
+  QueueJobs,
+  DatasetRunItemUpsertQueue,
+  redis,
+} from "@langfuse/shared/src/server";
 
 export const addDatasetRunItemsToEvalQueue = async ({
   projectId,
@@ -18,30 +21,17 @@ export const addDatasetRunItemsToEvalQueue = async ({
     const queue = DatasetRunItemUpsertQueue.getInstance();
 
     if (queue) {
-      await queue.add(
-        QueueJobs.DatasetRunItemUpsert,
-        {
-          payload: {
-            projectId,
-            datasetItemId: datasetItemId,
-            traceId,
-            observationId: observationId ?? undefined,
-          },
-          id: randomUUID(),
-          timestamp: new Date(),
-          name: QueueJobs.DatasetRunItemUpsert as const,
+      await queue.add(QueueJobs.DatasetRunItemUpsert, {
+        payload: {
+          projectId,
+          datasetItemId: datasetItemId,
+          traceId,
+          observationId: observationId ?? undefined,
         },
-        {
-          attempts: 5, // retry 3 times
-          backoff: {
-            type: "exponential",
-            delay: 1000,
-          },
-          delay: 30000, // 10 seconds
-          removeOnComplete: true,
-          removeOnFail: 1_000,
-        },
-      );
+        id: randomUUID(),
+        timestamp: new Date(),
+        name: QueueJobs.DatasetRunItemUpsert as const,
+      });
     }
   }
 };

web/src/ee/features/evals/server/router.ts

@@ -12,6 +12,9 @@ import {
   singleFilter,
   variableMapping,
   ChatMessageRole,
+  type JobConfiguration,
+  JobConfigState,
+  JobType,
   Prisma,
 } from "@langfuse/shared";
 import { decrypt } from "@langfuse/shared/encryption";
@@ -23,6 +26,47 @@ import {
 } from "@langfuse/shared/src/server";
 import { TRPCError } from "@trpc/server";
 import { EvalReferencedEvaluators } from "@/src/ee/features/evals/types";
+import { EvaluatorStatus } from "../types";
+import { traceException } from "@langfuse/shared/src/server";
+
+const APIEvaluatorSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  evalTemplateId: z.string(),
+  scoreName: z.string(),
+  targetObject: z.string(),
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
+  variableMapping: z.array(variableMapping),
+  sampling: z.instanceof(Prisma.Decimal),
+  delay: z.number(),
+  status: z.nativeEnum(JobConfigState),
+  jobType: z.nativeEnum(JobType),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+});
+
+type APIEvaluator = z.infer<typeof APIEvaluatorSchema>;
+
+/**
+ * Use this function when pulling a list of evaluators from the database before using in the application to ensure type safety.
+ * All evaluators are expected to pass the validation. If an evaluator fails validation, it will be logged to Otel.
+ * @param evaluators
+ * @returns list of validated evaluators
+ */
+const filterAndValidateDbEvaluatorList = (
+  evaluators: JobConfiguration[],
+  onParseError?: (error: z.ZodError) => void,
+): APIEvaluator[] =>
+  evaluators.reduce((acc, ts) => {
+    const result = APIEvaluatorSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      console.error("Evaluator parsing error: ", result.error);
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as APIEvaluator[]);
 
 export const CreateEvalTemplate = z.object({
   name: z.string().min(1),
@@ -42,6 +86,26 @@ export const CreateEvalTemplate = z.object({
     .default(EvalReferencedEvaluators.PERSIST),
 });
 
+const CreateEvalJobSchema = z.object({
+  projectId: z.string(),
+  evalTemplateId: z.string(),
+  scoreName: z.string().min(1),
+  target: z.string(),
+  filter: z.array(singleFilter).nullable(), // reusing the filter type from the tables
+  mapping: z.array(variableMapping),
+  sampling: z.number().gt(0).lte(1),
+  delay: z.number().gte(0).default(DEFAULT_TRACE_JOB_DELAY), // 10 seconds default
+});
+
+const UpdateEvalJobSchema = z.object({
+  scoreName: z.string().min(1).optional(),
+  filter: z.array(singleFilter).optional(),
+  variableMapping: z.array(variableMapping).optional(),
+  sampling: z.number().gt(0).lte(1).optional(),
+  delay: z.number().gte(0).optional(),
+  status: z.nativeEnum(EvaluatorStatus).optional(),
+});
+
 export const evalRouter = createTRPCRouter({
   allConfigs: protectedProjectProcedure
     .input(
@@ -314,6 +378,31 @@ export const evalRouter = createTRPCRouter({
       }
     }),
 
+  jobConfigsByTarget: protectedProjectProcedure
+    .input(z.object({ projectId: z.string(), targetObject: z.string() }))
+    .query(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "model-based-evaluations",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "evalJob:read",
+      });
+
+      const evaluators = await ctx.prisma.jobConfiguration.findMany({
+        where: {
+          projectId: input.projectId,
+          targetObject: input.targetObject,
+          status: "ACTIVE",
+        },
+      });
+
+      return filterAndValidateDbEvaluatorList(evaluators, traceException);
+    }),
+
   jobConfigsByTemplateName: protectedProjectProcedure
     .input(z.object({ projectId: z.string(), evalTemplateName: z.string() }))
     .query(async ({ input, ctx }) => {
@@ -360,18 +449,7 @@ export const evalRouter = createTRPCRouter({
     }),
 
   createJob: protectedProjectProcedure
-    .input(
-      z.object({
-        projectId: z.string(),
-        evalTemplateId: z.string(),
-        scoreName: z.string().min(1),
-        target: z.string(),
-        filter: z.array(singleFilter).nullable(), // re-using the filter type from the tables
-        mapping: z.array(variableMapping),
-        sampling: z.number().gte(0).lte(1),
-        delay: z.number().gte(0).default(DEFAULT_TRACE_JOB_DELAY), // 10 seconds default
-      }),
-    )
+    .input(CreateEvalJobSchema)
     .mutation(async ({ input, ctx }) => {
       try {
         throwIfNoEntitlement({
@@ -545,7 +623,7 @@ export const evalRouter = createTRPCRouter({
       z.object({
         projectId: z.string(),
         evalConfigId: z.string(),
-        updatedStatus: z.enum(["ACTIVE", "INACTIVE"]),
+        config: UpdateEvalJobSchema,
       }),
     )
     .mutation(async ({ input, ctx }) => {
@@ -565,9 +643,7 @@ export const evalRouter = createTRPCRouter({
           id: input.evalConfigId,
           projectId: input.projectId,
         },
-        data: {
-          status: input.updatedStatus,
-        },
+        data: input.config,
       });
 
       await auditLog({

web/src/ee/features/evals/types.ts

@@ -1,4 +1,11 @@
+import { z } from "zod";
+import { JobConfigState } from "@langfuse/shared";
+
 export enum EvalReferencedEvaluators {
   UPDATE = "update",
   PERSIST = "persist",
 }
+
+export const EvaluatorStatus = JobConfigState;
+export const EvaluatorStatusSchema = z.nativeEnum(EvaluatorStatus);
+export type EvaluatorStatusType = z.infer<typeof EvaluatorStatusSchema>;

web/src/ee/features/experiments/components/CreateExperimentsForm.tsx

@@ -0,0 +1,742 @@
+import React, { useEffect, useMemo, useState } from "react";
+import { Button } from "@/src/components/ui/button";
+import {
+  FormControl,
+  FormDescription,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from "@/src/components/ui/form";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import { Form } from "@/src/components/ui/form";
+import { Textarea } from "@/src/components/ui/textarea";
+import { ModelParameters } from "@/src/components/ModelParameters";
+import {
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandList,
+  Command,
+  CommandItem,
+} from "@/src/components/ui/command";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/src/components/ui/select";
+import { z, type ZodSchema } from "zod";
+import { cn } from "@/src/utils/tailwind";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/src/components/ui/popover";
+import {
+  ChevronDown,
+  CheckIcon,
+  Info,
+  CircleCheck,
+  Loader2,
+} from "lucide-react";
+import { api } from "@/src/utils/api";
+import {
+  Card,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/src/components/ui/card";
+import { showErrorToast } from "@/src/features/notifications/showErrorToast";
+import { useModelParams } from "@/src/ee/features/playground/page/hooks/useModelParams";
+import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
+import {
+  type ColumnDefinition,
+  datasetCol,
+  extractVariables,
+  type FilterCondition,
+  stringOptionsFilter,
+  ZodModelConfig,
+} from "@langfuse/shared";
+import { MultiSelectKeyValues } from "@/src/features/scores/components/multi-select-key-values";
+import { useHasProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
+import { Skeleton } from "@/src/components/ui/skeleton";
+import { Input } from "@/src/components/ui/input";
+import { EvaluatorStatus } from "@/src/ee/features/evals/types";
+
+const CreateExperimentData = z.object({
+  name: z.string().min(1, "Please enter a name").optional(),
+  promptId: z.string().min(1, "Please select a prompt"),
+  datasetId: z.string().min(1, "Please select a dataset"),
+  description: z.string().max(1000).optional(),
+  modelConfig: z.object({
+    provider: z.string().min(1, "Please select a provider"),
+    model: z.string().min(1, "Please select a model"),
+    modelParams: ZodModelConfig,
+  }),
+});
+
+export type CreateExperiment = z.infer<typeof CreateExperimentData>;
+
+const isDatasetTarget = <T extends ZodSchema>(
+  filters: FilterCondition[] | null,
+  condition: {
+    column: ColumnDefinition;
+    schema: T;
+    isValid: (filter: z.infer<T>) => boolean;
+  },
+): boolean => {
+  if (!filters) return true;
+
+  const { column, schema, isValid } = condition;
+  const datasetFilters = filters.filter(
+    (filter) =>
+      (filter.column === column.id || column.name) &&
+      schema.safeParse(filter).success,
+  );
+  return datasetFilters.every((filter): boolean => isValid(filter));
+};
+
+export const CreateExperimentsForm = ({
+  projectId,
+  setFormOpen,
+  defaultValues = {},
+  promptDefault,
+  handleExperimentSettled,
+  handleExperimentSuccess,
+}: {
+  projectId: string;
+  setFormOpen: (open: boolean) => void;
+  defaultValues?: Partial<CreateExperiment>;
+  promptDefault?: {
+    name: string;
+    version: number;
+  };
+  handleExperimentSuccess?: (data?: {
+    success: boolean;
+    datasetId: string;
+    runId: string;
+    runName: string;
+  }) => Promise<void>;
+  handleExperimentSettled?: (data?: {
+    success: boolean;
+    datasetId: string;
+    runId: string;
+    runName: string;
+  }) => Promise<void>;
+}) => {
+  const [open, setOpen] = useState(false);
+  const [evaluatorOptions, setEvaluatorOptions] = useState<
+    { key: string; value: string }[]
+  >([]);
+  const [selectedEvaluators, setSelectedEvaluators] = useState<
+    { key: string; value: string }[]
+  >([]);
+  const [selectedPromptName, setSelectedPromptName] = useState<string>(
+    promptDefault?.name ?? "",
+  );
+  const [selectedPromptVersion, setSelectedPromptVersion] = useState<
+    number | null
+  >(promptDefault?.version ?? null);
+
+  const {
+    modelParams,
+    updateModelParamValue,
+    setModelParamEnabled,
+    availableModels,
+    availableProviders,
+  } = useModelParams();
+
+  const form = useForm<CreateExperiment>({
+    resolver: zodResolver(CreateExperimentData),
+    defaultValues: {
+      promptId: "",
+      datasetId: "",
+      modelConfig: {},
+      ...defaultValues,
+    },
+  });
+
+  const hasExperimentWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "experiments:CUD",
+  });
+
+  const hasEvalReadAccess = useHasProjectAccess({
+    projectId,
+    scope: "evalJob:read",
+  });
+
+  const hasEvalWriteAccess = useHasProjectAccess({
+    projectId,
+    scope: "evalJob:CUD",
+  });
+
+  const promptMeta = api.prompts.allPromptMeta.useQuery({
+    projectId,
+  });
+
+  const datasets = api.datasets.allDatasetMeta.useQuery(
+    { projectId },
+    {
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+      staleTime: Infinity,
+    },
+  );
+
+  const promptId = form.watch("promptId");
+  const datasetId = form.watch("datasetId");
+
+  const evaluators = api.evals.jobConfigsByTarget.useQuery(
+    { projectId, targetObject: "dataset" },
+    {
+      enabled: hasEvalReadAccess && !!datasetId,
+      trpc: {
+        context: {
+          skipBatch: true,
+        },
+      },
+      refetchOnMount: false,
+      refetchOnWindowFocus: false,
+      refetchOnReconnect: false,
+    },
+  );
+
+  const expectedColumns = useMemo(() => {
+    const prompt = promptMeta.data?.find((p) => p.id === promptId);
+    if (!prompt) return [];
+
+    return extractVariables(
+      prompt.type === PromptType.Text
+        ? (prompt?.prompt?.toString() ?? "")
+        : JSON.stringify(prompt?.prompt),
+    );
+  }, [promptId, promptMeta.data]);
+
+  useEffect(() => {
+    if (evaluators.data) {
+      const isValidFilter = (filter: z.infer<typeof stringOptionsFilter>) => {
+        const filterIncludesId = filter.value.includes(datasetId);
+        if (filter.operator === "any of") {
+          return filterIncludesId;
+        } else {
+          return !filterIncludesId;
+        }
+      };
+
+      const initialEvaluators = evaluators.data.reduce<
+        { key: string; value: string }[]
+      >((acc, evaluator) => {
+        if (
+          isDatasetTarget(evaluator.filter, {
+            column: datasetCol,
+            schema: stringOptionsFilter,
+            isValid: isValidFilter,
+          })
+        ) {
+          acc.push({
+            key: evaluator.id,
+            value: evaluator.scoreName,
+          });
+        }
+        return acc;
+      }, []);
+
+      setEvaluatorOptions(initialEvaluators);
+      setSelectedEvaluators(initialEvaluators);
+    }
+  }, [evaluators.data, datasetId]);
+
+  const validationResult = api.experiments.validateConfig.useQuery(
+    {
+      projectId,
+      promptId: promptId as string,
+      datasetId: datasetId as string,
+    },
+    {
+      enabled: Boolean(promptId && datasetId),
+    },
+  );
+
+  const experimentMutation = api.experiments.createExperiment.useMutation({
+    onSuccess: handleExperimentSuccess ?? (() => {}),
+    onError: (error) => {
+      showErrorToast(
+        error.message || "Failed to trigger experiment run",
+        "Please try again.",
+      );
+    },
+    onSettled: handleExperimentSettled ?? (() => {}),
+  });
+
+  const archiveEvaluatorMutation = api.evals.updateEvalJob.useMutation();
+
+  // Watch model config changes and update form
+  useEffect(() => {
+    form.setValue("modelConfig", {
+      provider: modelParams.provider.value,
+      model: modelParams.model.value,
+      modelParams: getFinalModelParams(modelParams),
+    });
+  }, [modelParams, form]);
+
+  const onSubmit = async (data: CreateExperiment) => {
+    const experiment = {
+      ...data,
+      projectId,
+    };
+    await experimentMutation.mutateAsync(experiment);
+    form.reset();
+    setFormOpen(false);
+  };
+
+  const handleOnValueChange = (
+    values: { key: string; value: string }[],
+    changedValueId?: string,
+  ) => {
+    if (!changedValueId) return;
+    const evaluator = evaluators.data?.find((e) => e.id === changedValueId);
+    if (!evaluator) return;
+
+    if (evaluator.status === "INACTIVE") {
+      const confirmed = window.confirm(
+        `Are you sure you want to activate "${evaluator.scoreName}"? You can always always archive the evaluator.`,
+      );
+      if (!confirmed) {
+        return;
+      }
+    } else {
+      const confirmed = window.confirm(
+        `Are you sure you want to archive "${evaluator.scoreName}"? You can always always re-activate the evaluator.`,
+      );
+      if (!confirmed) {
+        return;
+      }
+    }
+
+    archiveEvaluatorMutation.mutate({
+      projectId,
+      evalConfigId: changedValueId,
+      config: {
+        status:
+          evaluator.status === EvaluatorStatus.INACTIVE
+            ? EvaluatorStatus.ACTIVE
+            : EvaluatorStatus.INACTIVE,
+      },
+    });
+
+    setSelectedEvaluators(values);
+  };
+
+  const promptsByName = useMemo(
+    () =>
+      promptMeta.data?.reduce<
+        Record<string, Array<{ version: number; id: string }>>
+      >((acc, prompt) => {
+        if (!acc[prompt.name]) {
+          acc[prompt.name] = [];
+        }
+        acc[prompt.name].push({ version: prompt.version, id: prompt.id });
+        return acc;
+      }, {}),
+    [promptMeta.data],
+  );
+
+  if (!hasExperimentWriteAccess) {
+    return null;
+  }
+
+  if (
+    !promptMeta.data ||
+    !datasets.data ||
+    (hasEvalReadAccess && !!datasetId && !evaluators.data)
+  ) {
+    return <Skeleton className="min-h-[70dvh] w-full" />;
+  }
+
+  return (
+    <Form {...form}>
+      <form className="space-y-6" onSubmit={form.handleSubmit(onSubmit)}>
+        <FormField
+          control={form.control}
+          name="name"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Experiment name (optional)</FormLabel>
+              <FormControl>
+                <Input {...field} type="string" />
+              </FormControl>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+        <FormField
+          control={form.control}
+          name="description"
+          render={({ field }) => (
+            <FormItem>
+              <FormLabel>Description (optional)</FormLabel>
+              <FormControl>
+                <Textarea
+                  {...field}
+                  placeholder="Add description..."
+                  className="focus:outline-none focus:ring-0 focus-visible:ring-0 focus-visible:ring-offset-0 active:ring-0"
+                />
+              </FormControl>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+        <FormField
+          control={form.control}
+          name="promptId"
+          render={() => (
+            <FormItem>
+              <FormLabel>Prompt</FormLabel>
+              {/* FIX: I need the command list in the popover to be scrollable, currently it's not */}
+              <div className="mb-2 flex gap-2">
+                <Popover open={open} onOpenChange={setOpen}>
+                  <PopoverTrigger asChild>
+                    <Button
+                      variant="outline"
+                      role="combobox"
+                      aria-expanded={open}
+                      className="w-2/3 justify-between px-2 font-normal"
+                    >
+                      {selectedPromptName || "Select a prompt"}
+                      <ChevronDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+                    </Button>
+                  </PopoverTrigger>
+                  <PopoverContent
+                    className="w-[--radix-popover-trigger-width] overflow-auto p-0"
+                    align="start"
+                  >
+                    <Command>
+                      <CommandInput
+                        placeholder="Search prompts..."
+                        className="h-9"
+                      />
+                      <CommandList>
+                        <CommandEmpty>No prompt found.</CommandEmpty>
+                        <CommandGroup>
+                          {promptsByName &&
+                            Object.entries(promptsByName).map(
+                              ([name, promptData]) => (
+                                <CommandItem
+                                  key={name}
+                                  onSelect={() => {
+                                    setSelectedPromptName(name);
+                                    const latestVersion = promptData[0];
+                                    setSelectedPromptVersion(
+                                      latestVersion.version,
+                                    );
+                                    form.setValue("promptId", latestVersion.id);
+                                    form.clearErrors("promptId");
+                                  }}
+                                >
+                                  {name}
+                                  <CheckIcon
+                                    className={cn(
+                                      "ml-auto h-4 w-4",
+                                      name === selectedPromptName
+                                        ? "opacity-100"
+                                        : "opacity-0",
+                                    )}
+                                  />
+                                </CommandItem>
+                              ),
+                            )}
+                        </CommandGroup>
+                      </CommandList>
+                    </Command>
+                  </PopoverContent>
+                </Popover>
+
+                <Popover>
+                  <PopoverTrigger asChild>
+                    <Button
+                      disabled={!selectedPromptName}
+                      variant="outline"
+                      role="combobox"
+                      className="w-1/3 justify-between px-2 font-normal"
+                    >
+                      {selectedPromptVersion
+                        ? `Version ${selectedPromptVersion}`
+                        : "Version"}
+                      <ChevronDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
+                    </Button>
+                  </PopoverTrigger>
+                  <PopoverContent
+                    className="w-[--radix-popover-trigger-width] p-0"
+                    align="start"
+                  >
+                    <Command>
+                      <CommandList>
+                        <CommandEmpty>No version found.</CommandEmpty>
+                        <CommandGroup className="overflow-y-auto">
+                          {promptsByName &&
+                          selectedPromptName &&
+                          promptsByName[selectedPromptName] ? (
+                            promptsByName[selectedPromptName].map((prompt) => (
+                              <CommandItem
+                                key={prompt.id}
+                                onSelect={() => {
+                                  setSelectedPromptVersion(prompt.version);
+                                  form.setValue("promptId", prompt.id);
+                                  form.clearErrors("promptId");
+                                }}
+                              >
+                                Version {prompt.version}
+                                <CheckIcon
+                                  className={cn(
+                                    "ml-auto h-4 w-4",
+                                    prompt.version === selectedPromptVersion
+                                      ? "opacity-100"
+                                      : "opacity-0",
+                                  )}
+                                />
+                              </CommandItem>
+                            ))
+                          ) : (
+                            <CommandItem disabled>
+                              No versions available
+                            </CommandItem>
+                          )}
+                        </CommandGroup>
+                      </CommandList>
+                    </Command>
+                  </PopoverContent>
+                </Popover>
+              </div>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+
+        <FormField
+          control={form.control}
+          name="modelConfig"
+          render={() => (
+            <FormItem>
+              <Card className="p-4">
+                <ModelParameters
+                  {...{
+                    modelParams,
+                    availableModels,
+                    availableProviders,
+                    updateModelParamValue: updateModelParamValue,
+                    setModelParamEnabled,
+                    modelParamsDescription:
+                      "Select a model which supports function calling.",
+                  }}
+                  evalModelsOnly
+                />
+              </Card>
+              {form.formState.errors.modelConfig && (
+                <p
+                  id="modelConfig"
+                  className={cn("text-sm font-medium text-destructive")}
+                >
+                  {[
+                    form.formState.errors.modelConfig?.model?.message,
+                    form.formState.errors.modelConfig?.provider?.message,
+                  ].join(", ")}
+                </p>
+              )}
+            </FormItem>
+          )}
+        />
+
+        <FormField
+          control={form.control}
+          name="datasetId"
+          render={({ field }) => (
+            <FormItem>
+              <div className="flex items-center gap-2">
+                <FormLabel>Dataset</FormLabel>
+                <Popover>
+                  <PopoverTrigger asChild>
+                    <span className="cursor-pointer text-xs text-muted-foreground">
+                      (expected columns)
+                    </span>
+                  </PopoverTrigger>
+                  <PopoverContent className="w-80">
+                    <div className="flex flex-col space-y-2">
+                      <h4 className="text-sm font-medium leading-none">
+                        Expected columns
+                      </h4>
+                      <span className="text-sm text-muted-foreground">
+                        {promptId ? (
+                          <div>
+                            <span>
+                              Given current prompt, dataset item input must
+                              contain at least one of these first-level JSON
+                              keys, mapped to a string value:
+                            </span>
+                            <ul className="my-2 ml-2 list-inside list-disc">
+                              {expectedColumns.map((col) => (
+                                <li key={col}>{col}</li>
+                              ))}
+                            </ul>
+                            <span>
+                              These will be used as the input to your prompt.
+                            </span>
+                          </div>
+                        ) : (
+                          "Please select a prompt first"
+                        )}
+                      </span>
+                    </div>
+                  </PopoverContent>
+                </Popover>
+              </div>
+              <Select onValueChange={field.onChange} defaultValue={field.value}>
+                <FormControl>
+                  <SelectTrigger>
+                    <SelectValue placeholder="Select a dataset" />
+                  </SelectTrigger>
+                </FormControl>
+                <SelectContent>
+                  {datasets.data?.map((dataset) => (
+                    <SelectItem value={dataset.id} key={dataset.id}>
+                      {dataset.name}
+                    </SelectItem>
+                  ))}
+                </SelectContent>
+              </Select>
+              <FormMessage />
+            </FormItem>
+          )}
+        />
+
+        {evaluators.data && !!datasetId ? (
+          <FormItem>
+            <FormLabel>Evaluators</FormLabel>
+            <FormDescription>
+              Will run against your experiment results.
+            </FormDescription>
+            <MultiSelectKeyValues
+              key={datasetId}
+              placeholder="Value"
+              align="end"
+              className="grid grid-cols-[auto,1fr,auto,auto] gap-2"
+              disabled={!hasEvalWriteAccess}
+              onValueChange={handleOnValueChange}
+              options={evaluatorOptions}
+              values={
+                selectedEvaluators as {
+                  value: string;
+                  key: string;
+                }[]
+              }
+              hideClearButton
+              controlButtons={
+                <CommandItem
+                  onSelect={() => {
+                    window.open(`/project/${projectId}/evals`, "_blank");
+                  }}
+                >
+                  Manage evaluators
+                </CommandItem>
+              }
+            />
+          </FormItem>
+        ) : (
+          <FormItem>
+            <FormLabel>Evaluators</FormLabel>
+            {hasEvalReadAccess ? (
+              <FormDescription>
+                Select a dataset first to set up evaluators.
+              </FormDescription>
+            ) : (
+              <FormDescription>
+                ⓘ You do not have access to view evaluators. Please contact your
+                admin to upgrade your role.
+              </FormDescription>
+            )}
+          </FormItem>
+        )}
+
+        <div className="mt-4 flex flex-col gap-4">
+          {validationResult.isLoading && Boolean(promptId && datasetId) && (
+            <Card className="relative overflow-hidden rounded-md shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm">
+                  <span>Validating configuration...</span>
+                  <Loader2 className="h-3 w-3 animate-spin" />
+                </CardTitle>
+                <CardDescription className="text-foreground">
+                  Checking dataset items against prompt variables
+                </CardDescription>
+              </CardHeader>
+            </Card>
+          )}
+          {validationResult.data?.isValid === false && (
+            <Card className="relative overflow-hidden rounded-md border-dark-yellow bg-light-yellow shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm text-dark-yellow">
+                  <span>Invalid configuration</span>
+                  {/* TODO: add link to docs explaining error cases */}
+                  <Info className="h-4 w-4" />
+                </CardTitle>
+                <CardDescription className="text-foreground">
+                  {validationResult.data?.message}
+                </CardDescription>
+              </CardHeader>
+            </Card>
+          )}
+          {validationResult.data?.isValid === true && (
+            <Card className="relative overflow-hidden rounded-md border-dark-green bg-light-green shadow-none group-data-[collapsible=icon]:hidden">
+              <CardHeader className="p-2">
+                <CardTitle className="flex items-center justify-between text-sm text-dark-green">
+                  <span>Valid configuration</span>
+                  <CircleCheck className="h-4 w-4" />
+                </CardTitle>
+                <div className="text-sm">
+                  Matches between dataset items and prompt variables
+                  <ul className="my-2 ml-2 list-inside list-disc">
+                    {Object.entries(
+                      validationResult.data.variablesMap ?? {},
+                    ).map(([variable, count]) => (
+                      <li key={variable}>
+                        <strong>{variable}:</strong> {count} /{" "}
+                        {validationResult.data?.isValid
+                          ? validationResult.data.totalItems
+                          : "unknown"}
+                      </li>
+                    ))}
+                  </ul>
+                  Items missing all prompt variables will be excluded from the
+                  experiment.
+                </div>
+              </CardHeader>
+            </Card>
+          )}
+
+          <div className="flex justify-end">
+            <Button
+              type="submit"
+              disabled={
+                Boolean(promptId && datasetId) &&
+                !validationResult.data?.isValid
+              }
+              loading={form.formState.isSubmitting}
+            >
+              Create
+            </Button>
+          </div>
+        </div>
+      </form>
+    </Form>
+  );
+};

web/src/ee/features/experiments/server/router.ts

@@ -0,0 +1,220 @@
+import { z } from "zod";
+import { randomUUID } from "crypto";
+import {
+  type ExperimentMetadata,
+  QueueJobs,
+  QueueName,
+  redis,
+  ZodModelConfig,
+  ExperimentCreateQueue,
+} from "@langfuse/shared/src/server";
+import { env } from "@/src/env.mjs";
+import {
+  createTRPCRouter,
+  protectedProjectProcedure,
+} from "@/src/server/api/trpc";
+import { PromptType } from "@/src/features/prompts/server/utils/validation";
+import {
+  type DatasetItem,
+  extractVariables,
+  UnauthorizedError,
+} from "@langfuse/shared";
+import { throwIfNoEntitlement } from "@/src/features/entitlements/server/hasEntitlement";
+import { throwIfNoProjectAccess } from "@/src/features/rbac/utils/checkProjectAccess";
+
+const ValidConfigResponse = z.object({
+  isValid: z.literal(true),
+  totalItems: z.number(),
+  variablesMap: z.record(z.string(), z.number()),
+});
+
+const InvalidConfigResponse = z.object({
+  isValid: z.literal(false),
+  message: z.string(),
+});
+
+const ConfigResponse = z.discriminatedUnion("isValid", [
+  ValidConfigResponse,
+  InvalidConfigResponse,
+]);
+
+const validateDatasetItems = (
+  datasetItems: DatasetItem[],
+  variables: string[],
+): Record<string, number> => {
+  const variableMap: Record<string, number> = {};
+
+  for (const { input } of datasetItems) {
+    if (!input) {
+      continue;
+    }
+
+    const inputKeys = Object.keys(input);
+
+    // For each variable, increment its count if it exists in this item
+    for (const variable of variables) {
+      if (inputKeys.includes(variable)) {
+        variableMap[variable] = (variableMap[variable] || 0) + 1;
+      }
+    }
+  }
+
+  return variableMap;
+};
+
+export const experimentsRouter = createTRPCRouter({
+  validateConfig: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        datasetId: z.string(),
+        promptId: z.string(),
+      }),
+    )
+    .output(ConfigResponse)
+    .query(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "experiments",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "experiments:CUD",
+      });
+
+      const prompt = await ctx.prisma.prompt.findFirst({
+        where: {
+          id: input.promptId,
+          projectId: input.projectId,
+        },
+      });
+
+      if (!prompt) {
+        return {
+          isValid: false,
+          message: "Selected prompt not found.",
+        };
+      }
+
+      const extractedVariables = extractVariables(
+        prompt?.type === PromptType.Text
+          ? (prompt.prompt?.toString() ?? "")
+          : JSON.stringify(prompt.prompt),
+      );
+
+      if (!Boolean(extractedVariables.length)) {
+        return {
+          isValid: false,
+          message: "Selected prompt has no variables.",
+        };
+      }
+
+      const datasetItems = await ctx.prisma.datasetItem.findMany({
+        where: {
+          datasetId: input.datasetId,
+          projectId: input.projectId,
+        },
+      });
+
+      if (!Boolean(datasetItems.length)) {
+        return {
+          isValid: false,
+          message: "Selected dataset is empty.",
+        };
+      }
+
+      const variablesMap = validateDatasetItems(
+        datasetItems,
+        extractedVariables,
+      );
+
+      if (!Boolean(Object.keys(variablesMap).length)) {
+        return {
+          isValid: false,
+          message: "No dataset item contains any variables.",
+        };
+      }
+
+      return {
+        isValid: true,
+        totalItems: datasetItems.length,
+        variablesMap: variablesMap,
+      };
+    }),
+
+  createExperiment: protectedProjectProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        name: z.string().optional(),
+        promptId: z.string().min(1, "Please select a prompt"),
+        datasetId: z.string().min(1, "Please select a dataset"),
+        description: z.string().max(1000).optional(),
+        modelConfig: z.object({
+          provider: z.string().min(1, "Please select a provider"),
+          model: z.string().min(1, "Please select a model"),
+          modelParams: ZodModelConfig,
+        }),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      throwIfNoEntitlement({
+        entitlement: "experiments",
+        projectId: input.projectId,
+        sessionUser: ctx.session.user,
+      });
+      throwIfNoProjectAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "experiments:CUD",
+      });
+
+      if (!redis || !env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
+        throw new UnauthorizedError("Experiment creation failed");
+      }
+
+      const metadata: ExperimentMetadata = {
+        prompt_id: input.promptId,
+        provider: input.modelConfig.provider,
+        model: input.modelConfig.model,
+        model_params: input.modelConfig.modelParams,
+      };
+      const name =
+        input.name ?? `${input.promptId}-${new Date().toISOString()}`;
+
+      const datasetRun = await ctx.prisma.datasetRuns.create({
+        data: {
+          name: name,
+          description: input.description,
+          datasetId: input.datasetId,
+          metadata: metadata,
+          projectId: input.projectId,
+        },
+      });
+
+      const queue = ExperimentCreateQueue.getInstance();
+
+      if (queue) {
+        await queue.add(QueueName.ExperimentCreate, {
+          name: QueueJobs.ExperimentCreateJob,
+          id: randomUUID(),
+          timestamp: new Date(),
+          payload: {
+            projectId: input.projectId,
+            datasetId: input.datasetId,
+            runId: datasetRun.id,
+            description: input.description,
+          },
+        });
+      }
+
+      return {
+        success: true,
+        datasetId: input.datasetId,
+        runId: datasetRun.id,
+        runName: name,
+      };
+    }),
+});

web/src/ee/features/multi-tenant-sso/types.ts

@@ -28,6 +28,20 @@ export const GithubProviderSchema = base.extend({
     .nullish(),
 });
 
+export const GithubEnterpriseProviderSchema = base.extend({
+  authProvider: z.literal("github-enterprise"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      enterprise: z.object({
+        baseUrl: z.string().url(),
+      }),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const GitlabProviderSchema = base.extend({
   authProvider: z.literal("gitlab"),
   authConfig: z
@@ -88,6 +102,18 @@ export const CognitoProviderSchema = base.extend({
     .nullish(),
 });
 
+export const KeycloakProviderSchema = base.extend({
+  authProvider: z.literal("keycloak"),
+  authConfig: z
+    .object({
+      clientId: z.string(),
+      clientSecret: z.string(),
+      issuer: z.string(),
+      allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
+    })
+    .nullish(),
+});
+
 export const CustomProviderSchema = base.extend({
   authProvider: z.literal("custom"),
   authConfig: z
@@ -97,6 +123,7 @@ export const CustomProviderSchema = base.extend({
       clientSecret: z.string(),
       issuer: z.string(),
       scope: z.string().nullish(),
+      idToken: z.boolean().optional().default(true),
       allowDangerousEmailAccountLinking: z.boolean().optional().default(false),
     })
     .nullish(),
@@ -104,21 +131,27 @@ export const CustomProviderSchema = base.extend({
 
 export type GoogleProviderSchema = z.infer<typeof GoogleProviderSchema>;
 export type GithubProviderSchema = z.infer<typeof GithubProviderSchema>;
+export type GithubEnterpriseProviderSchema = z.infer<
+  typeof GithubEnterpriseProviderSchema
+>;
 export type GitlabProviderSchema = z.infer<typeof GitlabProviderSchema>;
 export type Auth0ProviderSchema = z.infer<typeof Auth0ProviderSchema>;
 export type OktaProviderSchema = z.infer<typeof OktaProviderSchema>;
 export type AzureAdProviderSchema = z.infer<typeof AzureAdProviderSchema>;
 export type CognitoProviderSchema = z.infer<typeof CognitoProviderSchema>;
+export type KeycloakProviderSchema = z.infer<typeof KeycloakProviderSchema>;
 export type CustomProviderSchema = z.infer<typeof CustomProviderSchema>;
 
 export const SsoProviderSchema = z.discriminatedUnion("authProvider", [
   GoogleProviderSchema,
   GithubProviderSchema,
+  GithubEnterpriseProviderSchema,
   GitlabProviderSchema,
   Auth0ProviderSchema,
   OktaProviderSchema,
   AzureAdProviderSchema,
   CognitoProviderSchema,
+  KeycloakProviderSchema,
   CustomProviderSchema,
 ]);
 

web/src/ee/features/multi-tenant-sso/utils.ts

@@ -4,6 +4,7 @@ import GitHubProvider from "next-auth/providers/github";
 import GitLabProvider from "next-auth/providers/gitlab";
 import OktaProvider from "next-auth/providers/okta";
 import CognitoProvider from "next-auth/providers/cognito";
+import KeycloakProvider from "next-auth/providers/keycloak";
 import Auth0Provider from "next-auth/providers/auth0";
 import AzureADProvider from "next-auth/providers/azure-ad";
 import { isEeEnabled } from "@/src/ee/utils/isEeEnabled";
@@ -12,6 +13,7 @@ import { decrypt } from "@langfuse/shared/encryption";
 import { SsoProviderSchema } from "./types";
 import {
   CustomSSOProvider,
+  GitHubEnterpriseProvider,
   logger,
   traceException,
 } from "@langfuse/shared/src/server";
@@ -188,6 +190,12 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
       ...provider.authConfig,
       clientSecret: decrypt(provider.authConfig.clientSecret),
     });
+  else if (provider.authProvider === "keycloak")
+    return KeycloakProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+    });
   else if (provider.authProvider === "custom")
     return CustomSSOProvider({
       id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
@@ -197,6 +205,15 @@ const dbToNextAuthProvider = (provider: SsoProviderSchema): Provider | null => {
         params: { scope: provider.authConfig.scope ?? "openid email profile" },
       },
     });
+  else if (provider.authProvider === "github-enterprise")
+    return GitHubEnterpriseProvider({
+      id: getAuthProviderIdForSsoConfig(provider), // use the domain as the provider id as we use domain-specific credentials
+      ...provider.authConfig,
+      clientSecret: decrypt(provider.authConfig.clientSecret),
+      enterprise: {
+        baseUrl: provider.authConfig.enterprise.baseUrl,
+      },
+    });
   else {
     // Type check to ensure we handle all providers
     // eslint-disable-next-line @typescript-eslint/no-unused-vars

web/src/ee/features/playground/page/context/index.tsx

@@ -16,9 +16,9 @@ import usePlaygroundCache from "@/src/ee/features/playground/page/hooks/usePlayg
 import { getFinalModelParams } from "@/src/ee/utils/getFinalModelParams";
 import { usePostHogClientCapture } from "@/src/features/posthog-analytics/usePostHogClientCapture";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
-import { extractVariables } from "@/src/utils/string";
 import {
   ChatMessageRole,
+  extractVariables,
   type ChatMessageWithId,
   type PromptVariable,
   type UIModelParams,

web/src/env.mjs

@@ -1,6 +1,26 @@
 import { z } from "zod";
 import { createEnv } from "@t3-oss/env-nextjs";
 
+const zAuthMethod = z
+  .enum([
+    "client_secret_basic",
+    "client_secret_post",
+    "client_secret_jwt",
+    "private_key_jwt",
+    "tls_client_auth",
+    "self_signed_tls_client_auth",
+    "none",
+  ])
+  .optional()
+  .default("client_secret_basic");
+
+
+const zAuthChecks = z
+  .string()
+  .optional()
+  .transform((s) => s?.split(",").map((s) => s.trim()))
+  .pipe(z.array(z.enum(["nonce", "none", "pkce", "state"])).optional());
+
 export const env = createEnv({
   /**
    * Specify your server-side environment variables schema here. This way you can ensure the app
@@ -46,7 +66,7 @@ export const env = createEnv({
     LANGFUSE_DEFAULT_PROJECT_ROLE: z
       .enum(["OWNER", "ADMIN", "MEMBER", "VIEWER"])
       .optional(),
-    LANGFUSE_CSP_ENFORCE_HTTPS: z.enum(["true", "false"]).optional(),
+    LANGFUSE_CSP_ENFORCE_HTTPS: z.enum(["true", "false"]).optional().default("false"),
     // Telemetry
     TELEMETRY_ENABLED: z.enum(["true", "false"]).optional(),
     // AUTH
@@ -54,36 +74,68 @@ export const env = createEnv({
     AUTH_GOOGLE_CLIENT_SECRET: z.string().optional(),
     AUTH_GOOGLE_ALLOWED_DOMAINS: z.string().optional(),
     AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GOOGLE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GOOGLE_CHECKS: zAuthChecks,
     AUTH_GITHUB_CLIENT_ID: z.string().optional(),
     AUTH_GITHUB_CLIENT_SECRET: z.string().optional(),
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_GITHUB_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITHUB_CHECKS: zAuthChecks,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_ID: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_BASE_URL: z.string().optional(),
+    AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING: z
+      .enum(["true", "false"])
+      .optional(),
+    AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITHUB_ENTERPRISE_CHECKS: zAuthChecks,
     AUTH_GITLAB_CLIENT_ID: z.string().optional(),
     AUTH_GITLAB_CLIENT_SECRET: z.string().optional(),
     AUTH_GITLAB_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
     AUTH_GITLAB_ISSUER: z.string().optional(),
+    AUTH_GITLAB_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_GITLAB_CHECKS: zAuthChecks,
     AUTH_AZURE_AD_CLIENT_ID: z.string().optional(),
     AUTH_AZURE_AD_CLIENT_SECRET: z.string().optional(),
     AUTH_AZURE_AD_TENANT_ID: z.string().optional(),
     AUTH_AZURE_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_AZURE_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_AZURE_CHECKS: zAuthChecks,
     AUTH_OKTA_CLIENT_ID: z.string().optional(),
     AUTH_OKTA_CLIENT_SECRET: z.string().optional(),
     AUTH_OKTA_ISSUER: z.string().optional(),
     AUTH_OKTA_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_OKTA_CHECKS: zAuthChecks,
+    AUTH_OKTA_CLIENT_AUTH_METHOD: zAuthMethod,
     AUTH_AUTH0_CLIENT_ID: z.string().optional(),
     AUTH_AUTH0_CLIENT_SECRET: z.string().optional(),
     AUTH_AUTH0_ISSUER: z.string().url().optional(),
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_AUTH0_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_AUTH0_CHECKS: zAuthChecks,
     AUTH_COGNITO_CLIENT_ID: z.string().optional(),
     AUTH_COGNITO_CLIENT_SECRET: z.string().optional(),
     AUTH_COGNITO_ISSUER: z.string().url().optional(),
     AUTH_COGNITO_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_COGNITO_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_COGNITO_CHECKS: zAuthChecks,
+    AUTH_KEYCLOAK_CLIENT_ID: z.string().optional(),
+    AUTH_KEYCLOAK_CLIENT_SECRET: z.string().optional(),
+    AUTH_KEYCLOAK_ISSUER: z.string().optional(),
+    AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_KEYCLOAK_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_KEYCLOAK_CHECKS: zAuthChecks,
     AUTH_CUSTOM_CLIENT_ID: z.string().optional(),
     AUTH_CUSTOM_CLIENT_SECRET: z.string().optional(),
     AUTH_CUSTOM_ISSUER: z.string().url().optional(),
     AUTH_CUSTOM_NAME: z.string().optional(),
     AUTH_CUSTOM_SCOPE: z.string().optional(),
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD: zAuthMethod,
+    AUTH_CUSTOM_CHECKS: zAuthChecks,
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING: z.enum(["true", "false"]).optional(),
+    AUTH_CUSTOM_ID_TOKEN: z.enum(["true", "false"]).optional(),
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT: z.string().optional(),
+    AUTH_IGNORE_ACCOUNT_FIELDS: z.string().optional(),
     AUTH_DISABLE_USERNAME_PASSWORD: z.enum(["true", "false"]).optional(),
     AUTH_DISABLE_SIGNUP: z.enum(["true", "false"]).optional(),
     AUTH_SESSION_MAX_AGE: z.coerce
@@ -95,6 +147,8 @@ export const env = createEnv({
       )
       .optional()
       .default(30 * 24 * 60), // default to 30 days
+    AUTH_HTTP_PROXY: z.string().url().optional(),
+    AUTH_HTTPS_PROXY: z.string().url().optional(),
     // EMAIL
     EMAIL_FROM_ADDRESS: z.string().optional(),
     SMTP_CONNECTION_URL: z.string().optional(),
@@ -119,6 +173,7 @@ export const env = createEnv({
     CLICKHOUSE_URL: z.string().optional(),
     CLICKHOUSE_USER: z.string().optional(),
     CLICKHOUSE_PASSWORD: z.string().optional(),
+    CLICKHOUSE_CLUSTER_ENABLED: z.enum(["true", "false"]).default("false"),
     // EE ui customization
     LANGFUSE_UI_API_HOST: z.string().optional(),
     LANGFUSE_UI_DOCUMENTATION_HREF: z.string().url().optional(),
@@ -196,6 +251,7 @@ export const env = createEnv({
     LANGFUSE_READ_FROM_POSTGRES_ONLY: z.enum(["true", "false"]).default("true"),
     LANGFUSE_RETURN_FROM_CLICKHOUSE: z.enum(["true", "false"]).default("false"),
     LANGFUSE_EXPERIMENT_EXCLUDED_PROJECT_IDS: z.string().optional(),
+    LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS: z.string().optional(),
     LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE: z
       .enum(["true", "false"])
       .default("false"),
@@ -264,6 +320,8 @@ export const env = createEnv({
     NEXT_PUBLIC_SIGN_UP_DISABLED: process.env.NEXT_PUBLIC_SIGN_UP_DISABLED,
     LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:
       process.env.LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES,
+    LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS:
+      process.env.LANGFUSE_EXPERIMENT_EXCLUDED_OPERATIONS,
     LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES:
       process.env.LANGFUSE_DISABLE_EXPENSIVE_POSTGRES_QUERIES,
     LANGFUSE_TEAM_SLACK_WEBHOOK: process.env.LANGFUSE_TEAM_SLACK_WEBHOOK,
@@ -283,47 +341,85 @@ export const env = createEnv({
     AUTH_GOOGLE_ALLOWED_DOMAINS: process.env.AUTH_GOOGLE_ALLOWED_DOMAINS,
     AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GOOGLE_ALLOW_ACCOUNT_LINKING,
+    AUTH_GOOGLE_CLIENT_AUTH_METHOD: process.env.AUTH_GOOGLE_CLIENT_AUTH_METHOD,
+    AUTH_GOOGLE_CHECKS: process.env.AUTH_GOOGLE_CHECKS,
     AUTH_GITHUB_CLIENT_ID: process.env.AUTH_GITHUB_CLIENT_ID,
     AUTH_GITHUB_CLIENT_SECRET: process.env.AUTH_GITHUB_CLIENT_SECRET,
     AUTH_GITHUB_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GITHUB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITHUB_CLIENT_AUTH_METHOD: process.env.AUTH_GITHUB_CLIENT_AUTH_METHOD,
+    AUTH_GITHUB_CHECKS: process.env.AUTH_GITHUB_CHECKS,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_ID:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_ID,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_SECRET,
+    AUTH_GITHUB_ENTERPRISE_BASE_URL:
+      process.env.AUTH_GITHUB_ENTERPRISE_BASE_URL,
+    AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_GITHUB_ENTERPRISE_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD:
+      process.env.AUTH_GITHUB_ENTERPRISE_CLIENT_AUTH_METHOD,
+    AUTH_GITHUB_ENTERPRISE_CHECKS: process.env.AUTH_GITHUB_ENTERPRISE_CHECKS,
     AUTH_GITLAB_ISSUER: process.env.AUTH_GITLAB_ISSUER,
     AUTH_GITLAB_CLIENT_ID: process.env.AUTH_GITLAB_CLIENT_ID,
     AUTH_GITLAB_CLIENT_SECRET: process.env.AUTH_GITLAB_CLIENT_SECRET,
     AUTH_GITLAB_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_GITLAB_ALLOW_ACCOUNT_LINKING,
+    AUTH_GITLAB_CLIENT_AUTH_METHOD: process.env.AUTH_GITLAB_CLIENT_AUTH_METHOD,
+    AUTH_GITLAB_CHECKS: process.env.AUTH_GITLAB_CHECKS,
     AUTH_AZURE_AD_CLIENT_ID: process.env.AUTH_AZURE_AD_CLIENT_ID,
     AUTH_AZURE_AD_CLIENT_SECRET: process.env.AUTH_AZURE_AD_CLIENT_SECRET,
     AUTH_AZURE_AD_TENANT_ID: process.env.AUTH_AZURE_AD_TENANT_ID,
     AUTH_AZURE_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AZURE_ALLOW_ACCOUNT_LINKING,
+    AUTH_AZURE_CLIENT_AUTH_METHOD: process.env.AUTH_AZURE_CLIENT_AUTH_METHOD,
+    AUTH_AZURE_CHECKS: process.env.AUTH_AZURE_CHECKS,
     AUTH_OKTA_CLIENT_ID: process.env.AUTH_OKTA_CLIENT_ID,
     AUTH_OKTA_CLIENT_SECRET: process.env.AUTH_OKTA_CLIENT_SECRET,
     AUTH_OKTA_ISSUER: process.env.AUTH_OKTA_ISSUER,
     AUTH_OKTA_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_OKTA_ALLOW_ACCOUNT_LINKING,
+    AUTH_OKTA_CLIENT_AUTH_METHOD: process.env.AUTH_OKTA_CLIENT_AUTH_METHOD,
+    AUTH_OKTA_CHECKS: process.env.AUTH_OKTA_CHECKS,
     AUTH_AUTH0_CLIENT_ID: process.env.AUTH_AUTH0_CLIENT_ID,
     AUTH_AUTH0_CLIENT_SECRET: process.env.AUTH_AUTH0_CLIENT_SECRET,
     AUTH_AUTH0_ISSUER: process.env.AUTH_AUTH0_ISSUER,
     AUTH_AUTH0_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_AUTH0_ALLOW_ACCOUNT_LINKING,
+    AUTH_AUTH0_CLIENT_AUTH_METHOD: process.env.AUTH_AUTH0_CLIENT_AUTH_METHOD,
+    AUTH_AUTH0_CHECKS: process.env.AUTH_AUTH0_CHECKS,
     AUTH_COGNITO_CLIENT_ID: process.env.AUTH_COGNITO_CLIENT_ID,
     AUTH_COGNITO_CLIENT_SECRET: process.env.AUTH_COGNITO_CLIENT_SECRET,
     AUTH_COGNITO_ISSUER: process.env.AUTH_COGNITO_ISSUER,
     AUTH_COGNITO_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_COGNITO_ALLOW_ACCOUNT_LINKING,
+    AUTH_COGNITO_CLIENT_AUTH_METHOD: process.env.AUTH_COGNITO_CLIENT_AUTH_METHOD,
+    AUTH_COGNITO_CHECKS: process.env.AUTH_COGNITO_CHECKS,
+    AUTH_KEYCLOAK_CLIENT_ID: process.env.AUTH_KEYCLOAK_CLIENT_ID,
+    AUTH_KEYCLOAK_CLIENT_SECRET: process.env.AUTH_KEYCLOAK_CLIENT_SECRET,
+    AUTH_KEYCLOAK_ISSUER: process.env.AUTH_KEYCLOAK_ISSUER,
+    AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING:
+      process.env.AUTH_KEYCLOAK_ALLOW_ACCOUNT_LINKING,
+    AUTH_KEYCLOAK_CLIENT_AUTH_METHOD: process.env.AUTH_KEYCLOAK_CLIENT_AUTH_METHOD,
+    AUTH_KEYCLOAK_CHECKS: process.env.AUTH_KEYCLOAK_CHECKS,
     AUTH_CUSTOM_CLIENT_ID: process.env.AUTH_CUSTOM_CLIENT_ID,
     AUTH_CUSTOM_CLIENT_SECRET: process.env.AUTH_CUSTOM_CLIENT_SECRET,
     AUTH_CUSTOM_ISSUER: process.env.AUTH_CUSTOM_ISSUER,
     AUTH_CUSTOM_NAME: process.env.AUTH_CUSTOM_NAME,
     AUTH_CUSTOM_SCOPE: process.env.AUTH_CUSTOM_SCOPE,
+    AUTH_CUSTOM_CLIENT_AUTH_METHOD: process.env.AUTH_CUSTOM_CLIENT_AUTH_METHOD,
+    AUTH_CUSTOM_CHECKS: process.env.AUTH_CUSTOM_CHECKS,
     AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING:
       process.env.AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING,
+    AUTH_CUSTOM_ID_TOKEN: process.env.AUTH_CUSTOM_ID_TOKEN,
+    AUTH_IGNORE_ACCOUNT_FIELDS: process.env.AUTH_IGNORE_ACCOUNT_FIELDS,
     AUTH_DOMAINS_WITH_SSO_ENFORCEMENT:
       process.env.AUTH_DOMAINS_WITH_SSO_ENFORCEMENT,
     AUTH_DISABLE_USERNAME_PASSWORD: process.env.AUTH_DISABLE_USERNAME_PASSWORD,
     AUTH_DISABLE_SIGNUP: process.env.AUTH_DISABLE_SIGNUP,
     AUTH_SESSION_MAX_AGE: process.env.AUTH_SESSION_MAX_AGE,
+    AUTH_HTTP_PROXY: process.env.AUTH_HTTP_PROXY,
+    AUTH_HTTPS_PROXY: process.env.AUTH_HTTPS_PROXY,
     // Email
     EMAIL_FROM_ADDRESS: process.env.EMAIL_FROM_ADDRESS,
     SMTP_CONNECTION_URL: process.env.SMTP_CONNECTION_URL,
@@ -373,6 +469,7 @@ export const env = createEnv({
     CLICKHOUSE_URL: process.env.CLICKHOUSE_URL,
     CLICKHOUSE_USER: process.env.CLICKHOUSE_USER,
     CLICKHOUSE_PASSWORD: process.env.CLICKHOUSE_PASSWORD,
+    CLICKHOUSE_CLUSTER_ENABLED: process.env.CLICKHOUSE_CLUSTER_ENABLED,
     // EE ui customization
     LANGFUSE_UI_API_HOST: process.env.LANGFUSE_UI_API_HOST,
     LANGFUSE_UI_DOCUMENTATION_HREF: process.env.LANGFUSE_UI_DOCUMENTATION_HREF,

web/src/features/datasets/components/DatasetAggregateTableCell.tsx

@@ -47,6 +47,7 @@ const DatasetAggregateCell = ({
       refetchOnWindowFocus: false,
       refetchOnReconnect: false,
       staleTime: Infinity,
+      onError: () => {},
     },
   );
   const observation = api.observations.byId.useQuery(
@@ -66,6 +67,7 @@ const DatasetAggregateCell = ({
       refetchOnWindowFocus: false,
       refetchOnReconnect: false,
       staleTime: Infinity,
+      onError: () => {},
     },
   );
 
@@ -81,7 +83,10 @@ const DatasetAggregateCell = ({
       {variant === "peek" && actionButtons}
       <div className="flex flex-row items-center justify-center gap-1">
         <IOTableCell
-          isLoading={!!!observationId ? trace.isLoading : observation.isLoading}
+          isLoading={
+            (!!!observationId ? trace.isLoading : observation.isLoading) ||
+            !data
+          }
           data={data?.output}
           className={"bg-accent-light-green"}
           singleLine={singleLine}

web/src/features/datasets/components/DatasetCompareRunsTable.tsx

@@ -10,7 +10,7 @@ import { type ScoreAggregate } from "@langfuse/shared";
 import { type Prisma } from "@langfuse/shared";
 import { NumberParam } from "use-query-params";
 import { useQueryParams, withDefault } from "use-query-params";
-import { useMemo, useState } from "react";
+import { useMemo, useState, useCallback } from "react";
 import { usdFormatter } from "@/src/utils/numbers";
 import { getScoreDataTypeIcon } from "@/src/features/scores/components/ScoreDetailColumnHelpers";
 import { api, type RouterOutputs } from "@/src/utils/api";
@@ -24,6 +24,8 @@ import {
 } from "@/src/components/ui/dropdown-menu";
 import { DatasetCompareRunPeekView } from "@/src/features/datasets/components/DatasetCompareRunPeekView";
 import { useClickhouse } from "@/src/components/layouts/ClickhouseAdminToggle";
+import { getQueryKey } from "@trpc/react-query";
+import { useQueryClient } from "@tanstack/react-query";
 
 export type RunMetrics = {
   id: string;
@@ -47,6 +49,16 @@ export type DatasetCompareRunRowData = {
   runs?: RunAggregate;
 };
 
+const getRefetchInterval = (
+  runId: string,
+  localExperiments: { key: string; value: string }[],
+  unchangedCounts: Record<string, number>,
+) => {
+  if (unchangedCounts[runId] < 2) return 5000;
+  if (localExperiments.some((run) => run.key === runId)) return 3000;
+  return false;
+};
+
 const DATASET_RUN_METRICS = ["scores", "resourceMetrics"] as const;
 export type DatasetRunMetric = (typeof DATASET_RUN_METRICS)[number];
 
@@ -55,6 +67,7 @@ export function DatasetCompareRunsTable(props: {
   datasetId: string;
   runIds: string[];
   runsData?: RouterOutputs["datasets"]["baseRunDataByDatasetId"];
+  localExperiments: { key: string; value: string }[];
 }) {
   const [selectedMetrics, setSelectedMetrics] = useState<DatasetRunMetric[]>([
     "scores",
@@ -69,6 +82,10 @@ export function DatasetCompareRunsTable(props: {
     traceId: string;
     observationId?: string;
   } | null>(null);
+  const [unchangedCounts, setUnchangedCounts] = useState<
+    Record<string, number>
+  >({});
+  const queryClient = useQueryClient();
 
   const rowHeight = "l";
 
@@ -84,8 +101,51 @@ export function DatasetCompareRunsTable(props: {
     limit: paginationState.pageSize,
   });
   const queryClickhouse = useClickhouse();
-  // Individual queries for each run
-  const runs = (props.runIds ?? []).map((runId) => ({
+
+  // 1. First, separate the run definitions
+  const runQueries = useMemo(
+    () =>
+      (props.runIds ?? []).map((runId) => ({
+        runId,
+        queryKey: getQueryKey(api.datasets.runitemsByRunIdOrItemId, {
+          projectId: props.projectId,
+          datasetRunId: runId,
+          page: paginationState.pageIndex,
+          limit: paginationState.pageSize,
+          queryClickhouse,
+        }),
+      })),
+    [
+      props.runIds,
+      props.projectId,
+      paginationState.pageIndex,
+      paginationState.pageSize,
+      queryClickhouse,
+    ],
+  );
+
+  // 2. Track changes using onSuccess callback in the queries instead of useEffect
+  const handleQuerySuccess = useCallback(
+    (runId: string, newData: any) => {
+      setUnchangedCounts((prev) => {
+        const prevCount = prev[runId] || 0;
+        const queryKey = runQueries.find((r) => r.runId === runId)?.queryKey;
+        const prevData = queryClient.getQueryData(queryKey || []);
+
+        // Only increment if we have previous data and it matches the new data
+        if (prevData && JSON.stringify(prevData) === JSON.stringify(newData)) {
+          const newCount = prevCount + 1;
+          return { ...prev, [runId]: newCount };
+        }
+
+        return { ...prev, [runId]: 0 };
+      });
+    },
+    [queryClient, runQueries],
+  );
+
+  // 3. Use the queries with success callback
+  const runs = runQueries.map(({ runId }) => ({
     runId,
     items: api.datasets.runitemsByRunIdOrItemId.useQuery(
       {
@@ -99,8 +159,14 @@ export function DatasetCompareRunsTable(props: {
         refetchOnWindowFocus: false,
         refetchOnMount: false,
         refetchOnReconnect: false,
-        staleTime: 5 * 60 * 1000, // 5 minutes
+        staleTime: 5 * 60 * 1000,
         enabled: baseDatasetItems.isSuccess,
+        refetchInterval: getRefetchInterval(
+          runId,
+          props.localExperiments,
+          unchangedCounts,
+        ),
+        onSuccess: (data) => handleQuerySuccess(runId, data),
       },
     ),
   }));
@@ -139,7 +205,7 @@ export function DatasetCompareRunsTable(props: {
       {},
     );
 
-    return baseDatasetItems.data?.map(
+    return baseDatasetItems.data?.datasetItems.map(
       (item): DatasetCompareRunRowData => ({
         id: item.id,
         input: item.input ?? "null",
@@ -356,7 +422,7 @@ export function DatasetCompareRunsTable(props: {
                 }
         }
         pagination={{
-          totalCount: baseDatasetItems.data?.length ?? null,
+          totalCount: baseDatasetItems.data?.totalCount ?? null,
           onChange: setPaginationState,
           state: paginationState,
         }}

web/src/features/datasets/components/DatasetRunAggregateColumnHelpers.tsx

@@ -17,14 +17,19 @@ export const constructDatasetRunAggregateColumns = ({
   selectedMetrics,
   cellsLoading = false,
 }: {
-  runAggregateColumnProps: { id: string; name: string; description?: string }[];
+  runAggregateColumnProps: {
+    id: string;
+    name: string;
+    description?: string;
+    createdAt?: Date;
+  }[];
   projectId: string;
   scoreKeyToDisplayName: Map<string, string>;
   selectedMetrics: DatasetRunMetric[];
   cellsLoading?: boolean;
 }): LangfuseColumnDef<DatasetCompareRunRowData>[] => {
   return runAggregateColumnProps.map((col) => {
-    const { id, name, description } = col;
+    const { id, name, description, createdAt } = col;
 
     return {
       id,
@@ -40,7 +45,12 @@ export const constructDatasetRunAggregateColumns = ({
       cell: ({ row }: { row: Row<DatasetCompareRunRowData> }) => {
         const runData: RunAggregate = row.getValue("runs") ?? {};
 
-        if (cellsLoading) return <Skeleton className="h-3 w-1/2" />;
+        // if cell is loading or if run created at timestamp is less than 20 seconds ago, show skeleton
+        if (
+          cellsLoading ||
+          (createdAt && createdAt.getTime() + 20000 > Date.now())
+        )
+          return <Skeleton className="h-full min-h-0 w-full" />;
 
         if (!Boolean(Object.keys(runData).length)) return null;
         if (!runData.hasOwnProperty(id)) return null;

web/src/features/datasets/components/DatasetRunItemsTable.tsx

@@ -328,6 +328,7 @@ const TraceObservationIOCell = ({
         },
       },
       refetchOnMount: false, // prevents refetching loops
+      onError: () => {},
     },
   );
   const observation = api.observations.byId.useQuery(
@@ -344,6 +345,7 @@ const TraceObservationIOCell = ({
         },
       },
       refetchOnMount: false, // prevents refetching loops
+      onError: () => {},
     },
   );
 
@@ -351,7 +353,9 @@ const TraceObservationIOCell = ({
 
   return (
     <IOTableCell
-      isLoading={!!!observationId ? trace.isLoading : observation.isLoading}
+      isLoading={
+        (!!!observationId ? trace.isLoading : observation.isLoading) || !data
+      }
       data={io === "output" ? data?.output : data?.input}
       className={cn(io === "output" && "bg-accent-light-green")}
       singleLine={singleLine}

web/src/features/datasets/hooks/useDatasetRunAggregateColumns.ts

@@ -24,6 +24,7 @@ export function useDatasetRunAggregateColumns({
       name: runNameAndMetadata?.name ?? `run${runId}`,
       id: runId,
       description: runNameAndMetadata?.description ?? undefined,
+      createdAt: runNameAndMetadata?.createdAt,
     };
   });
 

web/src/features/datasets/server/dataset-router.ts

@@ -18,20 +18,14 @@ import {
   paginationZod,
 } from "@langfuse/shared";
 import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
-import {
-  getLatencyAndTotalCostForObservations,
-  getLatencyAndTotalCostForObservationsByTraces,
-  getScoresForObservations,
-  getScoresForTraces,
-  traceException,
-} from "@langfuse/shared/src/server";
 import { measureAndReturnApi } from "@/src/server/utils/checkClickhouseAccess";
-import Decimal from "decimal.js";
 import {
   createDatasetRunsTable,
   datasetRunsTableSchema,
   fetchDatasetItems,
+  getRunItemsByRunIdOrItemId,
 } from "@/src/features/datasets/server/service";
+import { traceException } from "@langfuse/shared/src/server";
 
 export const datasetRouter = createTRPCRouter({
   allDatasetMeta: protectedProjectProcedure
@@ -154,7 +148,13 @@ export const datasetRouter = createTRPCRouter({
     .query(async ({ input, ctx }) => {
       return ctx.prisma.datasetRuns.findMany({
         where: { datasetId: input.datasetId, projectId: input.projectId },
-        select: { name: true, id: true, metadata: true, description: true },
+        select: {
+          name: true,
+          id: true,
+          metadata: true,
+          description: true,
+          createdAt: true,
+        },
       });
     }),
   runsByDatasetId: protectedProjectProcedure
@@ -374,7 +374,7 @@ export const datasetRouter = createTRPCRouter({
       }),
     )
     .query(async ({ input, ctx }) => {
-      return ctx.prisma.datasetItem.findMany({
+      const datasetItems = await ctx.prisma.datasetItem.findMany({
         where: { datasetId: input.datasetId, projectId: input.projectId },
         select: {
           id: true,
@@ -382,10 +382,22 @@ export const datasetRouter = createTRPCRouter({
           expectedOutput: true,
           metadata: true,
         },
-        orderBy: { id: "asc" },
+        orderBy: { createdAt: "desc" },
         take: input.limit,
         skip: input.page * input.limit,
       });
+
+      const count = await ctx.prisma.datasetItem.count({
+        where: {
+          datasetId: input.datasetId,
+          projectId: input.projectId,
+        },
+      });
+
+      return {
+        datasetItems,
+        totalCount: count,
+      };
     }),
   updateDatasetItem: protectedProjectProcedure
     .input(
@@ -729,18 +741,50 @@ export const datasetRouter = createTRPCRouter({
         ),
     )
     .query(async ({ input, ctx }) => {
-      const runItems = await ctx.prisma.datasetRunItems.findMany({
-        where: {
-          projectId: input.projectId,
-          datasetRunId: input.datasetRunId,
-          datasetItemId: input.datasetItemId,
-        },
-        orderBy: {
-          datasetItemId: "asc", // Order by dataset item ID instead of createdAt
-        },
-        take: input.limit,
-        skip: input.page * input.limit,
-      });
+      const filterQuery =
+        input.datasetRunId && input.datasetItemId
+          ? Prisma.sql`AND (dri.dataset_run_id = ${input.datasetRunId} OR dri.dataset_item_id = ${input.datasetItemId})`
+          : input.datasetRunId
+            ? Prisma.sql`AND dri.dataset_run_id = ${input.datasetRunId}`
+            : input.datasetItemId
+              ? Prisma.sql`AND dri.dataset_item_id = ${input.datasetItemId}`
+              : Prisma.sql``;
+
+      const runItems = await ctx.prisma.$queryRaw<
+        Array<{
+          id: string;
+          traceId: string;
+          observationId: string | null;
+          createdAt: Date;
+          updatedAt: Date;
+          datasetItemCreatedAt: Date;
+          datasetItemId: string;
+          projectId: string;
+          datasetRunId: string;
+        }>
+      >`
+        SELECT 
+          di.id AS "datasetItemId",
+          di.created_at AS "datasetItemCreatedAt",
+          dri.id,
+          dri.trace_id AS "traceId",
+          dri.observation_id AS "observationId",
+          dri.created_at AS "createdAt",
+          dri.updated_at AS "updatedAt",
+          dri.project_id AS "projectId",
+          dri.dataset_run_id AS "datasetRunId"
+        FROM dataset_run_items dri
+        INNER JOIN dataset_items di
+          ON dri.dataset_item_id = di.id 
+          AND dri.project_id = di.project_id
+        WHERE 
+          dri.project_id = ${input.projectId}
+          ${filterQuery}
+        ORDER BY 
+          di.created_at DESC
+        LIMIT ${input.limit}
+        OFFSET ${input.page * input.limit}
+      `;
 
       if (runItems.length === 0) return { totalRunItems: 0, runItems: [] };
 
@@ -761,19 +805,7 @@ export const datasetRouter = createTRPCRouter({
             where: {
               projectId: ctx.session.projectId,
               traceId: {
-                in: runItems
-                  .filter((ri) => ri.observationId === null) // only include trace scores if run is not linked to an observation
-                  .map((ri) => ri.traceId),
-              },
-            },
-          });
-          const observationScores = await ctx.prisma.score.findMany({
-            where: {
-              projectId: ctx.session.projectId,
-              observationId: {
-                in: runItems
-                  .filter((ri) => ri.observationId !== null)
-                  .map((ri) => ri.observationId) as string[],
+                in: runItems.map((ri) => ri.traceId),
               },
             },
           });
@@ -834,10 +866,6 @@ export const datasetRouter = createTRPCRouter({
             traceScores,
             traceException,
           );
-          const validatedObservationScores = filterAndValidateDbScoreList(
-            observationScores,
-            traceException,
-          );
 
           const items = runItems.map((ri) => {
             return {
@@ -846,16 +874,9 @@ export const datasetRouter = createTRPCRouter({
               datasetItemId: ri.datasetItemId,
               observation: observations.find((o) => o.id === ri.observationId),
               trace: traces.find((t) => t.id === ri.traceId),
-              scores: aggregateScores([
-                ...validatedTraceScores.filter(
-                  (s) => s.traceId === ri.traceId && ri.observationId === null,
-                ),
-                ...validatedObservationScores.filter(
-                  (s) =>
-                    s.observationId === ri.observationId &&
-                    s.traceId === ri.traceId,
-                ),
-              ]),
+              scores: aggregateScores(
+                validatedTraceScores.filter((s) => s.traceId === ri.traceId),
+              ),
             };
           });
 
@@ -866,81 +887,13 @@ export const datasetRouter = createTRPCRouter({
           };
         },
         clickhouseExecution: async () => {
-          const [
-            traceScores,
-            observationScores,
-            observationAggregates,
-            traceAggregate,
-          ] = await Promise.all([
-            getScoresForTraces(
-              input.projectId,
-              runItems
-                .filter((ri) => ri.observationId === null) // only include trace scores if run is not linked to an observation
-                .map((ri) => ri.traceId),
-            ),
-            getScoresForObservations(
-              input.projectId,
-              runItems
-                .filter((ri) => ri.observationId !== null)
-                .map((ri) => ri.observationId) as string[],
-            ),
-            getLatencyAndTotalCostForObservations(
-              input.projectId,
-              runItems
-                .filter((ri) => ri.observationId !== null)
-                .map((ri) => ri.observationId) as string[],
-            ),
-            getLatencyAndTotalCostForObservationsByTraces(
-              input.projectId,
-              runItems.map((ri) => ri.traceId),
-            ),
-          ]);
-
-          const validatedTraceScores = filterAndValidateDbScoreList(
-            traceScores,
-            traceException,
-          );
-          const validatedObservationScores = filterAndValidateDbScoreList(
-            observationScores,
-            traceException,
-          );
-
-          const items = runItems.map((ri) => {
-            return {
-              id: ri.id,
-              createdAt: ri.createdAt,
-              datasetItemId: ri.datasetItemId,
-              observation: observationAggregates
-                .map((o) => ({
-                  id: o.id,
-                  latency: o.latency,
-                  calculatedTotalCost: new Decimal(o.totalCost),
-                }))
-                .find((o) => o.id === ri.observationId),
-              trace: traceAggregate
-                .map((t) => ({
-                  id: t.traceId,
-                  duration: t.latency,
-                  totalCost: t.totalCost,
-                }))
-                .find((t) => t.id === ri.traceId),
-              scores: aggregateScores([
-                ...validatedTraceScores.filter(
-                  (s) => s.traceId === ri.traceId && ri.observationId === null,
-                ),
-                ...validatedObservationScores.filter(
-                  (s) =>
-                    s.observationId === ri.observationId &&
-                    s.traceId === ri.traceId,
-                ),
-              ]),
-            };
-          });
-
           // Note: We early return in case of no run items, when adding parameters here, make sure to update the early return above
           return {
             totalRunItems,
-            runItems: items,
+            runItems: await getRunItemsByRunIdOrItemId(
+              input.projectId,
+              runItems,
+            ),
           };
         },
       });

web/src/features/datasets/server/service.ts

@@ -1,4 +1,10 @@
-import { paginationZod, Prisma, type PrismaClient } from "@langfuse/shared";
+import {
+  filterAndValidateDbScoreList,
+  paginationZod,
+  Prisma,
+  type PrismaClient,
+  type DatasetRunItems,
+} from "@langfuse/shared";
 import { prisma } from "@langfuse/shared/src/db";
 import { v4 } from "uuid";
 import { z } from "zod";
@@ -7,15 +13,20 @@ import {
   clickhouseCompliantRandomCharacters,
   commandClickhouse,
   convertToScore,
-  type FetchScoresReturnType,
+  getLatencyAndTotalCostForObservations,
+  getLatencyAndTotalCostForObservationsByTraces,
   getObservationsById,
+  getScoresForTraces,
   getTracesByIds,
   logger,
   queryClickhouse,
+  type ScoreRecordReadType,
+  traceException,
 } from "@langfuse/shared/src/server";
 import { aggregateScores } from "@/src/features/scores/lib/aggregateScores";
 import Decimal from "decimal.js";
 import { measureAndReturnApi } from "@/src/server/utils/checkClickhouseAccess";
+import { env } from "@/src/env.mjs";
 
 export const datasetRunsTableSchema = z.object({
   projectId: z.string(),
@@ -66,6 +77,7 @@ export const createDatasetRunsTable = async (input: DatasetRunsTableInput) => {
       tableName,
       clickhouseSession,
     );
+
     const obsAgg = await getObservationLatencyAndCostForDataset(
       input,
       tableName,
@@ -140,7 +152,7 @@ export const createTempTableInClickhouse = async (
   clickhouseSession: string,
 ) => {
   const query = `
-      CREATE TABLE IF NOT EXISTS ${tableName}
+      CREATE TABLE IF NOT EXISTS ${tableName} ${env.CLICKHOUSE_CLUSTER_ENABLED === "true" ? "ON CLUSTER default" : ""}
       (
           project_id String,    
           run_id String,  
@@ -148,7 +160,10 @@ export const createTempTableInClickhouse = async (
           dataset_id String,      
           trace_id String,
           observation_id Nullable(String)
-      )  ENGINE = Memory
+      )  
+      ENGINE = ${env.CLICKHOUSE_CLUSTER_ENABLED === "true" ? "ReplicatedMergeTree()" : "MergeTree()"} 
+      PRIMARY KEY (project_id, dataset_id, run_id, trace_id)
+
 
   `;
   await commandClickhouse({
@@ -198,6 +213,7 @@ export const getDatasetRunsFromPostgres = async (
         d.id = ${input.datasetId}
         AND d.project_id = ${input.projectId}
       GROUP BY runs.id, runs.name, runs.description, runs.metadata, runs.created_at, runs.updated_at
+      ORDER BY runs.created_at DESC
       LIMIT ${input.limit}
       OFFSET ${input.page * input.limit}
     `,
@@ -209,24 +225,24 @@ const getScoresFromTempTable = async (
   tableName: string,
   clickhouseSession: string,
 ) => {
+  // adds a setting to read data once it is replicated from the writer node.
+  // Only then, we can guarantee that the created mergetree before was replicated.
   const query = `
       SELECT 
         s.*,
         tmp.run_id
       FROM ${tableName} tmp JOIN scores s 
         ON tmp.project_id = s.project_id 
-        AND tmp.observation_id = s.observation_id 
         AND tmp.trace_id = s.trace_id
       WHERE s.project_id = {projectId: String}
       AND tmp.project_id = {projectId: String}
       AND tmp.dataset_id = {datasetId: String}
       ORDER BY s.event_ts DESC
       LIMIT 1 BY s.id, s.project_id
+      SETTINGS select_sequential_consistency = 1;
   `;
 
-  const rows = await queryClickhouse<
-    FetchScoresReturnType & { run_id: string }
-  >({
+  const rows = await queryClickhouse<ScoreRecordReadType & { run_id: string }>({
     query: query,
     params: {
       projectId: input.projectId,
@@ -480,3 +496,75 @@ export const fetchDatasetItems = async (input: DatasetRunItemsTableInput) => {
     }),
   };
 };
+
+export const getRunItemsByRunIdOrItemId = async (
+  projectId: string,
+  runItems: DatasetRunItems[],
+) => {
+  const [traceScores, observationAggregates, traceAggregate] =
+    await Promise.all([
+      getScoresForTraces(
+        projectId,
+        runItems.map((ri) => ri.traceId),
+      ),
+      getLatencyAndTotalCostForObservations(
+        projectId,
+        runItems
+          .filter((ri) => ri.observationId !== null)
+          .map((ri) => ri.observationId) as string[],
+      ),
+      getLatencyAndTotalCostForObservationsByTraces(
+        projectId,
+        runItems.map((ri) => ri.traceId),
+      ),
+    ]);
+
+  const validatedTraceScores = filterAndValidateDbScoreList(
+    traceScores,
+    traceException,
+  );
+
+  return runItems.map((ri) => {
+    const trace = traceAggregate
+      .map((t) => ({
+        id: t.traceId,
+        duration: t.latency,
+        totalCost: t.totalCost,
+      }))
+      .find((t) => t.id === ri.traceId) ?? {
+      // we default to the traceId provided. The traceId must not be missing.
+      id: ri.traceId,
+      totalCost: 0,
+      duration: 0,
+    };
+
+    const observation =
+      observationAggregates
+        .map((o) => ({
+          id: o.id,
+          latency: o.latency,
+          calculatedTotalCost: new Decimal(o.totalCost),
+        }))
+        .find((o) => o.id === ri.observationId) ??
+      (ri.observationId
+        ? // we default to the observationId provided. The observationId must not be missing
+          // in case it is on the dataset run item.
+          {
+            id: ri.observationId,
+            calculatedTotalCost: new Decimal(0),
+            latency: 0,
+          }
+        : undefined);
+
+    return {
+      id: ri.id,
+      createdAt: ri.createdAt,
+      datasetItemId: ri.datasetItemId,
+      observation,
+      trace,
+      scores: aggregateScores([
+        ...validatedTraceScores.filter((s) => s.traceId === ri.traceId),
+      ]),
+    };
+  });
+};

web/src/features/entitlements/constants/entitlements.ts

@@ -9,6 +9,7 @@ const entitlements = [
   "integration-posthog",
   "batch-export",
   "annotation-queues",
+  "experiments",
 ] as const;
 
 export type Entitlement = (typeof entitlements)[number];
@@ -20,6 +21,7 @@ const cloudAllPlansEntitlements: Entitlement[] = [
   "integration-posthog",
   "batch-export",
   "annotation-queues",
+  "experiments",
 ];
 
 export const entitlementAccess: Record<Plan, Entitlement[]> = {

web/src/features/ingest/usage.ts

@@ -234,7 +234,7 @@ const getTokensByModel = (model: TiktokenModel, text: string) => {
     encoding = getEncoding("cl100k_base");
   }
   const cleandedText = unicodeToBytesInString(text);
-  return encoding?.encode(cleandedText).length;
+  return encoding?.encode(cleandedText, "all").length;
 };
 
 interface Tokenizer {

web/src/features/media/server/getMediaStorageClient.ts

@@ -1,13 +1,16 @@
 import { env } from "@/src/env.mjs";
-import { S3StorageService } from "@langfuse/shared/src/server";
+import {
+  type StorageService,
+  StorageServiceFactory,
+} from "@langfuse/shared/src/server";
 
-let s3StorageServiceClient: S3StorageService;
+let s3StorageServiceClient: StorageService;
 
 export const getMediaStorageServiceClient = (
   bucketName: string,
-): S3StorageService => {
+): StorageService => {
   if (!s3StorageServiceClient) {
-    s3StorageServiceClient = new S3StorageService({
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
       bucketName,
       accessKeyId: env.LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID,
       secretAccessKey: env.LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY,

web/src/features/posthog-analytics/usePostHogClientCapture.ts

@@ -59,7 +59,13 @@ const events = {
     "version_delete_submit",
   ],
   session_detail: ["publish_button_click"],
-  eval_config: ["delete", "new_form_submit", "new_form_open"],
+  eval_config: [
+    "new_form_submit",
+    "new_form_open",
+    "activate",
+    "deactivate",
+    "update",
+  ],
   eval_templates: [
     "view_version",
     "new_form_open",

web/src/features/prompts/components/NewPromptForm/PromptChatMessages.tsx

@@ -7,12 +7,10 @@ import {
   ChatMessageRole,
   ChatMessageDefaultRoleSchema,
   type ChatMessageWithId,
+  ChatMessageListSchema,
 } from "@langfuse/shared";
 
-import {
-  ChatMessageListSchema,
-  type NewPromptFormSchemaType,
-} from "./validation";
+import { type NewPromptFormSchemaType } from "./validation";
 
 import type { ControllerRenderProps } from "react-hook-form";
 import type { MessagesContext } from "@/src/components/ChatMessages/types";

web/src/features/prompts/components/NewPromptForm/index.tsx

@@ -26,15 +26,18 @@ import {
 } from "@/src/features/prompts/server/utils/validation";
 import useProjectIdFromURL from "@/src/hooks/useProjectIdFromURL";
 import { api } from "@/src/utils/api";
-import { extractVariables, getIsCharOrUnderscore } from "@/src/utils/string";
 import { zodResolver } from "@hookform/resolvers/zod";
-import type { Prompt } from "@langfuse/shared";
+import {
+  type Prompt,
+  extractVariables,
+  getIsCharOrUnderscore,
+} from "@langfuse/shared";
 import { PromptChatMessages } from "./PromptChatMessages";
 import {
   NewPromptFormSchema,
   type NewPromptFormSchemaType,
-  PromptContentSchema,
-  type PromptContentType,
+  PromptVariantSchema,
+  type PromptVariant,
 } from "./validation";
 import { Input } from "@/src/components/ui/input";
 import Link from "next/link";
@@ -62,25 +65,25 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
   const utils = api.useUtils();
   const capture = usePostHogClientCapture();
 
-  let initialPromptContent: PromptContentType | null;
+  let initialPromptVariant: PromptVariant | null;
   try {
-    initialPromptContent = PromptContentSchema.parse({
+    initialPromptVariant = PromptVariantSchema.parse({
       type: initialPrompt?.type,
       prompt: initialPrompt?.prompt?.valueOf(),
     });
   } catch (err) {
-    initialPromptContent = null;
+    initialPromptVariant = null;
   }
 
   const defaultValues: NewPromptFormSchemaType = {
-    type: initialPromptContent?.type ?? PromptType.Text,
+    type: initialPromptVariant?.type ?? PromptType.Text,
     chatPrompt:
-      initialPromptContent?.type === PromptType.Chat
-        ? initialPromptContent?.prompt
+      initialPromptVariant?.type === PromptType.Chat
+        ? initialPromptVariant?.prompt
         : [],
     textPrompt:
-      initialPromptContent?.type === PromptType.Text
-        ? initialPromptContent?.prompt
+      initialPromptVariant?.type === PromptType.Text
+        ? initialPromptVariant?.prompt
         : "",
     name: initialPrompt?.name ?? "",
     config: JSON.stringify(initialPrompt?.config?.valueOf(), null, 2) || "{}",
@@ -258,8 +261,8 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
                 <TabsList className="flex w-full">
                   <TabsTrigger
                     disabled={
-                      Boolean(initialPromptContent) &&
-                      initialPromptContent?.type !== PromptType.Text
+                      Boolean(initialPromptVariant) &&
+                      initialPromptVariant?.type !== PromptType.Text
                     }
                     className="flex-1"
                     value={PromptType.Text}
@@ -268,8 +271,8 @@ export const NewPromptForm: React.FC<NewPromptFormProps> = (props) => {
                   </TabsTrigger>
                   <TabsTrigger
                     disabled={
-                      Boolean(initialPromptContent) &&
-                      initialPromptContent?.type !== PromptType.Chat
+                      Boolean(initialPromptVariant) &&
+                      initialPromptVariant?.type !== PromptType.Chat
                     }
                     className="flex-1"
                     value={PromptType.Chat}

web/src/features/prompts/components/NewPromptForm/validation.ts

@@ -1,14 +1,6 @@
 import { z } from "zod";
 import { PromptType } from "@/src/features/prompts/server/utils/validation";
-import { ChatMessageDefaultRoleSchema } from "@langfuse/shared";
-
-const ChatMessageSchema = z.object({
-  role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
-  content: z.string(),
-});
-
-export const ChatMessageListSchema = z.array(ChatMessageSchema);
-export const TextPromptSchema = z.string().min(1, "Enter a prompt");
+import { ChatMessageListSchema, TextPromptSchema } from "@langfuse/shared";
 
 const NewPromptBaseSchema = z.object({
   name: z.string().min(1, "Enter a name"),
@@ -39,7 +31,7 @@ export const NewPromptFormSchema = z.union([
 ]);
 export type NewPromptFormSchemaType = z.infer<typeof NewPromptFormSchema>;
 
-export const PromptContentSchema = z.union([
+export const PromptVariantSchema = z.union([
   z.object({
     type: z.literal(PromptType.Chat),
     prompt: ChatMessageListSchema,
@@ -49,7 +41,7 @@ export const PromptContentSchema = z.union([
     prompt: z.string(),
   }),
 ]);
-export type PromptContentType = z.infer<typeof PromptContentSchema>;
+export type PromptVariant = z.infer<typeof PromptVariantSchema>;
 
 function validateJson(content: string): boolean {
   try {