chore: release v3.117.2

* add obs name as mapping indicator

* fix mapper oai

* lint

* simplify

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx,notIn,alue
+ignore-words-list = afterall,vertx,notIn,alue,allTime
 

.devcontainer/Dockerfile

@@ -2,7 +2,7 @@
 FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:20-bookworm
 
 # Install golang-migrate for database migrations
-RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && \
+RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz && \
     chmod +x migrate && \
     mv migrate /usr/local/bin/migrate
 

.env.dev-redis-cluster.example

@@ -85,4 +85,3 @@ ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 
-LANGFUSE_EXPERIMENT_USE_OTEL_INGESTION_QUEUE="true"

.env.dev.example

@@ -33,6 +33,7 @@ SALT="salt"
 # Email
 EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
 SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
+CLOUD_CRM_EMAIL=""  # Optional BCC address for usage threshold emails (e.g., for CRM integration like HubSpot)
 
 # S3 Batch Exports
 LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
@@ -87,4 +88,16 @@ NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 # Slack credentials for development
 SLACK_CLIENT_ID=your_slack_client_id
 SLACK_CLIENT_SECRET=your_slack_client_secret
-SLACK_STATE_SECRET=your_slack_state_secret
+SLACK_STATE_SECRET=your_slack_state_secret
+
+# Langfuse AI instance for tracing, prompts
+LANGFUSE_AI_FEATURES_PUBLIC_KEY="pk-lf-1234567890"
+LANGFUSE_AI_FEATURES_SECRET_KEY="sk-lf-1234567890"
+LANGFUSE_AI_FEATURES_HOST="http://localhost:3000"
+LANGFUSE_AI_FEATURES_PROJECT_ID=7a88fb47-b4e2-43b8-a06c-a5ce950dc53a
+
+# Langfuse AI Bedrock credentials
+AWS_ACCESS_KEY_ID="A123456789"
+AWS_SECRET_ACCESS_KEY="SAK123456789"
+LANGFUSE_AWS_BEDROCK_REGION="eu-west-1"
+LANGFUSE_AWS_BEDROCK_MODEL="eu.anthropic.claude-3-haiku-20240307-v1:0"

.env.prod.example

@@ -270,6 +270,14 @@ LANGFUSE_ENABLE_BLOB_STORAGE_FILE_LOG=true
 # Rate limiting
 # LANGFUSE_RATE_LIMITS_ENABLED=
 
+# Free tier usage thresholds (Cloud deployments only)
+# Enable the queue consumer that monitors free tier usage (default: true, but requires cloud region)
+# QUEUE_CONSUMER_FREE_TIER_USAGE_THRESHOLD_QUEUE_IS_ENABLED=true
+# Enable enforcement: send emails and block orgs that exceed free tier limits (default: false)
+# LANGFUSE_FREE_TIER_USAGE_THRESHOLD_ENFORCEMENT_ENABLED=false
+# Optional BCC address for usage threshold emails (e.g., for CRM integration like HubSpot)
+# CLOUD_CRM_EMAIL=
+
 # Stripe
 # STRIPE_SECRET_KEY=
 # STRIPE_WEBHOOK_SIGNING_SECRET=

.github/workflows/pipeline.yml

@@ -148,7 +148,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -225,7 +225,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -320,7 +320,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -349,7 +349,80 @@ jobs:
       - name: Build
         run: pnpm --filter=worker... run build
       - name: run tests
-        run: pnpm --filter=worker run test
+        run: pnpm --filter=worker run test:exclude-llm-connections
+
+  test-worker-llm-connections:
+    timeout-minutes: 20
+    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
+    name: test-worker-llm-connections (node24, pg15)
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - name: Login to Docker Hub
+        if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+      - name: Use Node.js 24
+        uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm install
+      - name: Install golang-migrate for Clickhouse migrations
+        run: |
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
+          sudo mv migrate /usr/bin/migrate
+          which migrate
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+          cp .env.dev.example web/.env
+          cp .env.dev.example worker/.env
+      - name: Run + migrate
+        run: |
+          docker compose -f docker-compose.dev.yml up -d
+          sleep 5 # Wait for PostgreSQL to accept connections
+          docker compose ps
+        env:
+          POSTGRES_VERSION: 15
+      - name: Ensure no unhealthy status
+        run: |
+          if docker compose ps | grep "(unhealthy)"; then
+            echo "One or more services are unhealthy"
+            exit 1
+          else
+            echo "All services are healthy"
+          fi
+      - name: Seed DB
+        run: |
+          pnpm run db:migrate
+          pnpm run db:seed
+          pnpm run --filter=shared ch:up
+      - name: Build
+        run: pnpm --filter=worker... run build
+      - name: run llm connection tests
+        run: pnpm --filter=worker run test:llm-connections-only
+        env:
+          LANGFUSE_LLM_CONNECTION_OPENAI_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_OPENAI_KEY }}
+          LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_ANTHROPIC_KEY }}
+          LANGFUSE_LLM_CONNECTION_AZURE_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_AZURE_KEY }}
+          LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL: ${{ secrets.LANGFUSE_LLM_CONNECTION_AZURE_BASE_URL }}
+          LANGFUSE_LLM_CONNECTION_AZURE_MODEL: ${{ secrets.LANGFUSE_LLM_CONNECTION_AZURE_MODEL }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_ACCESS_KEY_ID }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_SECRET_ACCESS_KEY }}
+          LANGFUSE_LLM_CONNECTION_BEDROCK_REGION: ${{ secrets.LANGFUSE_LLM_CONNECTION_BEDROCK_REGION }}
+          LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_VERTEXAI_KEY }}
+          LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY: ${{ secrets.LANGFUSE_LLM_CONNECTION_GOOGLEAISTUDIO_KEY }}
 
   e2e-tests:
     runs-on: ubuntu-latest
@@ -381,7 +454,7 @@ jobs:
           cp .env.dev.example web/.env
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Run + migrate
@@ -437,7 +510,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.19.0/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -485,6 +558,7 @@ jobs:
         prettier-check,
         tests-web-sync,
         tests-worker,
+        test-worker-llm-connections,
         e2e-tests,
         test-docker-build,
         e2e-server-tests,

.github/workflows/sdk-api-spec.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'fern/**'
   workflow_dispatch:
 
 concurrency:
@@ -55,6 +57,10 @@ jobs:
           # Copy generated Python SDK files
           cp -r ../langfuse/generated/python/* langfuse/api/
 
+          # Remove unnecessary integration tests created by Fern
+          # (could not find the right option to prevent this in generation step)
+          rm -rf langfuse/api/tests
+
           # Install poetry and format
           pip install poetry
           poetry install --all-extras
@@ -69,15 +75,20 @@ jobs:
           # Close existing api-spec-bot PRs
           gh pr list --author langfuse-bot --state open --json number --jq '.[].number' | xargs -I {} gh pr close {}
 
+          # Get the GitHub username of the original commit author
+          cd ../langfuse
+          ORIGINAL_AUTHOR=$(gh api repos/langfuse/langfuse/commits/${GITHUB_SHA} --jq '.author.login')
+          cd ../langfuse-python
+
           # Create new branch and push changes
-          BRANCH_NAME="api-spec-bot-${{ github.sha }}"
+          BRANCH_NAME="api-spec-bot-${GITHUB_SHA::7}"
           git checkout -b "$BRANCH_NAME"
           git add .
-          git commit -m "feat(api): update API spec from langfuse/langfuse ${{ github.sha }}"
+          git commit -m "feat(api): update API spec from langfuse/langfuse ${GITHUB_SHA::7}"
           git push origin "$BRANCH_NAME"
 
-          # Create PR
-          gh pr create --title "feat(api): update API spec from langfuse/langfuse ${{ github.sha }}" --body ""
+          # Create PR with original author as reviewer
+          gh pr create --title "feat(api): update API spec from langfuse/langfuse ${GITHUB_SHA::7}" --body "" --reviewer "$ORIGINAL_AUTHOR"
 
       - name: Update TypeScript SDK
         env:
@@ -99,6 +110,15 @@ jobs:
           # Copy generated TypeScript SDK files
           cp -r ../langfuse/generated/typescript/* packages/core/src/api/
 
+          # Patch compilation error inducing output
+          # Without this patch, the JS SDK will not build due to
+          # "Error: packages/core build: src/api/core/auth/index.ts(1,10): error TS1205: Re-exporting a type when 'isolatedModules' is enabled requires using 'export type'."
+          if [ -f "packages/core/src/api/core/auth/index.ts" ]; then
+            if grep -q '^export { AuthProvider } from "./AuthProvider.js";$' packages/core/src/api/core/auth/index.ts; then
+              sed -i '1s/^export { AuthProvider }/export { type AuthProvider }/' packages/core/src/api/core/auth/index.ts
+            fi
+          fi
+
           # Install dependencies and format
           npm install -g pnpm
           pnpm install
@@ -113,12 +133,17 @@ jobs:
           # Close existing api-spec-bot PRs
           gh pr list --author langfuse-bot --state open --json number --jq '.[].number' | xargs -I {} gh pr close {}
 
+          # Get the GitHub username of the original commit author
+          cd ../langfuse
+          ORIGINAL_AUTHOR=$(gh api repos/langfuse/langfuse/commits/${GITHUB_SHA} --jq '.author.login')
+          cd ../langfuse-js
+
           # Create new branch and push changes
-          BRANCH_NAME="api-spec-bot-${{ github.sha }}"
+          BRANCH_NAME="api-spec-bot-${GITHUB_SHA::7}"
           git checkout -b "$BRANCH_NAME"
           git add .
-          git commit -m "feat(api): update API spec from langfuse/langfuse ${{ github.sha }}" --no-verify
+          git commit -m "feat(api): update API spec from langfuse/langfuse ${GITHUB_SHA::7}" --no-verify
           git push origin "$BRANCH_NAME"
 
-          # Create PR
-          gh pr create --title "feat(api): update API spec from langfuse/langfuse ${{ github.sha }}" --body ""
+          # Create PR with original author as reviewer
+          gh pr create --title "feat(api): update API spec from langfuse/langfuse ${GITHUB_SHA::7}" --body "" --reviewer "$ORIGINAL_AUTHOR"

CONTRIBUTING.md

@@ -111,12 +111,16 @@ Requirements
 - Node.js 24 as specified in the [.nvmrc](.nvmrc)
 - Pnpm v.9.5.0
 - Docker to run the database locally
+- Clickhouse client
 
 **Note:** You can also simply run Langfuse in a **GitHub Codespace** via the provided devcontainer. To do this, click on the green "Code" button in the top right corner of the repository and select "Open with Codespaces".
 
 **Steps**
 
-1. Install [golang-migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate#migrate-cli) as CLI
+1. Install development dependencies:
+	- [golang-migrate](https://github.com/golang-migrate/migrate/tree/master/cmd/migrate#migrate-cli) as CLI
+	- [clickhouse binary](https://clickhouse.com/docs/install) on macOS with brew: `brew install --cask clickhouse`
+
 2. Fork the repository and clone it locally
 
    ```bash

REVIEW.md

@@ -22,3 +22,7 @@
 
 - Highlight usage of `redis.call` invocations. Those may have suboptimal redis cluster routing and will raise errors. Instead, use the native call patterns.
   Example: `await redis?.call("SET", key, "1", "NX", "EX", TTLSeconds);` should use `await redis?.set(key, "1", "EX", TTLSeconds, "NX");` instead.
+
+## Langfuse Cloud
+
+- When attempting to confirm if the current environment is Langfuse Cloud in the frontend, use the `useLangfuseCloudRegion` hook and never environment variables directly.

docker-compose.build.yml

@@ -17,11 +17,11 @@ services:
     ports:
       - "3000:3000"
     environment: &langfuse-web-env
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
-      NEXTAUTH_SECRET: mysecret
-      SALT: mysalt
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # generate via `openssl rand -hex 32`
-      NEXTAUTH_URL: http://localhost:3000
+      DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres}
+      NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret}
+      SALT: ${SALT:-mysalt}
+      ENCRYPTION_KEY: ${ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000}
       TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
       LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-false}
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
@@ -82,8 +82,8 @@ services:
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
@@ -103,8 +103,8 @@ services:
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
     environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret}
     ports:
       - "9090:9000"
       - "9091:9001"
@@ -131,7 +131,7 @@ services:
       retries: 10
 
   postgres:
-    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-17}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -139,9 +139,11 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_DB: postgres
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      POSTGRES_DB: ${POSTGRES_DB:-postgres}
+      TZ: UTC
+      PGTZ: UTC
     ports:
       - 5432:5432
     volumes:

docker-compose.dev-azure.yml

@@ -4,8 +4,8 @@ services:
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
@@ -32,7 +32,7 @@ services:
       - 6379:6379
 
   postgres:
-    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-17}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -41,9 +41,11 @@ services:
       retries: 10
     command: ["postgres", "-c", "log_statement=all"]
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+      - TZ=UTC
+      - PGTZ=UTC
     ports:
       - 5432:5432
     volumes:

docker-compose.dev-redis-cluster.yml

@@ -4,8 +4,8 @@ services:
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
@@ -21,8 +21,8 @@ services:
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
     environment:
-      MINIO_ACCESS_KEY: minio
-      MINIO_SECRET_KEY: miniosecret
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret}
     ports:
       - 127.0.0.1:9090:9000
       - 127.0.0.1:9091:9001
@@ -36,7 +36,7 @@ services:
       start_period: 1s
 
   postgres:
-    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-17}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -45,9 +45,11 @@ services:
       retries: 10
     command: ["postgres", "-c", "log_statement=all"]
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+      - TZ=UTC
+      - PGTZ=UTC
     ports:
       - 127.0.0.1:5432:5432
     volumes:

docker-compose.dev.yml

@@ -1,11 +1,13 @@
 services:
   clickhouse:
-    image: docker.io/clickhouse/clickhouse-server:24.3
+    # Upgrade from 24.3 to check behaviour on new events table.
+    # Unit tests still verify 24.3 behaviour using -azure and -redis-cluster configs.
+    image: docker.io/clickhouse/clickhouse-server:25.8
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse}
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
@@ -21,8 +23,8 @@ services:
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
     environment:
-      MINIO_ACCESS_KEY: minio
-      MINIO_SECRET_KEY: miniosecret
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret}
     ports:
       - 127.0.0.1:9090:9000
       - 127.0.0.1:9091:9001
@@ -44,7 +46,7 @@ services:
       - 127.0.0.1:6379:6379
 
   postgres:
-    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-17}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -53,9 +55,11 @@ services:
       retries: 10
     command: ["postgres", "-c", "log_statement=all"]
     environment:
-      - POSTGRES_USER=postgres
-      - POSTGRES_PASSWORD=postgres
-      - POSTGRES_DB=postgres
+      - POSTGRES_USER=${POSTGRES_USER:-postgres}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+      - TZ=UTC
+      - PGTZ=UTC
     ports:
       - 127.0.0.1:5432:5432
     volumes:

docker-compose.yml

@@ -19,10 +19,10 @@ services:
     ports:
       - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
-      DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
-      SALT: "mysalt" # CHANGEME
-      ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
+      NEXTAUTH_URL: ${NEXTAUTH_URL:-http://localhost:3000}
+      DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/postgres} # CHANGEME
+      SALT: ${SALT:-mysalt} # CHANGEME
+      ENCRYPTION_KEY: ${ENCRYPTION_KEY:-0000000000000000000000000000000000000000000000000000000000000000} # CHANGEME: generate via `openssl rand -hex 32`
       TELEMETRY_ENABLED: ${TELEMETRY_ENABLED:-true}
       LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: ${LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES:-true}
       CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
@@ -74,7 +74,7 @@ services:
       - 3000:3000
     environment:
       <<: *langfuse-worker-env
-      NEXTAUTH_SECRET: mysecret # CHANGEME
+      NEXTAUTH_SECRET: ${NEXTAUTH_SECRET:-mysecret} # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}
       LANGFUSE_INIT_PROJECT_ID: ${LANGFUSE_INIT_PROJECT_ID:-}
@@ -91,8 +91,8 @@ services:
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
-      CLICKHOUSE_USER: clickhouse
-      CLICKHOUSE_PASSWORD: clickhouse # CHANGEME
+      CLICKHOUSE_USER: ${CLICKHOUSE_USER:-clickhouse}
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-clickhouse} # CHANGEME
     volumes:
       - langfuse_clickhouse_data:/var/lib/clickhouse
       - langfuse_clickhouse_logs:/var/log/clickhouse-server
@@ -113,8 +113,8 @@ services:
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
     environment:
-      MINIO_ROOT_USER: minio
-      MINIO_ROOT_PASSWORD: miniosecret # CHANGEME
+      MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minio}
+      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-miniosecret} # CHANGEME
     ports:
       - 9090:9000
       - 127.0.0.1:9091:9001
@@ -142,7 +142,7 @@ services:
       retries: 10
 
   postgres:
-    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-17}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -150,9 +150,11 @@ services:
       timeout: 3s
       retries: 10
     environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres # CHANGEME
-      POSTGRES_DB: postgres
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} # CHANGEME
+      POSTGRES_DB: ${POSTGRES_DB:-postgres}
+      TZ: UTC
+      PGTZ: UTC
     ports:
       - 127.0.0.1:5432:5432
     volumes:

ee/package.json

@@ -27,7 +27,7 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "https-proxy-agent": "^7.0.6",
-    "next": "15.5.2",
+    "next": "15.5.4",
     "next-auth": "^4.24.11",
     "zod": "^3.25.62"
   },

fern/apis/server/definition/score-configs.yml

@@ -35,6 +35,18 @@ service:
           type: string
           docs: The unique langfuse identifier of a score config
       response: commons.ScoreConfig
+
+    update:
+      docs: Update a score config
+      method: PATCH
+      path: /score-configs/{configId}
+      path-parameters:
+        configId:
+          type: string
+          docs: The unique langfuse identifier of a score config
+      request: UpdateScoreConfigRequest
+      response: commons.ScoreConfig
+
 types:
   ScoreConfigs:
     properties:
@@ -56,3 +68,23 @@ types:
       description:
         type: optional<string>
         docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage
+  UpdateScoreConfigRequest:
+    properties:
+      isArchived:
+        type: optional<boolean>
+        docs: The status of the score config showing if it is archived or not
+      name:
+        type: optional<string>
+        docs: The name of the score config
+      categories:
+        type: optional<list<commons.ConfigCategory>>
+        docs: Configure custom categories for categorical scores. Pass a list of objects with `label` and `value` properties. Categories are autogenerated for boolean configs and cannot be passed
+      minValue:
+        type: optional<double>
+        docs: Configure a minimum value for numerical scores. If not set, the minimum value defaults to -∞
+      maxValue:
+        type: optional<double>
+        docs: Configure a maximum value for numerical scores. If not set, the maximum value defaults to +∞
+      description:
+        type: optional<string>
+        docs: Description is shown across the Langfuse UI and can be used to e.g. explain the config categories in detail, why a numeric range was set, or provide additional context on config name or usage

fern/apis/server/definition/trace.yml

@@ -66,6 +66,32 @@ service:
           fields:
             type: optional<string>
             docs: "Comma-separated list of fields to include in the response. Available field groups: 'core' (always included), 'io' (input, output, metadata), 'scores', 'observations', 'metrics'. If not specified, all fields are returned. Example: 'core,scores,metrics'. Note: Excluded 'observations' or 'scores' fields return empty arrays; excluded 'metrics' returns -1 for 'totalCost' and 'latency'."
+          filter:
+            type: optional<string>
+            docs: |
+              JSON string containing an array of filter conditions. When provided, this takes precedence over legacy filter parameters (userId, name, sessionId, tags, version, release, environment, fromTimestamp, toTimestamp).
+              Each filter condition has the following structure:
+              ```json
+              [
+                {
+                  "type": string,           // Required. One of: "datetime", "string", "number", "stringOptions", "categoryOptions", "arrayOptions", "stringObject", "numberObject", "boolean", "null"
+                  "column": string,         // Required. Column to filter on
+                  "operator": string,       // Required. Operator based on type:
+                                            // - datetime: ">", "<", ">=", "<="
+                                            // - string: "=", "contains", "does not contain", "starts with", "ends with"
+                                            // - stringOptions: "any of", "none of"
+                                            // - categoryOptions: "any of", "none of"
+                                            // - arrayOptions: "any of", "none of", "all of"
+                                            // - number: "=", ">", "<", ">=", "<="
+                                            // - stringObject: "=", "contains", "does not contain", "starts with", "ends with"
+                                            // - numberObject: "=", ">", "<", ">=", "<="
+                                            // - boolean: "=", "<>"
+                                            // - null: "is null", "is not null"
+                  "value": any,             // Required (except for null type). Value to compare against. Type depends on filter type
+                  "key": string             // Required only for stringObject, numberObject, and categoryOptions types when filtering on nested fields like metadata
+                }
+              ]
+              ```
       response: Traces
     deleteMultiple:
       docs: Delete multiple traces

fern/apis/server/generators.yml

@@ -31,7 +31,7 @@ groups:
       #     client-class-name: LangfuseClient
 
       - name: fernapi/fern-typescript-node-sdk
-        version: 2.6.1
+        version: 2.12.3
         output:
           location: local-file-system
           path: ../../../generated/typescript

fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "langfuse",
-  "version": "0.56.0"
-}
+  "version": "0.77.5"
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.112.0",
+  "version": "3.117.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -40,7 +40,7 @@
     "husky": "^9.1.7",
     "prettier": "^3.6.2",
     "release-it": "^19.0.4",
-    "turbo": "^2.5.6"
+    "turbo": "^2.5.8"
   },
   "release-it": {
     "git": {

packages/config-eslint/library.js

@@ -2,10 +2,15 @@ const { resolve } = require("node:path");
 
 const project = resolve(process.cwd(), "tsconfig.json");
 
+// Handle eslint-config-turbo's default export
+const turboConfig = require("eslint-config-turbo");
+const turboConfigToUse = turboConfig.default || turboConfig;
+
 /** @type {import("eslint").Linter.Config} */
 module.exports = {
-  extends: ["eslint:recommended", "prettier", "eslint-config-turbo"],
-  plugins: ["only-warn"],
+  // extends: ["eslint:recommended", "prettier"require().default],
+  extends: ["eslint:recommended", "prettier"],
+  plugins: ["only-warn", "turbo"],
   globals: {
     React: true,
     JSX: true,
@@ -30,6 +35,7 @@ module.exports = {
   rules: {
     "no-redeclare": "off",
     "import/order": "off",
+    ...(turboConfigToUse.rules || {}),
   },
   overrides: [
     {
@@ -51,5 +57,6 @@ module.exports = {
         ],
       },
     },
+    ...(turboConfigToUse.overrides || []),
   ],
 };

packages/config-eslint/next.js

@@ -2,6 +2,9 @@ const { resolve } = require("node:path");
 
 const project = resolve(process.cwd(), "tsconfig.json");
 
+const turboConfig = require("eslint-config-turbo");
+const turboConfigToUse = turboConfig.default || turboConfig;
+
 /*
  * This is a custom ESLint configuration for use with
  * Next.js apps.
@@ -16,7 +19,6 @@ module.exports = {
   extends: [
     "plugin:@typescript-eslint/recommended",
     "plugin:@typescript-eslint/strict-type-checked",
-    "eslint-config-turbo",
   ],
   rules: {
     "@typescript-eslint/no-non-null-assertion": "off",
@@ -47,6 +49,7 @@ module.exports = {
   ignorePatterns: ["node_modules/", "dist/"],
   // add rules configurations here
   rules: {
+    ...(turboConfigToUse.rules || {}),
     "@typescript-eslint/consistent-type-imports": [
       "warn",
       {

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.6",
+    "eslint-config-turbo": "^2.5.8",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.7.2"
   }

packages/shared/clickhouse/migrations/clustered/0027_drop_project_environments_mvs.down.sql

@@ -0,0 +1,21 @@
+-- Recreate materialized views for project_environments table
+CREATE MATERIALIZED VIEW project_environments_traces_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM traces
+GROUP BY project_id;
+
+CREATE MATERIALIZED VIEW project_environments_observations_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM observations
+GROUP BY project_id;
+
+CREATE MATERIALIZED VIEW project_environments_scores_mv ON CLUSTER default TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM scores
+GROUP BY project_id;

packages/shared/clickhouse/migrations/clustered/0027_drop_project_environments_mvs.up.sql

@@ -0,0 +1,4 @@
+-- Drop materialized views feeding project_environments table
+DROP VIEW IF EXISTS project_environments_traces_mv ON CLUSTER default;
+DROP VIEW IF EXISTS project_environments_observations_mv ON CLUSTER default;
+DROP VIEW IF EXISTS project_environments_scores_mv ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0028_drop_traces_null_mvs.down.sql

@@ -0,0 +1,114 @@
+-- Recreate materialized views derived from traces_null table
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv ON CLUSTER default TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv ON CLUSTER default TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv ON CLUSTER default TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/migrations/clustered/0028_drop_traces_null_mvs.up.sql

@@ -0,0 +1,4 @@
+-- Drop materialized views derived from traces_null table
+DROP VIEW IF EXISTS traces_all_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_7d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_30d_amt_mv ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0029_drop_traces_null_and_amt_tables.down.sql

@@ -0,0 +1,179 @@
+-- Recreate traces_null and trace amt tables
+CREATE TABLE traces_null ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+CREATE TABLE traces_all_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+CREATE TABLE traces_7d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+CREATE TABLE traces_30d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;

packages/shared/clickhouse/migrations/clustered/0029_drop_traces_null_and_amt_tables.up.sql

@@ -0,0 +1,5 @@
+-- Drop traces_null and trace amt tables
+DROP TABLE IF EXISTS traces_null ON CLUSTER default;
+DROP TABLE IF EXISTS traces_all_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_7d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_30d_amt ON CLUSTER default;

packages/shared/clickhouse/migrations/unclustered/0027_drop_project_environments_mvs.down.sql

@@ -0,0 +1,21 @@
+-- Recreate materialized views for project_environments table
+CREATE MATERIALIZED VIEW project_environments_traces_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM traces
+GROUP BY project_id;
+
+CREATE MATERIALIZED VIEW project_environments_observations_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM observations
+GROUP BY project_id;
+
+CREATE MATERIALIZED VIEW project_environments_scores_mv TO project_environments AS
+SELECT
+    project_id,
+    groupUniqArray(environment) AS environments
+FROM scores
+GROUP BY project_id;

packages/shared/clickhouse/migrations/unclustered/0027_drop_project_environments_mvs.up.sql

@@ -0,0 +1,4 @@
+-- Drop materialized views feeding project_environments table
+DROP VIEW IF EXISTS project_environments_traces_mv;
+DROP VIEW IF EXISTS project_environments_observations_mv;
+DROP VIEW IF EXISTS project_environments_scores_mv;

packages/shared/clickhouse/migrations/unclustered/0028_drop_traces_null_mvs.down.sql

@@ -0,0 +1,114 @@
+-- Recreate materialized views derived from traces_null table
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/migrations/unclustered/0028_drop_traces_null_mvs.up.sql

@@ -0,0 +1,4 @@
+-- Drop materialized views derived from traces_null table
+DROP VIEW IF EXISTS traces_all_amt_mv;
+DROP VIEW IF EXISTS traces_7d_amt_mv;
+DROP VIEW IF EXISTS traces_30d_amt_mv;

packages/shared/clickhouse/migrations/unclustered/0029_drop_traces_null_and_amt_tables.down.sql

@@ -0,0 +1,179 @@
+-- Recreate traces_null and trace amt tables
+CREATE TABLE traces_null
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+CREATE TABLE traces_all_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+CREATE TABLE traces_7d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+CREATE TABLE traces_30d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;

packages/shared/clickhouse/migrations/unclustered/0029_drop_traces_null_and_amt_tables.up.sql

@@ -0,0 +1,5 @@
+-- Drop traces_null and trace amt tables
+DROP TABLE IF EXISTS traces_null;
+DROP TABLE IF EXISTS traces_all_amt;
+DROP TABLE IF EXISTS traces_7d_amt;
+DROP TABLE IF EXISTS traces_30d_amt;

packages/shared/clickhouse/scripts/dev-tables.sh

@@ -0,0 +1,261 @@
+#!/bin/bash
+
+# Development-only ClickHouse table creation script
+# This script is for creating experimental/development tables that are not yet
+# ready to be part of the official migration system.
+#
+# Usage:
+#   pnpm run ch:dev-tables  (from packages/shared/)
+#
+# This script is automatically run as part of:
+#   - pnpm run dx
+#   - pnpm run dx-f
+#   - pnpm run ch:reset
+
+# Load environment variables
+[ -f ../../.env ] && source ../../.env
+
+# Check if CLICKHOUSE_MIGRATION_URL is configured
+if [ -z "${CLICKHOUSE_MIGRATION_URL}" ]; then
+  echo "Error: CLICKHOUSE_MIGRATION_URL is not configured."
+  echo "Please set CLICKHOUSE_MIGRATION_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_USER is set
+if [ -z "${CLICKHOUSE_USER}" ]; then
+  echo "Error: CLICKHOUSE_USER is not set."
+  echo "Please set CLICKHOUSE_USER in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_PASSWORD is set
+if [ -z "${CLICKHOUSE_PASSWORD}" ]; then
+  echo "Error: CLICKHOUSE_PASSWORD is not set."
+  echo "Please set CLICKHOUSE_PASSWORD in your environment variables."
+  exit 1
+fi
+
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
+# Parse the CLICKHOUSE_MIGRATION_URL to extract host and port
+# Expected format: clickhouse://localhost:9000
+if [[ $CLICKHOUSE_MIGRATION_URL =~ ^clickhouse://([^:]+):([0-9]+)$ ]]; then
+    CLICKHOUSE_HOST="${BASH_REMATCH[1]}"
+    CLICKHOUSE_PORT="${BASH_REMATCH[2]}"
+elif [[ $CLICKHOUSE_MIGRATION_URL =~ ^clickhouse://([^:]+)$ ]]; then
+    CLICKHOUSE_HOST="${BASH_REMATCH[1]}"
+    CLICKHOUSE_PORT="9000"  # Default native protocol port
+else
+    echo "Error: Could not parse CLICKHOUSE_MIGRATION_URL: ${CLICKHOUSE_MIGRATION_URL}"
+    exit 1
+fi
+
+if ! command -v clickhouse &> /dev/null
+then
+	echo "Error: clickhouse binary could not be found. Please install ClickHouse client tools."
+	exit 1
+fi
+
+echo "Creating development tables in ClickHouse..."
+
+# Execute the CREATE TABLE statements
+# Add your development tables here using CREATE TABLE IF NOT EXISTS
+
+clickhouse client \
+  --host="${CLICKHOUSE_HOST}" \
+  --port="${CLICKHOUSE_PORT}" \
+  --user="${CLICKHOUSE_USER}" \
+  --password="${CLICKHOUSE_PASSWORD}" \
+  --database="${CLICKHOUSE_DB}" \
+  --multiquery <<EOF
+-- Create new events table for development setups.
+-- We expect this to be fully immutable and eventually replace observations.
+-- See LFE-5394 for ongoing discussion.
+-- Remove IF NOT EXISTS when moving this to prod migrations.
+CREATE TABLE IF NOT EXISTS events
+  (
+      org_id String, -- TODO: Unsure about this one
+      project_id String,
+      trace_id String,
+      span_id String,
+      parent_span_id Nullable(String),
+
+      start_time DateTime64(6),
+      end_time Nullable(DateTime64(6)),
+
+      -- Core properties
+      name String,
+      type LowCardinality(String),
+      environment LowCardinality(String) DEFAULT 'default',
+      version Nullable(String),
+
+      user_id String,
+      session_id String,
+
+      level LowCardinality(String),
+      status_message String, -- Threat '' and null the same for search
+      completion_start_time Nullable(DateTime64(6)),
+
+      -- Prompt
+      prompt_id Nullable(String),
+      prompt_name Nullable(String),
+      prompt_version Nullable(String),
+
+      -- Model
+      model_id Nullable(String),
+      provided_model_name Nullable(String),
+      model_parameters Nullable(String),
+
+      -- Usage
+      provided_usage_details Map(LowCardinality(String), UInt64),
+      usage_details Map(LowCardinality(String), UInt64),
+      provided_cost_details Map(LowCardinality(String), Decimal(18, 12)),
+      cost_details Map(LowCardinality(String), Decimal(18, 12)),
+      total_cost Decimal(18,12), -- 0 if not provided
+
+      -- I/O
+      input String CODEC(ZSTD(3)),
+      output String CODEC(ZSTD(3)),
+
+      -- TODO Metadata: Decide for approach
+      -- -- Approach 1: Use plain JSON type with default config
+      metadata JSON(max_dynamic_paths=1024, max_dynamic_types=32),
+      -- -- Approach 2: Uses ideas from https://www.uber.com/en-DE/blog/logging/
+      -- --             but uses Dynamic type to make this a single list
+      metadata_names Array(String),
+      metadata_values Array(Dynamic(max_types=32)),
+      -- -- Approach 3: 1:1 copy of https://www.uber.com/en-DE/blog/logging/
+      -- --             May require further high-level types and lots of thought during
+      -- --             write and query-time.
+      metadata_string_names Array(String),
+      metadata_string_values Array(String),
+      metadata_number_names Array(String),
+      metadata_number_values Array(Float64),
+      metadata_bool_names Array(String),
+      metadata_bool_values Array(UInt8),
+
+      -- Source metadata (Instrumentation)
+      source LowCardinality(String),
+      service_name Nullable(String),
+      service_version Nullable(String),
+      scope_name Nullable(String),
+      scope_version Nullable(String),
+      telemetry_sdk_language Nullable(String),
+      telemetry_sdk_name Nullable(String),
+      telemetry_sdk_version Nullable(String),
+
+      -- Generic props
+      blob_storage_file_path String,
+      event_raw String,
+      event_bytes UInt64,
+      created_at DateTime64(6) DEFAULT now(),
+      updated_at DateTime64(6) DEFAULT now(),
+      event_ts DateTime64(6),
+      is_deleted UInt8,
+
+      -- Indexes
+      INDEX idx_span_id span_id TYPE bloom_filter(0.01) GRANULARITY 1,
+      INDEX idx_trace_id trace_id TYPE bloom_filter(0.01) GRANULARITY 1,
+      INDEX idx_type type TYPE set(50) GRANULARITY 1,
+      INDEX idx_created_at created_at TYPE minmax GRANULARITY 1,
+      INDEX idx_updated_at updated_at TYPE minmax GRANULARITY 1,
+
+      -- Full Text Search Indexes (We should try different index sizes, e.g. 2048, 4096, or 8192)
+      INDEX idx_fts_input_1 input TYPE ngrambf_v1(1, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_input_2 input TYPE ngrambf_v1(2, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_input_4 input TYPE ngrambf_v1(4, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_input_8 input TYPE ngrambf_v1(8, 1024, 1, 0) GRANULARITY 1,
+
+      INDEX idx_fts_output_1 output TYPE ngrambf_v1(1, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_output_2 output TYPE ngrambf_v1(2, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_output_4 output TYPE ngrambf_v1(4, 1024, 1, 0) GRANULARITY 1,
+      INDEX idx_fts_output_8 output TYPE ngrambf_v1(8, 1024, 1, 0) GRANULARITY 1,
+  )
+  ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+  -- ENGINE = (Replicated)ReplacingMergeTree(event_ts, is_deleted)
+  PARTITION BY toYYYYMM(start_time)
+  ORDER BY (project_id, toUnixTimestamp(start_time), trace_id, span_id)
+EOF
+
+echo "Populating development tables with sample data..."
+
+clickhouse client \
+  --host="${CLICKHOUSE_HOST}" \
+  --port="${CLICKHOUSE_PORT}" \
+  --user="${CLICKHOUSE_USER}" \
+  --password="${CLICKHOUSE_PASSWORD}" \
+  --database="${CLICKHOUSE_DB}" \
+  --multiquery <<EOF
+  TRUNCATE events;
+  INSERT INTO events (org_id, project_id, trace_id, span_id, parent_span_id, start_time, end_time, name, type,
+                      environment, version, user_id, session_id, level, status_message, completion_start_time, prompt_id,
+                      prompt_name, prompt_version, model_id, provided_model_name, model_parameters,
+                      provided_usage_details, usage_details, provided_cost_details, cost_details, total_cost, input,
+                      output, metadata, metadata_names, metadata_values, metadata_string_names, metadata_string_values,
+                      metadata_number_names, metadata_number_values, metadata_bool_names, metadata_bool_values, source,
+                      service_name, service_version, scope_name, scope_version, telemetry_sdk_language,
+                      telemetry_sdk_name, telemetry_sdk_version, blob_storage_file_path, event_raw, event_bytes,
+                      created_at, updated_at, event_ts, is_deleted)
+  SELECT concat('o', project_id)                                                       AS org_id,
+         project_id,
+         trace_id,
+         id                                                                            AS span_id,
+         parent_observation_id                                                         AS parent_span_id,
+         start_time,
+         end_time,
+         name,
+         type,
+         environment,
+         version,
+         concat('u_', floor(randUniform(1, 100)))                                      AS user_id,
+         concat('s_', floor(randUniform(1, 100)))                                      AS session_id,
+         level,
+         ifNull(status_message, '')                                                    AS status_message,
+         completion_start_time,
+         prompt_id,
+         prompt_name,
+         CAST(prompt_version, 'Nullable(String)'),
+         internal_model_id                                                             AS model_id,
+         provided_model_name,
+         model_parameters,
+         provided_usage_details,
+         usage_details,
+         provided_cost_details,
+         cost_details,
+         ifNull(total_cost, 0)                                                         AS total_cost,
+         ifNull(input, '')                                                             AS input,
+         ifNull(output, '')                                                            AS output,
+         CAST(metadata, 'JSON'),
+         mapKeys(metadata)                                                             AS \`metadata.names\`,
+         mapValues(metadata)                                                           AS \`metadata.values\`,
+         mapKeys(metadata)                                                             AS metadata_string_names,
+         mapValues(metadata)                                                           AS metadata_string_values,
+         []                                                                            AS metadata_number_names,
+         []                                                                            AS metadata_number_values,
+         []                                                                            AS metadata_bool_names,
+         []                                                                            AS metadata_bool_values,
+         multiIf(mapContains(metadata, 'resourceAttributes'), 'otel', 'ingestion-api') AS source,
+         NULL                                                                          AS service_name,
+         NULL                                                                          AS service_version,
+         NULL                                                                          AS scope_name,
+         NULL                                                                          AS scope_version,
+         NULL                                                                          AS telemetry_sdk_language,
+         NULL                                                                          AS telemetry_sdk_name,
+         NULL                                                                          AS telemetry_sdk_version,
+         ''                                                                            AS blob_storage_file_path,
+         ''                                                                            AS event_raw,
+         0                                                                             AS event_bytes,
+         created_at,
+         updated_at,
+         event_ts,
+         is_deleted
+  FROM observations
+  WHERE (is_deleted = 0);
+EOF
+
+echo "Development tables created successfully (or already exist)."
+echo ""

packages/shared/package.json

@@ -47,7 +47,8 @@
     "ch:up": "bash clickhouse/scripts/up.sh",
     "ch:down": "bash clickhouse/scripts/down.sh",
     "ch:drop": "bash clickhouse/scripts/drop.sh",
-    "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed",
+    "ch:dev-tables": "bash clickhouse/scripts/dev-tables.sh",
+    "ch:reset": "pnpm run ch:down && pnpm run ch:up && pnpm run ch:seed && pnpm run ch:dev-tables",
     "ch:seed": "dotenv -e ../../.env -- ts-node -r tsconfig-paths/register -r dotenv/config --compiler-options '{\"module\":\"CommonJS\"}' scripts/seeder/seed-clickhouse.ts",
     "load:setup": "dotenv -e ../../.env -- tsx scripts/seeder/load-seed-clickhouse.ts"
   },
@@ -63,7 +64,7 @@
     "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.12.1",
     "@google-cloud/storage": "^7.17.0",
-    "@langchain/anthropic": "^0.3.27",
+    "@langchain/anthropic": "^0.3.29",
     "@langchain/aws": "^0.1.11",
     "@langchain/core": "^0.3.58",
     "@langchain/google-genai": "^0.2.12",
@@ -73,11 +74,12 @@
     "@prisma/client": "^6.10.1",
     "@react-email/components": "^0.5.1",
     "@react-email/render": "^1.2.1",
-    "@slack/oauth": "^3.0.3",
-    "@slack/web-api": "^7.9.3",
+    "@slack/oauth": "^3.0.4",
+    "@slack/web-api": "^7.10.0",
     "@types/bcryptjs": "^2.4.6",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.34.10",
+    "date-fns": "^3.3.1",
     "dd-trace": "^5.65.0",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.2",
@@ -86,7 +88,7 @@
     "ipaddr.js": "^2.2.0",
     "jsonpath-plus": "10.3.0",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.28",
+    "langchain": "^0.3.30",
     "langfuse-langchain": "3.38.4",
     "lodash": "^4.17.21",
     "lossless-json": "^4.1.1",
@@ -105,7 +107,7 @@
     "@types/node": "^24.3.0",
     "@types/nodemailer": "^6.4.16",
     "@types/pg": "^8.11.10",
-    "@types/react": "19.1.12",
+    "@types/react": "19.2.2",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
@@ -122,7 +124,7 @@
     "typescript": "^5.7.2"
   },
   "peerDependencies": {
-    "@types/react": "~19.1.12",
-    "react": "~19.1.1"
+    "@types/react": "~19.2.2",
+    "react": "~19.2.0"
   }
 }

packages/shared/prisma/generated/types.ts

@@ -332,6 +332,15 @@ export type BlobStorageIntegration = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type CloudSpendAlert = {
+  id: string;
+  org_id: string;
+  title: string;
+  threshold: string;
+  triggered_at: Timestamp | null;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type Comment = {
   id: string;
   project_id: string;
@@ -641,6 +650,11 @@ export type Organization = {
   updated_at: Generated<Timestamp>;
   cloud_config: unknown | null;
   metadata: unknown | null;
+  cloud_billing_cycle_anchor: Generated<Timestamp | null>;
+  cloud_billing_cycle_updated_at: Timestamp | null;
+  cloud_current_cycle_usage: number | null;
+  cloud_free_tier_usage_threshold_state: string | null;
+  ai_features_enabled: Generated<boolean>;
 };
 export type OrganizationMembership = {
   id: string;
@@ -846,6 +860,7 @@ export type DB = {
   batch_exports: BatchExport;
   billing_meter_backups: BillingMeterBackup;
   blob_storage_integrations: BlobStorageIntegration;
+  cloud_spend_alerts: CloudSpendAlert;
   comments: Comment;
   cron_jobs: CronJobs;
   dashboard_widgets: DashboardWidget;

packages/shared/prisma/migrations/20250925133604_organizations_add_ai_features_enabled/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "organizations" ADD COLUMN     "ai_features_enabled" BOOLEAN NOT NULL DEFAULT false;

packages/shared/prisma/migrations/20250930125453_job_executions_add_output_score_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "job_executions_project_id_job_output_score_id_idx" ON "job_executions"("project_id", "job_output_score_id");

packages/shared/prisma/migrations/20251001161539_organization_cloud_billing_cycle_columns/migration.sql

@@ -0,0 +1,21 @@
+-- AlterTable
+ALTER TABLE "organizations" ADD COLUMN     "cloud_billing_cycle_anchor" TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "cloud_billing_cycle_updated_at" TIMESTAMP(3),
+ADD COLUMN     "cloud_current_cycle_usage" INTEGER,
+ADD COLUMN     "cloud_free_tier_usage_threshold_state" TEXT;
+
+-- Backfill cloud_billing_cycle_anchor for existing organizations:
+-- Step 1: Set to NULL for orgs WITH active subscriptions (will be backfilled from Stripe by worker job)
+UPDATE "organizations"
+SET "cloud_billing_cycle_anchor" = NULL;
+
+-- Step 2: Set to created_at for orgs WITHOUT active subscriptions (free tier orgs)
+UPDATE "organizations"
+SET "cloud_billing_cycle_anchor" = "created_at"
+WHERE
+    "cloud_config" IS NULL
+    OR NOT (
+      "cloud_config"::jsonb -> 'stripe' ? 'activeSubscriptionId'
+      AND "cloud_config"::jsonb -> 'stripe' ->> 'activeSubscriptionId' IS NOT NULL
+      AND "cloud_config"::jsonb -> 'stripe' ->> 'activeSubscriptionId' != ''
+    );

packages/shared/prisma/migrations/20251002153814_add_backfill_billing_cycle_anchors_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('0199b890-1093-7d1f-b662-be3c03527e93', '20250102_backfill_billing_cycle_anchors', 'backfillBillingCycleAnchors', '{}');

packages/shared/prisma/migrations/20251006173445_add_cloud_spend_alerts/migration.sql

@@ -0,0 +1,15 @@
+-- CreateTable
+CREATE TABLE "cloud_spend_alerts" (
+    "id" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "threshold" DECIMAL(65,30) NOT NULL,
+    "triggered_at" TIMESTAMP(3),
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "cloud_spend_alerts_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "cloud_spend_alerts" ADD CONSTRAINT "cloud_spend_alerts_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20251006173446_optimize_cloud_spend_alerts_add_index/migration.sql

@@ -0,0 +1,3 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY "cloud_spend_alerts_org_id_idx" ON "cloud_spend_alerts"("org_id");
+

packages/shared/prisma/schema.prisma

@@ -104,17 +104,23 @@ model VerificationToken {
 }
 
 model Organization {
-  id                      String                   @id @default(cuid())
-  name                    String
-  createdAt               DateTime                 @default(now()) @map("created_at")
-  updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
-  cloudConfig             Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
-  metadata                Json?
-  organizationMemberships OrganizationMembership[]
-  projects                Project[]
-  MembershipInvitation    MembershipInvitation[]
-  ApiKey                  ApiKey[]
-  surveys                 Survey[]
+  id                        String                   @id @default(cuid())
+  name                      String
+  createdAt                 DateTime                 @default(now()) @map("created_at")
+  updatedAt                 DateTime                 @default(now()) @updatedAt @map("updated_at")
+  cloudConfig               Json?                    @map("cloud_config") // Langfuse Cloud, for zod schema see @/src/features/organizations/utils/cloudConfigSchema
+  metadata                  Json?
+  cloudBillingCycleAnchor          DateTime?                @default(now()) @map("cloud_billing_cycle_anchor")
+  cloudBillingCycleUpdatedAt       DateTime?                @map("cloud_billing_cycle_updated_at")
+  cloudCurrentCycleUsage           Int?                     @map("cloud_current_cycle_usage")
+  cloudFreeTierUsageThresholdState String?                  @map("cloud_free_tier_usage_threshold_state")
+  aiFeaturesEnabled         Boolean                  @default(false) @map("ai_features_enabled")
+  organizationMemberships   OrganizationMembership[]
+  projects                  Project[]
+  MembershipInvitation      MembershipInvitation[]
+  ApiKey                    ApiKey[]
+  surveys                   Survey[]
+  cloudSpendAlerts          CloudSpendAlert[]
 
   @@map("organizations")
 }
@@ -853,6 +859,8 @@ model EvalTemplate {
   @@map("eval_templates")
 }
 
+// We currently assume in the evalRouter that _all_ job_executions are for EVAL job_configs.
+// If we ever extend this, we need to adjust the filter condition there. ref.: fetchJobExecutionsByStatus.
 enum JobType {
   EVAL
 }
@@ -927,6 +935,7 @@ model JobExecution {
   @@index([projectId, jobConfigurationId, jobInputTraceId])
   @@index([projectId, status])
   @@index([projectId, id])
+  @@index([projectId, jobOutputScoreId])
   @@map("job_executions")
 }
 
@@ -1425,3 +1434,20 @@ enum SurveyName {
 
   @@map("SurveyName")
 }
+
+model CloudSpendAlert {
+  id         String    @id @default(cuid())
+  orgId      String    @map("org_id")
+  org        Organization @relation(fields: [orgId], references: [id], onDelete: Cascade)
+  
+  title      String    // e.g., "Production Alert"
+  threshold  Decimal   @map("threshold") // USD amount
+  
+  triggeredAt DateTime? @map("triggered_at") // Last trigger timestamp
+  
+  createdAt  DateTime  @default(now()) @map("created_at")
+  updatedAt  DateTime  @default(now()) @updatedAt @map("updated_at")
+
+  @@index([orgId])
+  @@map("cloud_spend_alerts")
+}

packages/shared/scripts/seeder/seed-postgres.ts

@@ -8,6 +8,7 @@ import {
   JobExecutionStatus,
   PrismaClient,
   type Project,
+  ScoreConfigCategoryDomain,
   ScoreDataType,
 } from "../../src/index";
 import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
@@ -29,11 +30,6 @@ import {
   generateEvalTraceId,
 } from "./utils/seed-helpers";
 
-type ConfigCategory = {
-  label: string;
-  value: number;
-};
-
 const options = {
   environment: { type: "string" },
 } as const;
@@ -726,6 +722,7 @@ async function generatePrompts(project: Project) {
         createdBy: version.createdBy,
         prompt: version.prompt,
         name: version.name,
+        type: version.type ?? "text",
         config: version.config,
         version: version.version,
         labels: version.labels,
@@ -747,7 +744,7 @@ async function generateConfigsForProject(projects: Project[]) {
       name: string;
       id: string;
       dataType: ScoreDataType;
-      categories: ConfigCategory[] | null;
+      categories: ScoreConfigCategoryDomain[] | null;
     }[]
   > = new Map();
 
@@ -797,7 +794,7 @@ async function generateConfigs(project: Project) {
     name: string;
     id: string;
     dataType: ScoreDataType;
-    categories: ConfigCategory[] | null;
+    categories: ScoreConfigCategoryDomain[] | null;
   }[] = [];
 
   const configs = [
@@ -874,7 +871,7 @@ async function generateQueuesForProject(
       name: string;
       id: string;
       dataType: ScoreDataType;
-      categories: ConfigCategory[] | null;
+      categories: ScoreConfigCategoryDomain[] | null;
     }[]
   >,
 ) {
@@ -898,7 +895,7 @@ async function generateQueues(
     name: string;
     id: string;
     dataType: ScoreDataType;
-    categories: ConfigCategory[] | null;
+    categories: ScoreConfigCategoryDomain[] | null;
   }[],
 ) {
   const queue = {

packages/shared/scripts/seeder/utils/framework-traces/README.md

@@ -0,0 +1,19 @@
+# Framework Traces
+
+This folder contains real traces produced through framework instrumentation.
+Most of them stem from here: https://langfuse.com/integrations/frameworks/agno-agents and so on.
+
+## How to add further traces
+1. Generate a trace
+2. Download the trace from the UI using the download button. This **excludes** the `input`/`output`/`metadata` fields of the observations
+3. In the trace, click `Log View (Beta)` and switch to `JSON` format. Click the copy all button and save this to a file in this folder.
+4. Run the script `npx ts-node merge-observations.ts trace-file.json observations.json trace-merged.json`
+5. Leave the `merged` file in this folder. Name it with the date of the trace as displayed in the UI.
+6. ???
+7. Profit
+
+## How to use the trace in the UI
+
+All trace ids are like `framework-frameworkName-traceId`, so search for framework in the trace table.
+We don't rewrite the time, so you have to filter for All Time most likely.
+Also, you can filter for `source: "framework-trace"` on the trace to find them.

packages/shared/scripts/seeder/utils/framework-traces/autogen-2025-06-06.json

@@ -0,0 +1,149 @@
+{
+  "trace": {
+    "id": "1b72c51fabed12ae7df83bfd4a09f545",
+    "projectId": "cloramnkj0002jz088vzn1ja4",
+    "name": "autogen-chat-trace",
+    "timestamp": "2025-06-06T11:31:33.965Z",
+    "environment": "default",
+    "tags": [
+      "autogen",
+      "dev"
+    ],
+    "bookmarked": false,
+    "release": null,
+    "version": "1.0.0",
+    "userId": "user_123",
+    "sessionId": "session_abc",
+    "public": true,
+    "input": "\"Say 'Hello World!'\"",
+    "output": "\"Hello World!\"",
+    "metadata": "{\"email\":\"user@langfuse.com\",\"resourceAttributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.33.1\",\"service.name\":\"unknown_service\"},\"scope\":{\"name\":\"langfuse-sdk\",\"version\":\"3.0.0\",\"attributes\":{\"public_key\":\"pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba\"}}}",
+    "createdAt": "2025-06-06T11:31:35.000Z",
+    "updatedAt": "2025-06-06T11:31:35.337Z",
+    "scores": [],
+    "latency": 0.645
+  },
+  "observations": [
+    {
+      "id": "95f4f51edeb603d9",
+      "traceId": "1b72c51fabed12ae7df83bfd4a09f545",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2025-06-06T11:31:33.965Z",
+      "endTime": "2025-06-06T11:31:34.610Z",
+      "name": "autogen-chat-trace",
+      "metadata": {
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "langfuse-sdk",
+          "version": "3.0.0",
+          "attributes": {
+            "public_key": "pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba"
+          }
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "1.0.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-06T11:31:35.295Z",
+      "updatedAt": "2025-06-06T11:31:35.337Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 645,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "83517f9062f5ada4",
+      "traceId": "1b72c51fabed12ae7df83bfd4a09f545",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "95f4f51edeb603d9",
+      "startTime": "2025-06-06T11:31:33.978Z",
+      "endTime": "2025-06-06T11:31:34.606Z",
+      "name": "chat gpt-4o",
+      "metadata": {
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "langfuse-sdk",
+          "version": "3.0.0",
+          "attributes": {
+            "public_key": "pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba"
+          }
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {
+        "seed": "",
+        "frequency_penalty": 0,
+        "max_tokens": -1,
+        "presence_penalty": 0,
+        "stop_sequences": "{\"arrayValue\":{}}",
+        "temperature": 1,
+        "top_p": 1,
+        "service_tier": "auto",
+        "user": "",
+        "is_stream": false
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-06T11:31:35.090Z",
+      "updatedAt": "2025-06-06T11:31:35.153Z",
+      "usageDetails": {
+        "input": 41,
+        "output": 3,
+        "total": 44
+      },
+      "costDetails": {
+        "total": 0.000025
+      },
+      "providedCostDetails": {
+        "total": 0.000025
+      },
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 628,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0.000025,
+      "inputUsage": 41,
+      "outputUsage": 3,
+      "totalUsage": 44,
+      "input": "system: You are a helpful AI assistant. Solve tasks using your tools. Reply with TERMINATE when the task has been completed.\nuser: Say 'Hello World!'",
+      "output": "Hello World!"
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/framework-traces/framework-trace-loader.ts

@@ -0,0 +1,203 @@
+import { readFileSync, readdirSync } from "fs";
+import path from "path";
+import {
+  createTrace,
+  createObservation,
+  logger,
+  type TraceRecordInsertType,
+  type ObservationRecordInsertType,
+  type ScoreRecordInsertType,
+} from "../../../../src/server";
+
+/**
+ * Loads framework traces from JSON files and converts them to ClickHouse insert types.
+ * Expected JSON structure:
+ * {
+ *   trace: { trace data without observations },
+ *   observations: [{ individual observation with id and input/output/metadata and all the other stuff}]
+ * }
+ */
+export class FrameworkTraceLoader {
+  private frameworkTracesDir: string;
+
+  constructor() {
+    this.frameworkTracesDir = __dirname;
+  }
+
+  /**
+   * Load and adapt all framework traces for a project.
+   */
+  loadTracesForProject(projectId: string): {
+    traces: TraceRecordInsertType[];
+    observations: ObservationRecordInsertType[];
+    scores: ScoreRecordInsertType[];
+  } {
+    const traces: TraceRecordInsertType[] = [];
+    const observations: ObservationRecordInsertType[] = [];
+    const scores: ScoreRecordInsertType[] = [];
+
+    const files = readdirSync(this.frameworkTracesDir).filter(
+      (file) => file.endsWith(".json") && file !== "package.json",
+    );
+
+    for (const file of files) {
+      const filePath = path.join(this.frameworkTracesDir, file);
+      const content = readFileSync(filePath, "utf-8");
+      const data = JSON.parse(content);
+
+      const frameworkName = file.replace(".json", "");
+      const adapted = this.adaptTrace(data, projectId, frameworkName);
+
+      traces.push(adapted.trace);
+      observations.push(...adapted.observations);
+      scores.push(...adapted.scores);
+
+      logger.info(
+        `Loaded 1 trace with ${adapted.observations.length} observations from ${frameworkName}`,
+      );
+    }
+
+    return { traces, observations, scores };
+  }
+
+  /**
+   * Adapt a single framework trace to a specific project.
+   */
+  private adaptTrace(
+    data: any,
+    projectId: string,
+    frameworkName: string,
+  ): {
+    trace: TraceRecordInsertType;
+    observations: ObservationRecordInsertType[];
+    scores: ScoreRecordInsertType[];
+  } {
+    const rawTrace = data.trace;
+    const rawObservations = data.observations || [];
+
+    const originalTraceId = rawTrace.id;
+    const newTraceId = originalTraceId;
+    // we just change the name for now.
+    // const newTraceId = `framework-${frameworkName}-${originalTraceId}-${projectId.slice(-8)}`;
+    const newName = `framework-${frameworkName}-${rawTrace.name}`;
+
+    // Parse trace metadata if it's a string
+    let metadata = rawTrace.metadata;
+    if (typeof metadata === "string") {
+      metadata = JSON.parse(metadata);
+    }
+
+    // Create trace
+    const trace = createTrace({
+      id: newTraceId,
+      project_id: projectId,
+      name: newName,
+      timestamp: new Date(rawTrace.timestamp).getTime(),
+      input: rawTrace.input,
+      output: rawTrace.output,
+      user_id: rawTrace.userId,
+      session_id: rawTrace.sessionId,
+      environment: rawTrace.environment,
+      metadata: {
+        ...metadata,
+        // metadata to filter for those kinda traces
+        source: "framework-trace",
+        framework: frameworkName,
+      },
+      release: rawTrace.release,
+      version: rawTrace.version,
+      public: rawTrace.public,
+      bookmarked: rawTrace.bookmarked,
+      tags: rawTrace.tags,
+    });
+
+    // Map observation IDs (old -> new)
+    const obsIdMap = new Map<string, string>();
+    for (const obs of rawObservations) {
+      const newObsId = `framework-${frameworkName}-${obs.id}-${projectId.slice(-8)}`;
+      obsIdMap.set(obs.id, newObsId);
+    }
+
+    // Create observations
+    const observations: ObservationRecordInsertType[] = [];
+    for (const obs of rawObservations) {
+      const newObsId = obsIdMap.get(obs.id)!;
+
+      // Map parent observation ID
+      const parentObservationId = obs.parentObservationId
+        ? obsIdMap.get(obs.parentObservationId) || null
+        : null;
+
+      // Convert input/output to strings, preserving null
+      const input =
+        obs.input === undefined || obs.input === null
+          ? obs.input
+          : JSON.stringify(obs.input);
+      const output =
+        obs.output === undefined || obs.output === null
+          ? obs.output
+          : JSON.stringify(obs.output);
+
+      // Parse metadata if it's a string
+      let obsMetadata = obs.metadata;
+      if (typeof obsMetadata === "string") {
+        obsMetadata = JSON.parse(obsMetadata);
+      }
+
+      // Use usage/cost details from JSON if present, otherwise reconstruct from flat fields
+      const usageDetails =
+        obs.usageDetails && Object.keys(obs.usageDetails).length > 0
+          ? obs.usageDetails
+          : obs.totalUsage > 0
+            ? {
+                input: obs.inputUsage,
+                output: obs.outputUsage,
+                total: obs.totalUsage,
+              }
+            : undefined;
+
+      const costDetails =
+        obs.costDetails && Object.keys(obs.costDetails).length > 0
+          ? obs.costDetails
+          : obs.totalCost > 0
+            ? {
+                input: (obs.totalCost * obs.inputUsage) / obs.totalUsage,
+                output: (obs.totalCost * obs.outputUsage) / obs.totalUsage,
+                total: obs.totalCost,
+              }
+            : undefined;
+
+      observations.push(
+        createObservation({
+          id: newObsId,
+          trace_id: newTraceId,
+          project_id: projectId,
+          type: obs.type,
+          parent_observation_id: parentObservationId,
+          start_time: new Date(obs.startTime).getTime(),
+          end_time: obs.endTime ? new Date(obs.endTime).getTime() : undefined,
+          name: obs.name,
+          metadata: obsMetadata,
+          level: obs.level,
+          status_message: obs.statusMessage,
+          input,
+          output,
+          provided_model_name: obs.model,
+          model_parameters: obs.modelParameters
+            ? JSON.stringify(obs.modelParameters)
+            : undefined,
+          prompt_name: obs.promptName,
+          prompt_version: obs.promptVersion,
+          usage_details: usageDetails,
+          provided_usage_details: obs.providedUsageDetails,
+          cost_details: costDetails,
+          provided_cost_details: obs.providedCostDetails,
+          total_cost: obs.totalCost,
+          environment: rawTrace.environment,
+        }),
+      );
+    }
+
+    return { trace, observations, scores: [] };
+  }
+}

packages/shared/scripts/seeder/utils/framework-traces/koog-2025-08-26.json

@@ -0,0 +1,863 @@
+{
+  "trace": {
+    "id": "dff173a675b759ce1b70e522b27d6846",
+    "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+    "name": "agent.f9f77d1a-437f-4ba6-890e-b592a50cbd21",
+    "timestamp": "2025-08-26T19:40:59.145Z",
+    "environment": "default",
+    "tags": [],
+    "bookmarked": false,
+    "release": null,
+    "version": "0.3.0",
+    "userId": null,
+    "sessionId": null,
+    "public": true,
+    "input": null,
+    "output": null,
+    "metadata": "{\"attributes\":{\"gen_ai.request.model\":\"gpt-4o\",\"gen_ai.system\":\"openai\",\"gen_ai.agent.id\":\"f9f77d1a-437f-4ba6-890e-b592a50cbd21\",\"gen_ai.operation.name\":\"create_agent\"},\"resourceAttributes\":{\"os.arch\":\"aarch64\",\"os.type\":\"Mac OS X\",\"os.version\":\"15.4.1\",\"service.instance.time\":\"2025-08-26T19:40:59.130398Z\",\"service.name\":\"ai.koog\",\"service.version\":\"0.3.0\"},\"scope\":{\"name\":\"ai.koog\",\"version\":\"0.3.0\",\"attributes\":{}}}",
+    "createdAt": "2025-08-26T19:40:59.145Z",
+    "updatedAt": "2025-08-26T19:41:05.434Z",
+    "scores": [],
+    "latency": 5.843
+  },
+  "observations": [
+    {
+      "id": "1b7200380e3316e7",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "AGENT",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2025-08-26T19:40:59.145Z",
+      "endTime": "2025-08-26T19:41:04.988Z",
+      "name": "agent.f9f77d1a-437f-4ba6-890e-b592a50cbd21",
+      "metadata": {
+        "attributes": {
+          "gen_ai.request.model": "gpt-4o",
+          "gen_ai.system": "openai",
+          "gen_ai.agent.id": "f9f77d1a-437f-4ba6-890e-b592a50cbd21",
+          "gen_ai.operation.name": "create_agent"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:05.401Z",
+      "updatedAt": "2025-08-26T19:41:05.420Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 5843,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "bb189bc3771ff83d",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "AGENT",
+      "environment": "default",
+      "parentObservationId": "1b7200380e3316e7",
+      "startTime": "2025-08-26T19:40:59.150Z",
+      "endTime": "2025-08-26T19:41:04.988Z",
+      "name": "run.6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+      "metadata": {
+        "attributes": {
+          "koog.agent.strategy.name": "single_run_sequential",
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "gen_ai.system": "openai",
+          "gen_ai.agent.id": "f9f77d1a-437f-4ba6-890e-b592a50cbd21",
+          "gen_ai.operation.name": "invoke_agent"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:05.330Z",
+      "updatedAt": "2025-08-26T19:41:05.354Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 5838,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "7578374e12a7faa5",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "bb189bc3771ff83d",
+      "startTime": "2025-08-26T19:41:04.984Z",
+      "endTime": "2025-08-26T19:41:04.985Z",
+      "name": "node.__finish__",
+      "metadata": {
+        "langgraph_node": "__finish__",
+        "langgraph_step": 4,
+        "attributes": {
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "koog.node.name": "__finish__",
+          "langfuse.observation.metadata.langgraph_node": "__finish__",
+          "langfuse.observation.metadata.langgraph_step": "4"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:05.313Z",
+      "updatedAt": "2025-08-26T19:41:05.340Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "e44e72cf2d221781",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "6021df46275ad2d8",
+      "startTime": "2025-08-26T19:41:03.041Z",
+      "endTime": "2025-08-26T19:41:04.983Z",
+      "name": "llm.chat",
+      "metadata": {
+        "attributes": {
+          "gen_ai.prompt.4.content": [
+            {
+              "function": {
+                "name": "ByeTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_dOeEHJDI0liHrCdFWTpkTsuO",
+              "type": "function"
+            }
+          ],
+          "gen_ai.prompt.5.role": "tool",
+          "gen_ai.completion.0.content": "Hello my darling Bob! And, bye my dear Bob!",
+          "gen_ai.request.temperature": "1",
+          "gen_ai.prompt.3.role": "tool",
+          "gen_ai.prompt.0.role": "system",
+          "gen_ai.prompt.5.content": "Bye my dear Bob!",
+          "gen_ai.completion.0.role": "assistant",
+          "gen_ai.prompt.4.role": "tool",
+          "gen_ai.request.model": "gpt-4o",
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "gen_ai.prompt.0.content": "You are a nice and polite assistant.",
+          "gen_ai.prompt.1.role": "user",
+          "gen_ai.response.finish_reasons": [
+            "stop"
+          ],
+          "gen_ai.prompt.1.content": "Greet and say goodbye to the user in the same message. Use provided tools to generate a proper greeting. The user name is Bob.",
+          "gen_ai.system": "openai",
+          "gen_ai.prompt.2.content": [
+            {
+              "function": {
+                "name": "HelloTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_n2HaWg9KFP0BZDqXv6G4b3nJ",
+              "type": "function"
+            }
+          ],
+          "gen_ai.operation.name": "chat",
+          "gen_ai.prompt.2.role": "tool",
+          "gen_ai.prompt.3.content": "Hello my darling Bob!"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": {
+        "temperature": 1
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:05.314Z",
+      "updatedAt": "2025-08-26T19:41:05.333Z",
+      "usageDetails": {
+        "input": 168,
+        "output": 19,
+        "total": 187
+      },
+      "costDetails": {
+        "input": 0.00042,
+        "output": 0.00019,
+        "total": 0.00061
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1942,
+      "timeToFirstToken": null,
+      "inputCost": 0.00042,
+      "outputCost": 0.00019,
+      "totalCost": 0.00061,
+      "inputUsage": 168,
+      "outputUsage": 19,
+      "totalUsage": 187,
+      "input": [
+        {
+          "role": "system",
+          "content": "You are a nice and polite assistant."
+        },
+        {
+          "role": "user",
+          "content": "Greet and say goodbye to the user in the same message. Use provided tools to generate a proper greeting. The user name is Bob."
+        },
+        {
+          "content": [
+            {
+              "function": {
+                "name": "HelloTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_n2HaWg9KFP0BZDqXv6G4b3nJ",
+              "type": "function"
+            }
+          ],
+          "role": "tool"
+        },
+        {
+          "role": "tool",
+          "content": "Hello my darling Bob!"
+        },
+        {
+          "content": [
+            {
+              "function": {
+                "name": "ByeTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_dOeEHJDI0liHrCdFWTpkTsuO",
+              "type": "function"
+            }
+          ],
+          "role": "tool"
+        },
+        {
+          "role": "tool",
+          "content": "Bye my dear Bob!"
+        }
+      ],
+      "output": [
+        {
+          "content": "Hello my darling Bob! And, bye my dear Bob!",
+          "role": "assistant"
+        }
+      ]
+    },
+    {
+      "id": "6021df46275ad2d8",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "bb189bc3771ff83d",
+      "startTime": "2025-08-26T19:41:03.038Z",
+      "endTime": "2025-08-26T19:41:04.984Z",
+      "name": "node.nodeSendToolResult",
+      "metadata": {
+        "langgraph_node": "nodeSendToolResult",
+        "langgraph_step": 3,
+        "attributes": {
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "koog.node.name": "nodeSendToolResult",
+          "langfuse.observation.metadata.langgraph_node": "nodeSendToolResult",
+          "langfuse.observation.metadata.langgraph_step": "3"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:05.305Z",
+      "updatedAt": "2025-08-26T19:41:05.324Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1946,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "33188493784a060d",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "5555afb6655e4d44",
+      "startTime": "2025-08-26T19:41:02.023Z",
+      "endTime": "2025-08-26T19:41:03.037Z",
+      "name": "tool.ByeTool",
+      "metadata": {
+        "attributes": {
+          "input.value": {
+            "name": "Bob"
+          },
+          "output.value": "Bye my dear Bob!",
+          "gen_ai.tool.description": "A tool that says bye to the user",
+          "gen_ai.tool.call.id": "call_dOeEHJDI0liHrCdFWTpkTsuO",
+          "gen_ai.tool.name": "ByeTool"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:03.511Z",
+      "updatedAt": "2025-08-26T19:41:03.531Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1014,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "name": "Bob"
+      },
+      "output": "Bye my dear Bob!"
+    },
+    {
+      "id": "66f269083a2fff81",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "5555afb6655e4d44",
+      "startTime": "2025-08-26T19:41:02.023Z",
+      "endTime": "2025-08-26T19:41:03.034Z",
+      "name": "tool.HelloTool",
+      "metadata": {
+        "attributes": {
+          "input.value": {
+            "name": "Bob"
+          },
+          "output.value": "Hello my darling Bob!",
+          "gen_ai.tool.description": "A tool greets the user",
+          "gen_ai.tool.call.id": "call_n2HaWg9KFP0BZDqXv6G4b3nJ",
+          "gen_ai.tool.name": "HelloTool"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:03.431Z",
+      "updatedAt": "2025-08-26T19:41:03.479Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1011,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "name": "Bob"
+      },
+      "output": "Hello my darling Bob!"
+    },
+    {
+      "id": "5555afb6655e4d44",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "bb189bc3771ff83d",
+      "startTime": "2025-08-26T19:41:02.017Z",
+      "endTime": "2025-08-26T19:41:03.038Z",
+      "name": "node.nodeExecuteTool",
+      "metadata": {
+        "langgraph_node": "nodeExecuteTool",
+        "langgraph_step": 2,
+        "attributes": {
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "koog.node.name": "nodeExecuteTool",
+          "langfuse.observation.metadata.langgraph_node": "nodeExecuteTool",
+          "langfuse.observation.metadata.langgraph_step": "2"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:03.432Z",
+      "updatedAt": "2025-08-26T19:41:03.456Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1021,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "6a0770019fe7c2b4",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "bb189bc3771ff83d",
+      "startTime": "2025-08-26T19:40:59.171Z",
+      "endTime": "2025-08-26T19:41:02.017Z",
+      "name": "node.nodeCallLLM",
+      "metadata": {
+        "langgraph_node": "nodeCallLLM",
+        "langgraph_step": 1,
+        "attributes": {
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "koog.node.name": "nodeCallLLM",
+          "langfuse.observation.metadata.langgraph_node": "nodeCallLLM",
+          "langfuse.observation.metadata.langgraph_step": "1"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:02.422Z",
+      "updatedAt": "2025-08-26T19:41:02.440Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2846,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "042f9350c3729147",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "6a0770019fe7c2b4",
+      "startTime": "2025-08-26T19:40:59.176Z",
+      "endTime": "2025-08-26T19:41:02.016Z",
+      "name": "llm.chat",
+      "metadata": {
+        "attributes": {
+          "gen_ai.completion.0.content": [
+            {
+              "function": {
+                "name": "HelloTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_n2HaWg9KFP0BZDqXv6G4b3nJ",
+              "type": "function"
+            }
+          ],
+          "gen_ai.request.temperature": "1",
+          "gen_ai.prompt.0.role": "system",
+          "gen_ai.completion.0.role": "assistant",
+          "gen_ai.request.model": "gpt-4o",
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "gen_ai.completion.0.finish_reason": "tool_calls",
+          "gen_ai.prompt.0.content": "You are a nice and polite assistant.",
+          "gen_ai.prompt.1.role": "user",
+          "gen_ai.response.finish_reasons": [
+            "tool_calls"
+          ],
+          "gen_ai.prompt.1.content": "Greet and say goodbye to the user in the same message. Use provided tools to generate a proper greeting. The user name is Bob.",
+          "gen_ai.completion.1.finish_reason": "tool_calls",
+          "gen_ai.system": "openai",
+          "gen_ai.completion.1.role": "assistant",
+          "gen_ai.completion.1.content": [
+            {
+              "function": {
+                "name": "ByeTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_dOeEHJDI0liHrCdFWTpkTsuO",
+              "type": "function"
+            }
+          ],
+          "gen_ai.operation.name": "chat"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": {
+        "temperature": 1
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:41:02.328Z",
+      "updatedAt": "2025-08-26T19:41:02.356Z",
+      "usageDetails": {
+        "input": 47,
+        "output": 106,
+        "total": 153
+      },
+      "costDetails": {
+        "input": 0.0001175,
+        "output": 0.00106,
+        "total": 0.0011775
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2840,
+      "timeToFirstToken": null,
+      "inputCost": 0.0001175,
+      "outputCost": 0.00106,
+      "totalCost": 0.0011775,
+      "inputUsage": 47,
+      "outputUsage": 106,
+      "totalUsage": 153,
+      "input": [
+        {
+          "role": "system",
+          "content": "You are a nice and polite assistant."
+        },
+        {
+          "role": "user",
+          "content": "Greet and say goodbye to the user in the same message. Use provided tools to generate a proper greeting. The user name is Bob."
+        }
+      ],
+      "output": [
+        {
+          "content": [
+            {
+              "function": {
+                "name": "HelloTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_n2HaWg9KFP0BZDqXv6G4b3nJ",
+              "type": "function"
+            }
+          ],
+          "role": "assistant",
+          "finish_reason": "tool_calls"
+        },
+        {
+          "finish_reason": "tool_calls",
+          "role": "assistant",
+          "content": [
+            {
+              "function": {
+                "name": "ByeTool",
+                "arguments": {
+                  "name": "Bob"
+                }
+              },
+              "id": "call_dOeEHJDI0liHrCdFWTpkTsuO",
+              "type": "function"
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "id": "890e9114a4449257",
+      "traceId": "dff173a675b759ce1b70e522b27d6846",
+      "projectId": "cmcmdwcag00c2ad077xp1qnyc",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "bb189bc3771ff83d",
+      "startTime": "2025-08-26T19:40:59.153Z",
+      "endTime": "2025-08-26T19:40:59.153Z",
+      "name": "node.__start__",
+      "metadata": {
+        "langgraph_node": "__start__",
+        "langgraph_step": 0,
+        "attributes": {
+          "gen_ai.conversation.id": "6f8d2449-33ed-4c7c-82e7-5410e34ee4e6",
+          "koog.node.name": "__start__",
+          "langfuse.observation.metadata.langgraph_node": "__start__",
+          "langfuse.observation.metadata.langgraph_step": "0"
+        },
+        "resourceAttributes": {
+          "os.arch": "aarch64",
+          "os.type": "Mac OS X",
+          "os.version": "15.4.1",
+          "service.instance.time": "2025-08-26T19:40:59.130398Z",
+          "service.name": "ai.koog",
+          "service.version": "0.3.0"
+        },
+        "scope": {
+          "name": "ai.koog",
+          "version": "0.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "0.3.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-08-26T19:40:59.897Z",
+      "updatedAt": "2025-08-26T19:40:59.915Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 0,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/framework-traces/llamaindex-2025-06-05.json

@@ -0,0 +1,182 @@
+{
+  "trace": {
+    "id": "12ea412956f99347b0503c1144acd0ec",
+    "projectId": "cloramnkj0002jz088vzn1ja4",
+    "name": "llama-index-trace",
+    "timestamp": "2025-06-05T15:45:52.971Z",
+    "environment": "default",
+    "tags": [
+      "llama-index",
+      "rag"
+    ],
+    "bookmarked": false,
+    "release": null,
+    "version": "1.0.0",
+    "userId": "user_123",
+    "sessionId": "session_abc",
+    "public": true,
+    "input": "\"What is Langfuse?\"",
+    "output": "\"Langfuse is a tool designed to help developers monitor and debug applications that utilize large language models (LLMs). It provides features for tracking and analyzing the performance of LLMs, enabling developers to gain insights into how these models are functioning within their applications. Langfuse can be particularly useful for identifying issues, optimizing performance, and ensuring that the integration of language models into applications is smooth and effective.\"",
+    "metadata": "{\"email\":\"user@langfuse.com\",\"resourceAttributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.33.1\",\"service.name\":\"unknown_service\"},\"scope\":{\"name\":\"langfuse-sdk\",\"version\":\"3.0.0\",\"attributes\":{\"public_key\":\"pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba\"}}}",
+    "createdAt": "2025-06-05T15:45:56.000Z",
+    "updatedAt": "2025-06-05T15:45:56.029Z",
+    "scores": [],
+    "latency": 2.779
+  },
+  "observations": [
+    {
+      "id": "3344aabdd4e66110",
+      "traceId": "12ea412956f99347b0503c1144acd0ec",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2025-06-05T15:45:52.971Z",
+      "endTime": "2025-06-05T15:45:55.750Z",
+      "name": "llama-index-trace",
+      "metadata": {
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "langfuse-sdk",
+          "version": "3.0.0",
+          "attributes": {
+            "public_key": "pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba"
+          }
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "1.0.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-05T15:45:55.996Z",
+      "updatedAt": "2025-06-05T15:45:56.027Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2779,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "7ca8b22f6628d0c7",
+      "traceId": "12ea412956f99347b0503c1144acd0ec",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "3344aabdd4e66110",
+      "startTime": "2025-06-05T15:45:52.971Z",
+      "endTime": "2025-06-05T15:45:55.747Z",
+      "name": "OpenAI.complete",
+      "metadata": {
+        "attributes": {
+          "llm.model_name": "gpt-4o",
+          "llm.invocation_parameters": {
+            "context_window": 128000,
+            "num_output": -1,
+            "is_chat_model": true,
+            "is_function_calling_model": true,
+            "model_name": "gpt-4o",
+            "system_role": "system"
+          },
+          "llm.provider": "openai",
+          "llm.system": "openai",
+          "input.value": {
+            "args": [
+              "What is Langfuse?"
+            ]
+          },
+          "input.mime_type": "application/json",
+          "llm.prompts": [
+            "What is Langfuse?"
+          ],
+          "output.value": "Langfuse is a tool designed to help developers monitor and debug applications that utilize large language models (LLMs). It provides features for tracking and analyzing the performance of LLMs, enabling developers to gain insights into how these models are functioning within their applications. Langfuse can be particularly useful for identifying issues, optimizing performance, and ensuring that the integration of language models into applications is smooth and effective.",
+          "llm.token_count.prompt": "13",
+          "llm.token_count.prompt_details.cache_read": "0",
+          "llm.token_count.prompt_details.audio": "0",
+          "llm.token_count.completion": "81",
+          "llm.token_count.completion_details.reasoning": "0",
+          "llm.token_count.completion_details.audio": "0",
+          "llm.token_count.total": "94",
+          "openinference.span.kind": "LLM"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.llama_index",
+          "version": "4.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {
+        "context_window": 128000,
+        "num_output": -1,
+        "is_chat_model": true,
+        "is_function_calling_model": true,
+        "model_name": "gpt-4o",
+        "system_role": "system"
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-05T15:45:55.866Z",
+      "updatedAt": "2025-06-05T15:45:56.021Z",
+      "usageDetails": {
+        "input": 13,
+        "prompt_details.cache_read": 0,
+        "prompt_details.audio": 0,
+        "output": 81,
+        "completion_details.reasoning": 0,
+        "completion_details.audio": 0,
+        "total": 94
+      },
+      "costDetails": {
+        "input": 0.0000325,
+        "output": 0.00081,
+        "total": 0.000842499999
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2776,
+      "timeToFirstToken": null,
+      "inputCost": 0.0000325,
+      "outputCost": 0.00081,
+      "totalCost": 0.000842499999,
+      "inputUsage": 13,
+      "outputUsage": 81,
+      "totalUsage": 94,
+      "input": {
+        "args": [
+          "What is Langfuse?"
+        ]
+      },
+      "output": "Langfuse is a tool designed to help developers monitor and debug applications that utilize large language models (LLMs). It provides features for tracking and analyzing the performance of LLMs, enabling developers to gain insights into how these models are functioning within their applications. Langfuse can be particularly useful for identifying issues, optimizing performance, and ensuring that the integration of language models into applications is smooth and effective."
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/framework-traces/merge-observations.ts

@@ -0,0 +1,67 @@
+import { readFileSync, writeFileSync } from "fs";
+
+/**
+ * Merges detailed observation data (input/output/metadata) into the base trace file.
+ * how to use? SEE README.md
+ *
+ * Usage: ts-node merge-observations.ts <base-file> <detailed-obs-file> <output-file>
+ *
+ * Example:
+ *   ts-node merge-observations.ts pydantic-base.json pydantic-details.json pydantic-ai.json
+ *
+ */
+
+const [baseFile, detailedFile, outputFile] = process.argv.slice(2);
+
+if (!baseFile || !detailedFile || !outputFile) {
+  console.error(
+    "Usage: ts-node merge-observations.ts <base-file> <detailed-obs-file> <output-file>",
+  );
+  process.exit(1);
+}
+
+const baseData = JSON.parse(readFileSync(baseFile, "utf-8"));
+const detailedObs = JSON.parse(readFileSync(detailedFile, "utf-8"));
+
+// Build ID to detailed observation map
+const obsMap = new Map<string, any>();
+for (const obs of Object.values(detailedObs)) {
+  obsMap.set((obs as any).id, obs);
+}
+
+// Merge input/output/metadata into observations array
+const mergedObservations = baseData.observations.map((obs: any) => {
+  const detailed = obsMap.get(obs.id);
+  if (detailed) {
+    return {
+      ...obs,
+      input: detailed.input,
+      output: detailed.output,
+      metadata: detailed.metadata,
+    };
+  }
+  return obs;
+});
+
+// demo project
+const PROJECT_ID = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+// Remove observations from trace, create final structure
+const output = {
+  trace: {
+    ...baseData.trace,
+    projectId: PROJECT_ID,
+  },
+  observations: mergedObservations.map((obs: any) => ({
+    ...obs,
+    projectId: PROJECT_ID,
+  })),
+};
+
+// Remove observations array from trace if it exists
+delete (output.trace as any).observations;
+
+writeFileSync(outputFile, JSON.stringify(output, null, 2));
+
+console.log(`Merged ${mergedObservations.length} observations`);
+console.log(`Wrote to ${outputFile}`);

packages/shared/scripts/seeder/utils/framework-traces/openai-agents-2025-09-30.json

@@ -0,0 +1,931 @@
+{
+  "trace": {
+    "id": "fee618f96dc31e0ca38b2f7b26eb8b29",
+    "projectId": "cloramnkj0002jz088vzn1ja4",
+    "name": "Agent workflow",
+    "timestamp": "2025-09-30T09:03:59.962Z",
+    "environment": "default",
+    "tags": [],
+    "bookmarked": false,
+    "release": null,
+    "version": null,
+    "userId": null,
+    "sessionId": null,
+    "public": true,
+    "input": null,
+    "output": null,
+    "metadata": "{\"attributes\":{\"openinference.span.kind\":\"AGENT\"},\"resourceAttributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.37.0\",\"service.name\":\"unknown_service\"},\"scope\":{\"name\":\"openinference.instrumentation.openai_agents\",\"version\":\"1.3.0\",\"attributes\":{}}}",
+    "createdAt": "2025-09-30T09:04:03.000Z",
+    "updatedAt": "2025-09-30T09:04:07.465Z",
+    "scores": [
+      {
+        "id": "0721bd4f-a35c-4d29-a2c6-38879e1d758e",
+        "timestamp": "2025-09-30T09:04:30.125Z",
+        "projectId": "cloramnkj0002jz088vzn1ja4",
+        "environment": "default",
+        "name": "toxicity",
+        "source": "EVAL",
+        "authorUserId": null,
+        "comment": "The generation contains aggressive and dismissive language that undermines trust in professionals and promotes a negative view of the public's intelligence, which is harmful and disrespectful.",
+        "metadata": "{}",
+        "configId": null,
+        "createdAt": "2025-09-30T09:04:31.000Z",
+        "updatedAt": "2025-09-30T09:04:30.197Z",
+        "queueId": null,
+        "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+        "observationId": null,
+        "sessionId": null,
+        "datasetRunId": null,
+        "value": 0.8,
+        "stringValue": null,
+        "dataType": "NUMERIC"
+      },
+      {
+        "id": "f60dc1c4-17af-4512-ada9-f2440f437ce1",
+        "timestamp": "2025-09-30T09:04:28.687Z",
+        "projectId": "cloramnkj0002jz088vzn1ja4",
+        "environment": "default",
+        "name": "hallucination",
+        "source": "EVAL",
+        "authorUserId": null,
+        "comment": "There is no specific generation provided to evaluate for hallucination. Therefore, it is impossible to assess or score the degree of hallucination in this case.",
+        "metadata": "{}",
+        "configId": null,
+        "createdAt": "2025-09-30T09:04:29.000Z",
+        "updatedAt": "2025-09-30T09:04:28.773Z",
+        "queueId": null,
+        "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+        "observationId": null,
+        "sessionId": null,
+        "datasetRunId": null,
+        "value": 0,
+        "stringValue": null,
+        "dataType": "NUMERIC"
+      }
+    ],
+    "latency": 2.19
+  },
+  "observations": [
+    {
+      "id": "e5a76f27f51ad40e",
+      "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "AGENT",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2025-09-30T09:03:59.962Z",
+      "endTime": "2025-09-30T09:04:02.152Z",
+      "name": "Agent workflow",
+      "metadata": {
+        "attributes": {
+          "openinference.span.kind": "AGENT"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.37.0",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.openai_agents",
+          "version": "1.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-09-30T09:04:07.382Z",
+      "updatedAt": "2025-09-30T09:04:07.382Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2190,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "d950b668796240b5",
+      "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "AGENT",
+      "environment": "default",
+      "parentObservationId": "e5a76f27f51ad40e",
+      "startTime": "2025-09-30T09:03:59.962Z",
+      "endTime": "2025-09-30T09:04:02.152Z",
+      "name": "Hello world",
+      "metadata": {
+        "attributes": {
+          "llm.system": "openai",
+          "graph.node.id": "Hello world",
+          "openinference.span.kind": "AGENT"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.37.0",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.openai_agents",
+          "version": "1.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-09-30T09:04:07.382Z",
+      "updatedAt": "2025-09-30T09:04:07.382Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 2190,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null
+    },
+    {
+      "id": "90d94774e8e3724d",
+      "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "d950b668796240b5",
+      "startTime": "2025-09-30T09:04:01.001Z",
+      "endTime": "2025-09-30T09:04:02.151Z",
+      "name": "response",
+      "metadata": {
+        "attributes": {
+          "llm.system": "openai",
+          "output.mime_type": "application/json",
+          "output.value": {
+            "id": "resp_0f00ca5b7e22bb4c0068db9d0110d8819da676bd1f2dd03a40",
+            "created_at": 1759223041,
+            "error": null,
+            "incomplete_details": null,
+            "instructions": "You are a helpful agent.",
+            "metadata": {},
+            "model": "gpt-4.1-2025-04-14",
+            "object": "response",
+            "output": [
+              {
+                "id": "msg_0f00ca5b7e22bb4c0068db9d019a78819d9fe1e4d3b6c96b68",
+                "content": [
+                  {
+                    "annotations": [],
+                    "text": "The weather in Tokyo is currently sunny. If you need more details like temperature or forecast for the upcoming days, just let me know!",
+                    "type": "output_text",
+                    "logprobs": []
+                  }
+                ],
+                "role": "assistant",
+                "status": "completed",
+                "type": "message"
+              }
+            ],
+            "parallel_tool_calls": true,
+            "temperature": 1,
+            "tool_choice": "auto",
+            "tools": [
+              {
+                "name": "get_weather",
+                "parameters": {
+                  "properties": {
+                    "city": {
+                      "title": "City",
+                      "type": "string"
+                    }
+                  },
+                  "required": ["city"],
+                  "title": "get_weather_args",
+                  "type": "object",
+                  "additionalProperties": false
+                },
+                "strict": true,
+                "type": "function",
+                "description": null
+              }
+            ],
+            "top_p": 1,
+            "background": false,
+            "conversation": null,
+            "max_output_tokens": null,
+            "max_tool_calls": null,
+            "previous_response_id": null,
+            "prompt": null,
+            "prompt_cache_key": null,
+            "reasoning": {
+              "effort": null,
+              "generate_summary": null,
+              "summary": null
+            },
+            "safety_identifier": null,
+            "service_tier": "default",
+            "status": "completed",
+            "text": {
+              "format": {
+                "type": "text"
+              },
+              "verbosity": "medium"
+            },
+            "top_logprobs": 0,
+            "truncation": "disabled",
+            "usage": {
+              "input_tokens": 85,
+              "input_tokens_details": {
+                "cached_tokens": 0
+              },
+              "output_tokens": 29,
+              "output_tokens_details": {
+                "reasoning_tokens": 0
+              },
+              "total_tokens": 114
+            },
+            "user": null,
+            "billing": {
+              "payer": "developer"
+            },
+            "store": true
+          },
+          "llm.tools.0.tool.json_schema": {
+            "type": "function",
+            "function": {
+              "name": "get_weather",
+              "description": null,
+              "parameters": {
+                "properties": {
+                  "city": {
+                    "title": "City",
+                    "type": "string"
+                  }
+                },
+                "required": ["city"],
+                "title": "get_weather_args",
+                "type": "object",
+                "additionalProperties": false
+              },
+              "strict": true
+            }
+          },
+          "llm.token_count.completion": "29",
+          "llm.token_count.prompt": "85",
+          "llm.token_count.total": "114",
+          "llm.token_count.prompt_details.cache_read": "0",
+          "llm.token_count.completion_details.reasoning": "0",
+          "llm.output_messages.0.message.role": "assistant",
+          "llm.output_messages.0.message.contents.0.message_content.type": "text",
+          "llm.output_messages.0.message.contents.0.message_content.text": "The weather in Tokyo is currently sunny. If you need more details like temperature or forecast for the upcoming days, just let me know!",
+          "llm.input_messages.0.message.role": "system",
+          "llm.input_messages.0.message.content": "You are a helpful agent.",
+          "llm.model_name": "gpt-4.1-2025-04-14",
+          "llm.invocation_parameters": {
+            "id": "resp_0f00ca5b7e22bb4c0068db9d0110d8819da676bd1f2dd03a40",
+            "created_at": 1759223041,
+            "instructions": "You are a helpful agent.",
+            "metadata": {},
+            "model": "gpt-4.1-2025-04-14",
+            "parallel_tool_calls": true,
+            "temperature": 1,
+            "tool_choice": "auto",
+            "top_p": 1,
+            "background": false,
+            "reasoning": {},
+            "service_tier": "default",
+            "text": {
+              "format": {
+                "type": "text"
+              },
+              "verbosity": "medium"
+            },
+            "top_logprobs": 0,
+            "truncation": "disabled",
+            "billing": {
+              "payer": "developer"
+            },
+            "store": true
+          },
+          "input.mime_type": "application/json",
+          "input.value": [
+            {
+              "content": "What's the weather in Tokyo?",
+              "role": "user"
+            },
+            {
+              "arguments": {
+                "city": "Tokyo"
+              },
+              "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+              "name": "get_weather",
+              "type": "function_call",
+              "id": "fc_0f00ca5b7e22bb4c0068db9d00d258819d987f4b9b63189e76",
+              "status": "completed"
+            },
+            {
+              "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+              "output": "The weather in Tokyo is sunny.",
+              "type": "function_call_output"
+            }
+          ],
+          "llm.input_messages.1.message.role": "user",
+          "llm.input_messages.1.message.content": "What's the weather in Tokyo?",
+          "llm.input_messages.2.message.role": "assistant",
+          "llm.input_messages.2.message.tool_calls.0.tool_call.id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+          "llm.input_messages.2.message.tool_calls.0.tool_call.function.name": "get_weather",
+          "llm.input_messages.2.message.tool_calls.0.tool_call.function.arguments": {
+            "city": "Tokyo"
+          },
+          "llm.input_messages.3.message.role": "tool",
+          "llm.input_messages.3.message.tool_call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+          "llm.input_messages.3.message.content": "The weather in Tokyo is sunny.",
+          "openinference.span.kind": "LLM"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.37.0",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.openai_agents",
+          "version": "1.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {
+        "id": "resp_0f00ca5b7e22bb4c0068db9d0110d8819da676bd1f2dd03a40",
+        "created_at": 1759223041,
+        "instructions": "You are a helpful agent.",
+        "metadata": "{}",
+        "model": "gpt-4.1-2025-04-14",
+        "parallel_tool_calls": "true",
+        "temperature": 1,
+        "tool_choice": "auto",
+        "top_p": 1,
+        "background": "false",
+        "reasoning": "{}",
+        "service_tier": "default",
+        "text": "{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"}",
+        "top_logprobs": 0,
+        "truncation": "disabled",
+        "billing": "{\"payer\":\"developer\"}",
+        "store": "true"
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-09-30T09:04:07.382Z",
+      "updatedAt": "2025-09-30T09:04:07.382Z",
+      "usageDetails": {
+        "output": 29,
+        "input": 85,
+        "total": 114,
+        "prompt_details.cache_read": 0,
+        "completion_details.reasoning": 0
+      },
+      "costDetails": {
+        "output": 0.000232,
+        "input": 0.00017,
+        "total": 0.000402
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4.1-2025-04-14",
+      "internalModelId": "cm7qahw732891bpmzy45r3x70",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1150,
+      "timeToFirstToken": null,
+      "inputCost": 0.00017,
+      "outputCost": 0.000232,
+      "totalCost": 0.000402,
+      "inputUsage": 85,
+      "outputUsage": 29,
+      "totalUsage": 114,
+      "input": [
+        {
+          "content": "What's the weather in Tokyo?",
+          "role": "user"
+        },
+        {
+          "arguments": {
+            "city": "Tokyo"
+          },
+          "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+          "name": "get_weather",
+          "type": "function_call",
+          "id": "fc_0f00ca5b7e22bb4c0068db9d00d258819d987f4b9b63189e76",
+          "status": "completed"
+        },
+        {
+          "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+          "output": "The weather in Tokyo is sunny.",
+          "type": "function_call_output"
+        }
+      ],
+      "output": {
+        "id": "resp_0f00ca5b7e22bb4c0068db9d0110d8819da676bd1f2dd03a40",
+        "created_at": 1759223041,
+        "error": null,
+        "incomplete_details": null,
+        "instructions": "You are a helpful agent.",
+        "metadata": {},
+        "model": "gpt-4.1-2025-04-14",
+        "object": "response",
+        "output": [
+          {
+            "id": "msg_0f00ca5b7e22bb4c0068db9d019a78819d9fe1e4d3b6c96b68",
+            "content": [
+              {
+                "annotations": [],
+                "text": "The weather in Tokyo is currently sunny. If you need more details like temperature or forecast for the upcoming days, just let me know!",
+                "type": "output_text",
+                "logprobs": []
+              }
+            ],
+            "role": "assistant",
+            "status": "completed",
+            "type": "message"
+          }
+        ],
+        "parallel_tool_calls": true,
+        "temperature": 1,
+        "tool_choice": "auto",
+        "tools": [
+          {
+            "name": "get_weather",
+            "parameters": {
+              "properties": {
+                "city": {
+                  "title": "City",
+                  "type": "string"
+                }
+              },
+              "required": ["city"],
+              "title": "get_weather_args",
+              "type": "object",
+              "additionalProperties": false
+            },
+            "strict": true,
+            "type": "function",
+            "description": null
+          }
+        ],
+        "top_p": 1,
+        "background": false,
+        "conversation": null,
+        "max_output_tokens": null,
+        "max_tool_calls": null,
+        "previous_response_id": null,
+        "prompt": null,
+        "prompt_cache_key": null,
+        "reasoning": {
+          "effort": null,
+          "generate_summary": null,
+          "summary": null
+        },
+        "safety_identifier": null,
+        "service_tier": "default",
+        "status": "completed",
+        "text": {
+          "format": {
+            "type": "text"
+          },
+          "verbosity": "medium"
+        },
+        "top_logprobs": 0,
+        "truncation": "disabled",
+        "usage": {
+          "input_tokens": 85,
+          "input_tokens_details": {
+            "cached_tokens": 0
+          },
+          "output_tokens": 29,
+          "output_tokens_details": {
+            "reasoning_tokens": 0
+          },
+          "total_tokens": 114
+        },
+        "user": null,
+        "billing": {
+          "payer": "developer"
+        },
+        "store": true
+      }
+    },
+    {
+      "id": "98870087af69bf06",
+      "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "d950b668796240b5",
+      "startTime": "2025-09-30T09:03:59.963Z",
+      "endTime": "2025-09-30T09:04:00.999Z",
+      "name": "response",
+      "metadata": {
+        "attributes": {
+          "llm.system": "openai",
+          "output.mime_type": "application/json",
+          "output.value": {
+            "id": "resp_0f00ca5b7e22bb4c0068db9d000be4819d824c447c54b65bc1",
+            "created_at": 1759223040,
+            "error": null,
+            "incomplete_details": null,
+            "instructions": "You are a helpful agent.",
+            "metadata": {},
+            "model": "gpt-4.1-2025-04-14",
+            "object": "response",
+            "output": [
+              {
+                "arguments": {
+                  "city": "Tokyo"
+                },
+                "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+                "name": "get_weather",
+                "type": "function_call",
+                "id": "fc_0f00ca5b7e22bb4c0068db9d00d258819d987f4b9b63189e76",
+                "status": "completed"
+              }
+            ],
+            "parallel_tool_calls": true,
+            "temperature": 1,
+            "tool_choice": "auto",
+            "tools": [
+              {
+                "name": "get_weather",
+                "parameters": {
+                  "properties": {
+                    "city": {
+                      "title": "City",
+                      "type": "string"
+                    }
+                  },
+                  "required": ["city"],
+                  "title": "get_weather_args",
+                  "type": "object",
+                  "additionalProperties": false
+                },
+                "strict": true,
+                "type": "function",
+                "description": null
+              }
+            ],
+            "top_p": 1,
+            "background": false,
+            "conversation": null,
+            "max_output_tokens": null,
+            "max_tool_calls": null,
+            "previous_response_id": null,
+            "prompt": null,
+            "prompt_cache_key": null,
+            "reasoning": {
+              "effort": null,
+              "generate_summary": null,
+              "summary": null
+            },
+            "safety_identifier": null,
+            "service_tier": "default",
+            "status": "completed",
+            "text": {
+              "format": {
+                "type": "text"
+              },
+              "verbosity": "medium"
+            },
+            "top_logprobs": 0,
+            "truncation": "disabled",
+            "usage": {
+              "input_tokens": 55,
+              "input_tokens_details": {
+                "cached_tokens": 0
+              },
+              "output_tokens": 15,
+              "output_tokens_details": {
+                "reasoning_tokens": 0
+              },
+              "total_tokens": 70
+            },
+            "user": null,
+            "billing": {
+              "payer": "developer"
+            },
+            "store": true
+          },
+          "llm.tools.0.tool.json_schema": {
+            "type": "function",
+            "function": {
+              "name": "get_weather",
+              "description": null,
+              "parameters": {
+                "properties": {
+                  "city": {
+                    "title": "City",
+                    "type": "string"
+                  }
+                },
+                "required": ["city"],
+                "title": "get_weather_args",
+                "type": "object",
+                "additionalProperties": false
+              },
+              "strict": true
+            }
+          },
+          "llm.token_count.completion": "15",
+          "llm.token_count.prompt": "55",
+          "llm.token_count.total": "70",
+          "llm.token_count.prompt_details.cache_read": "0",
+          "llm.token_count.completion_details.reasoning": "0",
+          "llm.output_messages.0.message.role": "assistant",
+          "llm.output_messages.0.message.tool_calls.0.tool_call.id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+          "llm.output_messages.0.message.tool_calls.0.tool_call.function.name": "get_weather",
+          "llm.output_messages.0.message.tool_calls.0.tool_call.function.arguments": {
+            "city": "Tokyo"
+          },
+          "llm.input_messages.0.message.role": "system",
+          "llm.input_messages.0.message.content": "You are a helpful agent.",
+          "llm.model_name": "gpt-4.1-2025-04-14",
+          "llm.invocation_parameters": {
+            "id": "resp_0f00ca5b7e22bb4c0068db9d000be4819d824c447c54b65bc1",
+            "created_at": 1759223040,
+            "instructions": "You are a helpful agent.",
+            "metadata": {},
+            "model": "gpt-4.1-2025-04-14",
+            "parallel_tool_calls": true,
+            "temperature": 1,
+            "tool_choice": "auto",
+            "top_p": 1,
+            "background": false,
+            "reasoning": {},
+            "service_tier": "default",
+            "text": {
+              "format": {
+                "type": "text"
+              },
+              "verbosity": "medium"
+            },
+            "top_logprobs": 0,
+            "truncation": "disabled",
+            "billing": {
+              "payer": "developer"
+            },
+            "store": true
+          },
+          "input.mime_type": "application/json",
+          "input.value": [
+            {
+              "content": "What's the weather in Tokyo?",
+              "role": "user"
+            }
+          ],
+          "llm.input_messages.1.message.role": "user",
+          "llm.input_messages.1.message.content": "What's the weather in Tokyo?",
+          "openinference.span.kind": "LLM"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.37.0",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.openai_agents",
+          "version": "1.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {
+        "id": "resp_0f00ca5b7e22bb4c0068db9d000be4819d824c447c54b65bc1",
+        "created_at": 1759223040,
+        "instructions": "You are a helpful agent.",
+        "metadata": "{}",
+        "model": "gpt-4.1-2025-04-14",
+        "parallel_tool_calls": "true",
+        "temperature": 1,
+        "tool_choice": "auto",
+        "top_p": 1,
+        "background": "false",
+        "reasoning": "{}",
+        "service_tier": "default",
+        "text": "{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"}",
+        "top_logprobs": 0,
+        "truncation": "disabled",
+        "billing": "{\"payer\":\"developer\"}",
+        "store": "true"
+      },
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-09-30T09:04:02.217Z",
+      "updatedAt": "2025-09-30T09:04:02.217Z",
+      "usageDetails": {
+        "output": 15,
+        "input": 55,
+        "total": 70,
+        "prompt_details.cache_read": 0,
+        "completion_details.reasoning": 0
+      },
+      "costDetails": {
+        "output": 0.00012,
+        "input": 0.00011,
+        "total": 0.00023
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4.1-2025-04-14",
+      "internalModelId": "cm7qahw732891bpmzy45r3x70",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1036,
+      "timeToFirstToken": null,
+      "inputCost": 0.00011,
+      "outputCost": 0.00012,
+      "totalCost": 0.00023,
+      "inputUsage": 55,
+      "outputUsage": 15,
+      "totalUsage": 70,
+      "input": [
+        {
+          "content": "What's the weather in Tokyo?",
+          "role": "user"
+        }
+      ],
+      "output": {
+        "id": "resp_0f00ca5b7e22bb4c0068db9d000be4819d824c447c54b65bc1",
+        "created_at": 1759223040,
+        "error": null,
+        "incomplete_details": null,
+        "instructions": "You are a helpful agent.",
+        "metadata": {},
+        "model": "gpt-4.1-2025-04-14",
+        "object": "response",
+        "output": [
+          {
+            "arguments": {
+              "city": "Tokyo"
+            },
+            "call_id": "call_Kud0j0DxWSmLzv9W5m6qVqXn",
+            "name": "get_weather",
+            "type": "function_call",
+            "id": "fc_0f00ca5b7e22bb4c0068db9d00d258819d987f4b9b63189e76",
+            "status": "completed"
+          }
+        ],
+        "parallel_tool_calls": true,
+        "temperature": 1,
+        "tool_choice": "auto",
+        "tools": [
+          {
+            "name": "get_weather",
+            "parameters": {
+              "properties": {
+                "city": {
+                  "title": "City",
+                  "type": "string"
+                }
+              },
+              "required": ["city"],
+              "title": "get_weather_args",
+              "type": "object",
+              "additionalProperties": false
+            },
+            "strict": true,
+            "type": "function",
+            "description": null
+          }
+        ],
+        "top_p": 1,
+        "background": false,
+        "conversation": null,
+        "max_output_tokens": null,
+        "max_tool_calls": null,
+        "previous_response_id": null,
+        "prompt": null,
+        "prompt_cache_key": null,
+        "reasoning": {
+          "effort": null,
+          "generate_summary": null,
+          "summary": null
+        },
+        "safety_identifier": null,
+        "service_tier": "default",
+        "status": "completed",
+        "text": {
+          "format": {
+            "type": "text"
+          },
+          "verbosity": "medium"
+        },
+        "top_logprobs": 0,
+        "truncation": "disabled",
+        "usage": {
+          "input_tokens": 55,
+          "input_tokens_details": {
+            "cached_tokens": 0
+          },
+          "output_tokens": 15,
+          "output_tokens_details": {
+            "reasoning_tokens": 0
+          },
+          "total_tokens": 70
+        },
+        "user": null,
+        "billing": {
+          "payer": "developer"
+        },
+        "store": true
+      }
+    },
+    {
+      "id": "5b99c3e3411ed17b",
+      "traceId": "fee618f96dc31e0ca38b2f7b26eb8b29",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "TOOL",
+      "environment": "default",
+      "parentObservationId": "d950b668796240b5",
+      "startTime": "2025-09-30T09:04:01.000Z",
+      "endTime": "2025-09-30T09:04:01.000Z",
+      "name": "get_weather",
+      "metadata": {
+        "attributes": {
+          "llm.system": "openai",
+          "tool.name": "get_weather",
+          "input.value": {
+            "city": "Tokyo"
+          },
+          "input.mime_type": "application/json",
+          "output.value": "The weather in Tokyo is sunny.",
+          "openinference.span.kind": "TOOL"
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.37.0",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "openinference.instrumentation.openai_agents",
+          "version": "1.3.0",
+          "attributes": {}
+        }
+      },
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-09-30T09:04:02.217Z",
+      "updatedAt": "2025-09-30T09:04:02.217Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 0,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "city": "Tokyo"
+      },
+      "output": "The weather in Tokyo is sunny."
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/framework-traces/openai-assistants-2024-05-29.json

@@ -0,0 +1,338 @@
+{
+  "trace": {
+    "id": "b3b7b128-5664-4f42-9fab-31999da9e2f1",
+    "projectId": "cloramnkj0002jz088vzn1ja4",
+    "name": "run_math_tutor",
+    "timestamp": "2024-05-29T13:46:10.728Z",
+    "environment": "default",
+    "tags": [],
+    "bookmarked": false,
+    "release": null,
+    "version": null,
+    "userId": null,
+    "sessionId": null,
+    "public": true,
+    "input": "\"{\\\"args\\\":[\\\"I need to solve the equation `3x + 11 = 14`. Can you help me?\\\"],\\\"kwargs\\\":{}}\"",
+    "output": "\"\\\"Sure, first subtract 11 from both sides to get `3x = 3`, then divide both sides by 3 to solve for `x`. The solution is `x = 1`.\\\"\"",
+    "metadata": "{}",
+    "createdAt": "2024-05-29T13:46:11.304Z",
+    "updatedAt": "2024-05-29T13:46:34.171Z",
+    "latency": 9.236
+  },
+  "observations": [
+    {
+      "id": "0d8f5427-c91a-44c8-86c5-e12d806edd57",
+      "traceId": "b3b7b128-5664-4f42-9fab-31999da9e2f1",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "0d6c73e1-be16-436d-a905-b4108912fa93",
+      "startTime": "2024-05-29T13:46:19.963Z",
+      "endTime": null,
+      "name": "",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2024-05-29T13:46:20.714Z",
+      "updatedAt": "2024-05-29T13:46:20.714Z",
+      "usageDetails": {
+        "input": 49,
+        "output": 40,
+        "total": 89
+      },
+      "costDetails": {
+        "input": 0.00147,
+        "output": 0.0024,
+        "total": 0.00387
+      },
+      "providedCostDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "model": "gpt-4",
+      "internalModelId": "clrntkjgy000f08jx79v9g1xj",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": null,
+      "timeToFirstToken": null,
+      "inputCost": 0.00147,
+      "outputCost": 0.0024,
+      "totalCost": 0.00387,
+      "inputUsage": 49,
+      "outputUsage": 40,
+      "totalUsage": 89,
+      "input": [
+        {
+          "role": "assistant",
+          "content": "I am a math tutor that likes to help math students, how can I help?"
+        },
+        {
+          "role": "user",
+          "content": "I need to solve the equation `3x + 11 = 14`. Can you help me?"
+        }
+      ],
+      "output": "Sure, first subtract 11 from both sides to get `3x = 3`, then divide both sides by 3 to solve for `x`. The solution is `x = 1`.",
+      "metadata": {}
+    },
+    {
+      "id": "0d6c73e1-be16-436d-a905-b4108912fa93",
+      "traceId": "b3b7b128-5664-4f42-9fab-31999da9e2f1",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2024-05-29T13:46:17.971Z",
+      "endTime": "2024-05-29T13:46:19.964Z",
+      "name": "get_response",
+      "metadata": "{}",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2024-05-29T13:46:18.345Z",
+      "updatedAt": "2024-05-29T13:46:20.796Z",
+      "usageDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "costDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "providedCostDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1993,
+      "timeToFirstToken": null,
+      "inputCost": 0,
+      "outputCost": 0,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "args": [
+          "thread_fbFB5caAj0sEpE1c5ICMDs7I",
+          "run_RsJmiMTrgkBzb3FvyCvRoqpd"
+        ],
+        "kwargs": {}
+      },
+      "output": [
+        "Sure, first subtract 11 from both sides to get `3x = 3`, then divide both sides by 3 to solve for `x`. The solution is `x = 1`.",
+        {
+          "id": "run_AherTwpS29w0GC8zlKDBhhlZ",
+          "model": "gpt-4",
+          "tools": [],
+          "top_p": 1,
+          "usage": null,
+          "object": "thread.run",
+          "status": "queued",
+          "metadata": {},
+          "failed_at": null,
+          "thread_id": "thread_DhsY59BBhtKWN8YYq44cULrH",
+          "created_at": 1716990330,
+          "expires_at": 1716990930,
+          "last_error": null,
+          "started_at": null,
+          "temperature": 1,
+          "tool_choice": "auto",
+          "assistant_id": "asst_tjgFtRASnPoQaWpks1ezPfSp",
+          "cancelled_at": null,
+          "completed_at": null,
+          "instructions": "You are a personal math tutor. Answer questions briefly, in a sentence or less.",
+          "tool_resources": {},
+          "required_action": null,
+          "response_format": "auto",
+          "max_prompt_tokens": null,
+          "incomplete_details": null,
+          "truncation_strategy": {
+            "type": "auto",
+            "last_messages": null
+          },
+          "max_completion_tokens": null
+        }
+      ]
+    },
+    {
+      "id": "e584f95b-e8ef-4d43-bcb1-fd296f57f94b",
+      "traceId": "b3b7b128-5664-4f42-9fab-31999da9e2f1",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2024-05-29T13:46:11.057Z",
+      "endTime": "2024-05-29T13:46:12.959Z",
+      "name": "run_assistant",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2024-05-29T13:46:11.480Z",
+      "updatedAt": "2024-05-29T13:46:13.161Z",
+      "usageDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "costDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "providedCostDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 1902,
+      "timeToFirstToken": null,
+      "inputCost": 0,
+      "outputCost": 0,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "args": [
+          "asst_01wKwjx3rT7CNprPxzE8azMl",
+          "I need to solve the equation `3x + 11 = 14`. Can you help me?"
+        ],
+        "kwargs": {}
+      },
+      "output": [
+        {
+          "id": "run_RsJmiMTrgkBzb3FvyCvRoqpd",
+          "model": "gpt-4",
+          "tools": [],
+          "top_p": 1,
+          "usage": null,
+          "object": "thread.run",
+          "status": "queued",
+          "metadata": {},
+          "failed_at": null,
+          "thread_id": "thread_fbFB5caAj0sEpE1c5ICMDs7I",
+          "created_at": 1716990372,
+          "expires_at": 1716990972,
+          "last_error": null,
+          "started_at": null,
+          "temperature": 1,
+          "tool_choice": "auto",
+          "assistant_id": "asst_01wKwjx3rT7CNprPxzE8azMl",
+          "cancelled_at": null,
+          "completed_at": null,
+          "instructions": "You are a personal math tutor. Answer questions briefly, in a sentence or less.",
+          "tool_resources": {},
+          "required_action": null,
+          "response_format": "auto",
+          "max_prompt_tokens": null,
+          "incomplete_details": null,
+          "truncation_strategy": {
+            "type": "auto",
+            "last_messages": null
+          },
+          "max_completion_tokens": null
+        },
+        {
+          "id": "thread_fbFB5caAj0sEpE1c5ICMDs7I",
+          "object": "thread",
+          "metadata": {},
+          "created_at": 1716990371,
+          "tool_resources": {
+            "file_search": null,
+            "code_interpreter": null
+          }
+        }
+      ],
+      "metadata": {}
+    },
+    {
+      "id": "be59b862-98dd-4b0e-b92e-c150b9777c2f",
+      "traceId": "b3b7b128-5664-4f42-9fab-31999da9e2f1",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2024-05-29T13:46:10.728Z",
+      "endTime": "2024-05-29T13:46:11.056Z",
+      "name": "create_assistant",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2024-05-29T13:46:11.350Z",
+      "updatedAt": "2024-05-29T13:46:11.405Z",
+      "usageDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "costDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "providedCostDetails": {
+        "input": 0,
+        "output": 0,
+        "total": 0
+      },
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 328,
+      "timeToFirstToken": null,
+      "inputCost": 0,
+      "outputCost": 0,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": {
+        "args": [],
+        "kwargs": {}
+      },
+      "output": {
+        "id": "asst_01wKwjx3rT7CNprPxzE8azMl",
+        "name": "Math Tutor",
+        "model": "gpt-4",
+        "tools": [],
+        "top_p": 1,
+        "object": "assistant",
+        "metadata": {},
+        "created_at": 1716990370,
+        "description": null,
+        "temperature": 1,
+        "instructions": "You are a personal math tutor. Answer questions briefly, in a sentence or less.",
+        "tool_resources": {
+          "file_search": null,
+          "code_interpreter": null
+        },
+        "response_format": "auto"
+      },
+      "metadata": {}
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/framework-traces/pydantic-ai-trace-2025-06-06.json

@@ -0,0 +1,312 @@
+{
+  "trace": {
+    "id": "25f4bdeebaab60e6e1bee7e8469554bc",
+    "projectId": "cloramnkj0002jz088vzn1ja4",
+    "name": "pydantic-ai-qa-trace",
+    "timestamp": "2025-06-06T14:40:28.562Z",
+    "environment": "default",
+    "tags": ["dev", "pydantic-ai"],
+    "bookmarked": false,
+    "release": null,
+    "version": "1.0.0",
+    "userId": "user_123",
+    "sessionId": "session_abc",
+    "public": true,
+    "input": "\"What is Langfuse?\"",
+    "output": "\"Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions.\"",
+    "metadata": "{\"email\":\"user@langfuse.com\",\"resourceAttributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.33.1\",\"service.name\":\"unknown_service\"},\"scope\":{\"name\":\"langfuse-sdk\",\"version\":\"3.0.0\",\"attributes\":{\"public_key\":\"pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba\"}}}",
+    "createdAt": "2025-06-06T14:40:34.000Z",
+    "updatedAt": "2025-06-06T14:40:33.635Z",
+    "latency": 4.795
+  },
+  "observations": [
+    {
+      "id": "538fed87d11686d3",
+      "traceId": "25f4bdeebaab60e6e1bee7e8469554bc",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": null,
+      "startTime": "2025-06-06T14:40:28.562Z",
+      "endTime": "2025-06-06T14:40:33.357Z",
+      "name": "pydantic-ai-qa-trace",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": "1.0.0",
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-06T14:40:33.595Z",
+      "updatedAt": "2025-06-06T14:40:33.630Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 4795,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": null,
+      "metadata": {
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "langfuse-sdk",
+          "version": "3.0.0",
+          "attributes": {
+            "public_key": "pk-lf-5855d85e-3943-497e-bd10-f50ad414bcba"
+          }
+        }
+      }
+    },
+    {
+      "id": "0a4fac2553f22f2e",
+      "traceId": "25f4bdeebaab60e6e1bee7e8469554bc",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "SPAN",
+      "environment": "default",
+      "parentObservationId": "538fed87d11686d3",
+      "startTime": "2025-06-06T14:40:28.562Z",
+      "endTime": "2025-06-06T14:40:33.357Z",
+      "name": "qa_agent run",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": null,
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-06T14:40:33.596Z",
+      "updatedAt": "2025-06-06T14:40:33.641Z",
+      "usageDetails": {},
+      "costDetails": {},
+      "providedCostDetails": {},
+      "model": null,
+      "internalModelId": null,
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 4795,
+      "timeToFirstToken": null,
+      "inputCost": null,
+      "outputCost": null,
+      "totalCost": 0,
+      "inputUsage": 0,
+      "outputUsage": 0,
+      "totalUsage": 0,
+      "input": null,
+      "output": [
+        {
+          "content": "You are a helpful assistant that answers questions clearly and concisely.",
+          "role": "system",
+          "gen_ai.message.index": 0,
+          "event.name": "gen_ai.system.message"
+        },
+        {
+          "content": "What is Langfuse?",
+          "role": "user",
+          "gen_ai.message.index": 0,
+          "event.name": "gen_ai.user.message"
+        },
+        {
+          "role": "assistant",
+          "content": "Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions.",
+          "gen_ai.message.index": 1,
+          "event.name": "gen_ai.assistant.message"
+        }
+      ],
+      "metadata": {
+        "attributes": {
+          "model_name": "gpt-4o",
+          "agent_name": "qa_agent",
+          "logfire.msg": "qa_agent run",
+          "final_result": "Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions.",
+          "gen_ai.usage.input_tokens": "31",
+          "gen_ai.usage.output_tokens": "96",
+          "all_messages_events": [
+            {
+              "content": "You are a helpful assistant that answers questions clearly and concisely.",
+              "role": "system",
+              "gen_ai.message.index": 0,
+              "event.name": "gen_ai.system.message"
+            },
+            {
+              "content": "What is Langfuse?",
+              "role": "user",
+              "gen_ai.message.index": 0,
+              "event.name": "gen_ai.user.message"
+            },
+            {
+              "role": "assistant",
+              "content": "Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions.",
+              "gen_ai.message.index": 1,
+              "event.name": "gen_ai.assistant.message"
+            }
+          ],
+          "logfire.json_schema": {
+            "type": "object",
+            "properties": {
+              "all_messages_events": {
+                "type": "array"
+              },
+              "final_result": {
+                "type": "object"
+              }
+            }
+          }
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "pydantic-ai",
+          "version": "0.2.15",
+          "attributes": {}
+        }
+      }
+    },
+    {
+      "id": "5cb389a3bdd21a6e",
+      "traceId": "25f4bdeebaab60e6e1bee7e8469554bc",
+      "projectId": "cloramnkj0002jz088vzn1ja4",
+      "type": "GENERATION",
+      "environment": "default",
+      "parentObservationId": "0a4fac2553f22f2e",
+      "startTime": "2025-06-06T14:40:28.562Z",
+      "endTime": "2025-06-06T14:40:33.356Z",
+      "name": "chat gpt-4o",
+      "level": "DEFAULT",
+      "statusMessage": null,
+      "version": null,
+      "modelParameters": {},
+      "completionStartTime": null,
+      "promptId": null,
+      "createdAt": "2025-06-06T14:40:33.440Z",
+      "updatedAt": "2025-06-06T14:40:33.478Z",
+      "usageDetails": {
+        "input": 31,
+        "output": 96,
+        "total": 127
+      },
+      "costDetails": {
+        "input": 0.0000775,
+        "output": 0.00096,
+        "total": 0.0010375
+      },
+      "providedCostDetails": {},
+      "model": "gpt-4o",
+      "internalModelId": "b9854a5c92dc496b997d99d20",
+      "promptName": null,
+      "promptVersion": null,
+      "latency": 4794,
+      "timeToFirstToken": null,
+      "inputCost": 0.0000775,
+      "outputCost": 0.00096,
+      "totalCost": 0.0010375,
+      "inputUsage": 31,
+      "outputUsage": 96,
+      "totalUsage": 127,
+      "input": [
+        {
+          "content": "You are a helpful assistant that answers questions clearly and concisely.",
+          "role": "system",
+          "gen_ai.system": "openai",
+          "gen_ai.message.index": 0,
+          "event.name": "gen_ai.system.message"
+        },
+        {
+          "content": "What is Langfuse?",
+          "role": "user",
+          "gen_ai.system": "openai",
+          "gen_ai.message.index": 0,
+          "event.name": "gen_ai.user.message"
+        }
+      ],
+      "output": {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": "Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions."
+        },
+        "gen_ai.system": "openai",
+        "event.name": "gen_ai.choice"
+      },
+      "metadata": {
+        "attributes": {
+          "gen_ai.operation.name": "chat",
+          "gen_ai.system": "openai",
+          "gen_ai.request.model": "gpt-4o",
+          "server.address": "api.openai.com",
+          "model_request_parameters": {
+            "function_tools": [],
+            "allow_text_output": true,
+            "output_tools": []
+          },
+          "gen_ai.usage.input_tokens": "31",
+          "gen_ai.usage.output_tokens": "96",
+          "gen_ai.response.model": "gpt-4o-2024-08-06",
+          "events": [
+            {
+              "content": "You are a helpful assistant that answers questions clearly and concisely.",
+              "role": "system",
+              "gen_ai.system": "openai",
+              "gen_ai.message.index": 0,
+              "event.name": "gen_ai.system.message"
+            },
+            {
+              "content": "What is Langfuse?",
+              "role": "user",
+              "gen_ai.system": "openai",
+              "gen_ai.message.index": 0,
+              "event.name": "gen_ai.user.message"
+            },
+            {
+              "index": 0,
+              "message": {
+                "role": "assistant",
+                "content": "Langfuse is a tool designed for observability and monitoring of applications that utilize large language models (LLMs). It provides features to trace, log, and visualize requests, helping developers understand the behavior of their LLM-powered applications more effectively. This can be particularly useful for debugging, performance optimization, and ensuring reliable operation of systems that depend on LLMs. Langfuse is designed to integrate easily with existing infrastructure, supporting both self-hosted implementations and cloud-based solutions."
+              },
+              "gen_ai.system": "openai",
+              "event.name": "gen_ai.choice"
+            }
+          ],
+          "logfire.json_schema": {
+            "type": "object",
+            "properties": {
+              "events": {
+                "type": "array"
+              },
+              "model_request_parameters": {
+                "type": "object"
+              }
+            }
+          }
+        },
+        "resourceAttributes": {
+          "telemetry.sdk.language": "python",
+          "telemetry.sdk.name": "opentelemetry",
+          "telemetry.sdk.version": "1.33.1",
+          "service.name": "unknown_service"
+        },
+        "scope": {
+          "name": "pydantic-ai",
+          "version": "0.2.15",
+          "attributes": {}
+        }
+      }
+    }
+  ]
+}

packages/shared/scripts/seeder/utils/postgres-seed-constants.ts

@@ -400,6 +400,177 @@ export const SEED_CHAT_ML_PROMPTS = [
 ];
 
 export const SEED_PROMPT_VERSIONS = [
+  {
+    createdBy: "user-1",
+    type: "chat",
+    prompt: [
+      {
+        role: "system",
+        content: `## Role
+You are a Langfuse filter generator. Your sole function is to parse user queries about AI traces and output the corresponding filter array in JSON format. Map natural language to appropriate column names, operators, and values.
+
+## Available columns and their types:
+- bookmarked (boolean): Starred/bookmarked traces
+- id (string): Trace ID
+- name (stringOptions): Trace name
+- environment (stringOptions): Environment (dev, prod, etc.)
+- timestamp (datetime): When the trace occurred
+- userId (string): User identifier
+- sessionId (string): Session identifier
+- metadata (stringObject): Custom metadata
+- version (string): Version identifier
+- release (string): Release identifier
+- level (stringOptions): Log level - DEBUG, DEFAULT, WARNING, ERROR
+- tags (arrayOptions): Custom tags
+- inputTokens (number): Input token count
+- outputTokens (number): Output token count
+- totalTokens (number): Total token count
+- tokens (number): Alias for total tokens
+- errorCount (number): Count of error-level observations
+- warningCount (number): Count of warning-level observations
+- defaultCount (number): Count of default-level observations
+- debugCount (number): Count of debug-level observations
+- scores_avg (numberObject): Numeric evaluation scores
+- score_categories (categoryOptions): Categorical evaluation scores
+- latency (number): Latency in seconds
+- inputCost (number): Input cost in USD
+- outputCost (number): Output cost in USD
+- totalCost (number): Total cost in USD
+
+## Type-Operator Compatibility Rules
+
+**string type** - Use for text matching operations:
+- Operators: "=", "contains", "does not contain", "starts with", "ends with"
+- Value: Always a single string
+- Example: {"type": "string", "column": "userId", "operator": "contains", "value": "john"}
+
+**stringOptions type** - Use ONLY for selecting from predefined options:
+- Operators: "any of", "none of" ONLY
+- Value: Always an array of strings
+- Use when filtering columns like name, environment, level with multiple possible values
+- Example: {"type": "stringOptions", "column": "environment", "operator": "any of", "value": ["dev", "staging"]}
+
+**arrayOptions type** - Use for tag filtering:
+- Operators: "any of", "none of", "all of"
+- Value: Always an array of strings
+- Example: {"type": "arrayOptions", "column": "tags", "operator": "any of", "value": ["important", "bug"]}
+
+**number type** - Use for numeric comparisons:
+- Operators: "=", ">", "<", ">=", "<="
+- Value: Always a number
+- Example: {"type": "number", "column": "latency", "operator": ">", "value": 2.5}
+
+**datetime type** - Use for time-based filtering:
+- Operators: ">", "<", ">=", "<="
+- Value: Always a date string
+- Example: {"type": "datetime", "column": "timestamp", "operator": ">", "value": "2024-01-01T00:00:00Z"}
+
+**boolean type** - Use for true/false values:
+- Operators: "=", "<>"
+- Value: Always true or false
+- Example: {"type": "boolean", "column": "bookmarked", "operator": "=", "value": true}
+
+**stringObject type** - Use for metadata key-value searches:
+- Operators: "=", "contains", "does not contain", "starts with", "ends with"
+- Value: Always a string
+- Requires "key" field for the metadata key
+- Example: {"type": "stringObject", "column": "metadata", "key": "userId", "operator": "=", "value": "123"}
+
+**numberObject type** - Use for numeric score searches:
+- Operators: "=", ">", "<", ">=", "<="
+- Value: Always a number
+- Requires "key" field for the score name
+- Example: {"type": "numberObject", "column": "scores_avg", "key": "quality", "operator": ">", "value": 0.8}
+
+## Output Format
+
+Please respond ONLY with valid JSON. Do not include any explanation or extra text.
+The response should look like this without the leading EOF and trailing EOF.
+
+EOF
+{
+  "filters": [
+    {
+      "type": "stringOptions|number|string|categoryOptions",
+      "value": "value or array",
+      "column": "exact column name",
+      "operator": "= | > | < | any of"
+    }
+  ]
+}
+EOF
+
+## Intent Parsing Guidelines
+
+**Temporal expressions:**
+- "today", "yesterday", "last week" → timestamp filters
+- "after 2pm", "before noon", "since Monday" → timestamp with appropriate operators
+- Relative times: "last 24 hours", "past 3 days" → calculate from current time
+
+**Performance queries:**
+- "slow", "high latency", "taking too long" → latency > threshold
+- "expensive", "costly", "high cost" → totalCost > threshold
+- "many tokens", "token heavy" → totalTokens > threshold
+- "cheap", "fast", "quick" → use < operators
+
+**Error/Quality queries:**
+- "errors", "failed", "broken" → level = ERROR or errorCount > 0
+- "warnings" → level = WARNING or warningCount > 0
+- "successful", "working" → level = DEFAULT or errorCount = 0
+
+**User/Session queries:**
+- "user john", "by user", "user ID" → userId filters
+- "session abc", "in session" → sessionId filters
+
+**Environment queries:**
+- "prod", "production" → environment = production
+- "dev", "development", "staging" → environment matching
+- "live", "deployed" → typically production environment
+
+**Metadata/Tags:**
+- "tagged with", "has tag" → tags contains
+- "metadata contains", "custom field" → metadata object queries
+
+**Comparison operators:**
+- "more than", "over", "above", "greater" → >
+- "less than", "under", "below", "fewer" → <
+- "at least", "minimum" → >=
+- "at most", "maximum" → <=
+- "exactly", "equal to" → =
+- "not", "except", "excluding" → not_equals or not_contains
+
+**Text search operations:**
+- "contains", "includes", "has" → use "string" type with "contains" operator
+- "equals", "is exactly" → use "string" type with "=" operator
+- "starts with", "begins with" → use "string" type with "starts with" operator
+
+**Multi-option selections:**
+- "environment is dev or staging" → use "stringOptions" type with "any of" operator
+- "name is one of X, Y, Z" → use "stringOptions" type with "any of" operator
+- "level is ERROR or WARNING" → use "stringOptions" type with "any of" operator
+
+## Current DateTime
+
+This is the current datetime: {{currentDatetime}}
+Use it as a reference for datetime based queries when needed.
+
+## Examples
+
+Input: "I want to see all dev traces"
+Output: {"filters":[{"type":"stringOptions","value":["development","dev"],"column":"environment","operator":"any of"}]}
+
+Input: "Show traces with version starting with v2"
+Output: {"filters":[{"type":"string","value":"v2","column":"version","operator":"starts with"}]}`,
+      },
+      {
+        role: "user",
+        content: "{{userPrompt}}",
+      },
+    ],
+    name: "get-filter-conditions-from-query",
+    version: 1,
+    labels: ["production", "latest"],
+  },
   {
     createdBy: "user-1",
     prompt: "Prompt 4 version 1 content with {{variable}}",

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -1,6 +1,7 @@
 import { FileContent, SeederOptions } from "./types";
 import { DataGenerator } from "./data-generators";
 import { ClickHouseQueryBuilder } from "./clickhouse-builder";
+import { FrameworkTraceLoader } from "./framework-traces/framework-trace-loader";
 import { EVAL_TRACE_COUNT, SEED_DATASETS } from "./postgres-seed-constants";
 import {
   clickhouseClient,
@@ -324,6 +325,9 @@ export class SeederOrchestrator {
       // Create traces for a realistic chat session
       await this.createSupportChatSessionTraces(projectIds);
 
+      // create traces from real examples for each framework source
+      await this.createFrameworkTraces(projectIds);
+
       // Log completion statistics (commented out to reduce terminal noise)
       await this.logStatistics();
 
@@ -404,4 +408,34 @@ export class SeederOrchestrator {
       }
     }
   }
+
+  // create traces from real examples for each framework source
+  // useful for testing rendering
+  async createFrameworkTraces(projectIds: string[]): Promise<void> {
+    logger.info(`Creating framework traces for ${projectIds.length} projects.`);
+
+    const loader = new FrameworkTraceLoader();
+
+    for (const projectId of projectIds) {
+      logger.info(`Processing framework traces for project ${projectId}`);
+
+      const { traces, observations, scores } =
+        loader.loadTracesForProject(projectId);
+
+      try {
+        if (traces.length > 0) {
+          await this.queryBuilder.executeTracesInsert(traces);
+        }
+        if (observations.length > 0) {
+          await this.queryBuilder.executeObservationsInsert(observations);
+        }
+        if (scores.length > 0) {
+          await this.queryBuilder.executeScoresInsert(scores);
+        }
+      } catch (error) {
+        logger.error(`✗ Framework traces insert failed:`, error);
+        throw error;
+      }
+    }
+  }
 }

packages/shared/src/domain/dataset-run-items.ts

@@ -25,4 +25,14 @@ export const DatasetRunItemSchema = z.object({
   datasetItemMetadata: MetadataDomain,
 });
 
-export type DatasetRunItemDomain = z.infer<typeof DatasetRunItemSchema>;
+// Conditional type for dataset run item domain with optional IO
+export type DatasetRunItemDomain<WithIO extends boolean = true> =
+  WithIO extends true
+    ? z.infer<typeof DatasetRunItemSchema>
+    : Omit<
+        z.infer<typeof DatasetRunItemSchema>,
+        | "datasetRunMetadata"
+        | "datasetItemInput"
+        | "datasetItemExpectedOutput"
+        | "datasetItemMetadata"
+      >;

packages/shared/src/domain/score-configs.ts

@@ -0,0 +1,121 @@
+import { z } from "zod/v4";
+import { isPresent } from "../utils/typeChecks";
+
+// Category type, used for categorical and boolean configs
+export const ScoreConfigCategory = z.object({
+  label: z.string().min(1),
+  value: z.number(),
+});
+
+// Numeric config fields
+export const NumericConfigFields = z.object({
+  maxValue: z.number().nullish(),
+  minValue: z.number().nullish(),
+  dataType: z.literal("NUMERIC"),
+  categories: z.undefined().nullish(),
+});
+
+// Boolean config fields
+export const BooleanConfigFields = z.object({
+  dataType: z.literal("BOOLEAN"),
+  maxValue: z.number().nullish(),
+  minValue: z.number().nullish(),
+  categories: z
+    .array(ScoreConfigCategory)
+    .length(2, "Boolean data type must have exactly 2 categories.")
+    .refine((categories) => {
+      const expectedCategories = [
+        { label: "True", value: 1 },
+        { label: "False", value: 0 },
+      ];
+      return categories.every(
+        (category, index) =>
+          category.label === expectedCategories[index].label &&
+          category.value === expectedCategories[index].value,
+      );
+    }),
+});
+
+// Category config fields and types
+export const validateCategories = (
+  categories: z.infer<typeof ScoreConfigCategory>[],
+  ctx: z.RefinementCtx,
+) => {
+  const uniqueNames = new Set<string>();
+  const uniqueValues = new Set<number>();
+
+  for (const category of categories) {
+    if (uniqueNames.has(category.label)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category label: ${category.label}, category labels must be unique`,
+      });
+      return;
+    }
+    uniqueNames.add(category.label);
+
+    if (uniqueValues.has(category.value)) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Duplicate category value: ${category.value}, category values must be unique`,
+      });
+      return;
+    }
+    uniqueValues.add(category.value);
+  }
+};
+
+export const CategoricalConfigFields = z.object({
+  maxValue: z.undefined().nullish(),
+  minValue: z.undefined().nullish(),
+  dataType: z.literal("CATEGORICAL"),
+  categories: z.array(ScoreConfigCategory).superRefine(validateCategories),
+});
+
+const ScoreConfigBase = z.object({
+  id: z.string(),
+  name: z.string().min(1).max(35),
+  isArchived: z.boolean(),
+  description: z.string().nullish(),
+  createdAt: z.coerce.date(),
+  updatedAt: z.coerce.date(),
+  projectId: z.string(),
+});
+
+export const validateNumericRangeFields = (
+  data: Pick<ScoreConfigDomain, "maxValue" | "minValue" | "dataType">,
+  ctx: z.RefinementCtx,
+): void | Promise<void> => {
+  if (data.dataType === "NUMERIC") {
+    if (
+      isPresent(data.maxValue) &&
+      isPresent(data.minValue) &&
+      data.maxValue <= data.minValue
+    ) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: "Maximum value must be greater than Minimum value",
+      });
+    }
+  }
+};
+
+export const ScoreConfigSchema = z
+  .discriminatedUnion("dataType", [
+    z.object({
+      ...ScoreConfigBase.shape,
+      ...NumericConfigFields.shape,
+    }),
+    z.object({
+      ...ScoreConfigBase.shape,
+      ...CategoricalConfigFields.shape,
+    }),
+    z.object({
+      ...ScoreConfigBase.shape,
+      ...BooleanConfigFields.shape,
+    }),
+  ])
+  .superRefine(validateNumericRangeFields);
+
+export type ScoreConfigDomain = z.infer<typeof ScoreConfigSchema>;
+export type ScoreConfigCategoryDomain = z.infer<typeof ScoreConfigCategory>;

packages/shared/src/domain/scores.ts

@@ -1,6 +1,6 @@
-import { ScoreDataType } from "@prisma/client";
 import z from "zod/v4";
 import { MetadataDomain } from "./traces";
+import { ScoreDataType } from "@prisma/client";
 
 export const ScoreSource = {
   ANNOTATION: "ANNOTATION",

packages/shared/src/env.ts

@@ -131,38 +131,6 @@ const EnvSchema = z.object({
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(600_000), // 10 minutes
   LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
   LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
-  LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS: z
-    .string()
-    .optional()
-    .transform((s) =>
-      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
-    ),
-  LANGFUSE_EXPERIMENT_SAMPLING_RATE: z.coerce
-    .number()
-    .min(0)
-    .max(1)
-    .default(0.1),
-  LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS: z
-    .string()
-    .optional()
-    .transform((s) =>
-      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
-    ),
-  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_EXPERIMENT_WHITELISTED_AMT_TABLES: z
-    .string()
-    .optional()
-    .transform((s) =>
-      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
-    ),
   LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
     .string()
     .optional()
@@ -231,6 +199,14 @@ const EnvSchema = z.object({
     .int()
     .positive()
     .default(120_000), // 2 minutes
+
+  LANGFUSE_AWS_BEDROCK_REGION: z.string().optional(),
+
+  // Langfuse AI Features
+  LANGFUSE_AI_FEATURES_PUBLIC_KEY: z.string().optional(),
+  LANGFUSE_AI_FEATURES_SECRET_KEY: z.string().optional(),
+  LANGFUSE_AI_FEATURES_HOST: z.string().optional(),
+  LANGFUSE_AI_FEATURES_PROJECT_ID: z.string().optional(),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/scoreConfigs/index.ts

@@ -0,0 +1 @@
+export * from "./validation";

packages/shared/src/features/scoreConfigs/validation.ts

@@ -0,0 +1,49 @@
+import { z } from "zod/v4";
+import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
+import {
+  ScoreConfigDomain,
+  ScoreConfigSchema,
+} from "../../domain/score-configs";
+
+/**
+ * Use this function when pulling a list of score configs from the database before using in the application to ensure type safety.
+ * All score configs are expected to pass the validation. If a score fails validation, it will be logged to Otel.
+ * @param scoreConfigs
+ * @returns list of validated score configs
+ */
+export const filterAndValidateDbScoreConfigList = (
+  scoreConfigs: ScoreConfigDbType[],
+  onParseError?: (error: z.ZodError) => void, // eslint-disable-line no-unused-vars
+): ScoreConfigDomain[] =>
+  scoreConfigs.reduce((acc, ts) => {
+    const result = ScoreConfigSchema.safeParse(ts);
+    if (result.success) {
+      acc.push(result.data);
+    } else {
+      onParseError?.(result.error);
+    }
+    return acc;
+  }, [] as ScoreConfigDomain[]);
+
+/**
+ * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
+ * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
+ * @param scoreConfig
+ * @returns validated score config
+ * @throws error if score fails validation
+ */
+export const validateDbScoreConfig = (
+  scoreConfig: ScoreConfigDbType,
+): ScoreConfigDomain => ScoreConfigSchema.parse(scoreConfig);
+
+/**
+ * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
+ * This function will NOT throw an error by default. The score is expected to pass the validation.
+ * @param scoreConfig
+ * @returns score config validation object:
+ * - success: true if the score config passes validation
+ * - data: the validated score config if success is true
+ * - error: the error object if success is false
+ */
+export const validateDbScoreConfigSafe = (scoreConfig: ScoreConfigDbType) =>
+  ScoreConfigSchema.safeParse(scoreConfig);

packages/shared/src/features/scores/index.ts

@@ -1,2 +1 @@
 export * from "./interfaces";
-export * from "./scoreConfigTypes";

packages/shared/src/features/scores/interfaces/ingestion/validation.ts

@@ -2,7 +2,7 @@ import z from "zod/v4";
 import { applyScoreValidation } from "../../../../utils/scores";
 import { PostScoreBodyFoundationSchema } from "../shared";
 import { isPresent } from "../../../../utils/typeChecks";
-import { Category as ConfigCategory } from "../../scoreConfigTypes";
+import { ScoreConfigCategory } from "../../../../domain/score-configs";
 
 export const ScoreBodyWithoutConfig = applyScoreValidation(
   z.discriminatedUnion("dataType", [
@@ -54,7 +54,7 @@ const ScorePropsAgainstConfigNumeric = z
 const ScorePropsAgainstConfigCategorical = z
   .object({
     value: z.string(),
-    categories: z.array(ConfigCategory),
+    categories: z.array(ScoreConfigCategory),
     dataType: z.literal("CATEGORICAL"),
   })
   .superRefine((data, ctx) => {

packages/shared/src/features/scores/interfaces/ui/types.ts

@@ -19,10 +19,9 @@ export type NumericAggregate = BaseAggregate & {
   average: number;
 };
 
-export type ScoreAggregate = Record<
-  string,
-  CategoricalAggregate | NumericAggregate
->;
+export type AggregatedScoreData = CategoricalAggregate | NumericAggregate;
+
+export type ScoreAggregate = Record<string, AggregatedScoreData>;
 
 export type ScoreSimplified = {
   id: string;

packages/shared/src/features/scores/scoreConfigTypes.ts

@@ -1,236 +0,0 @@
-import { z } from "zod/v4";
-
-import { ScoreConfig as ScoreConfigDbType } from "@prisma/client";
-
-import { isPresent } from "../../utils/typeChecks";
-import {
-  jsonSchema,
-  paginationMetaResponseZod,
-  publicApiPaginationZod,
-} from "../../utils/zod";
-
-/**
- * Types to use across codebase
- */
-export type ConfigCategory = z.infer<typeof Category>;
-export type ValidatedScoreConfig = z.infer<typeof ValidatedScoreConfigSchema>;
-
-const validateCategories = (
-  categories: ConfigCategory[],
-  ctx: z.RefinementCtx,
-) => {
-  const uniqueNames = new Set<string>();
-  const uniqueValues = new Set<number>();
-
-  for (const category of categories) {
-    if (uniqueNames.has(category.label)) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Duplicate category label: ${category.label}, category labels must be unique`,
-      });
-      return;
-    }
-    uniqueNames.add(category.label);
-
-    if (uniqueValues.has(category.value)) {
-      ctx.addIssue({
-        code: z.ZodIssueCode.custom,
-        message: `Duplicate category value: ${category.value}, category values must be unique`,
-      });
-      return;
-    }
-    uniqueValues.add(category.value);
-  }
-};
-
-/**
- * Objects
- */
-export const Category = z.object({
-  label: z.string().min(1),
-  value: z.number(),
-});
-
-const Categories = z.array(Category);
-
-const NumericScoreConfig = z.object({
-  maxValue: z.number().nullish(),
-  minValue: z.number().nullish(),
-  dataType: z.literal("NUMERIC"),
-  categories: z.undefined().nullish(),
-});
-
-const CategoricalScoreConfig = z.object({
-  maxValue: z.undefined().nullish(),
-  minValue: z.undefined().nullish(),
-  dataType: z.literal("CATEGORICAL"),
-  categories: jsonSchema.superRefine((categories, ctx) => {
-    const parseResult = Categories.safeParse(categories);
-    if (!parseResult.success) {
-      ctx.addIssue({
-        code: "custom",
-        message:
-          "Category must be an array of objects with label value pairs, where labels and values are unique.",
-      } as z.core.$ZodIssueCustom);
-      return;
-    }
-
-    validateCategories(categories as ConfigCategory[], ctx);
-  }),
-});
-
-const BooleanScoreConfig = z.object({
-  maxValue: z.undefined().nullish(),
-  minValue: z.undefined().nullish(),
-  dataType: z.literal("BOOLEAN"),
-  categories: z
-    .array(Category)
-    .length(2, "Boolean data type must have exactly 2 categories.")
-    .refine((categories) => {
-      const expectedCategories = [
-        { label: "True", value: 1 },
-        { label: "False", value: 0 },
-      ];
-      return categories.every(
-        (category, index) =>
-          category.label === expectedCategories[index].label &&
-          category.value === expectedCategories[index].value,
-      );
-    }),
-});
-
-const ScoreConfigBase = z.object({
-  id: z.string(),
-  name: z.string().min(1).max(35),
-  isArchived: z.boolean(),
-  description: z.string().nullish(),
-  createdAt: z.coerce.date(),
-  updatedAt: z.coerce.date(),
-  projectId: z.string(),
-});
-
-const ScoreConfigPostBase = z.object({
-  name: z.string(),
-  description: z.string().optional(),
-});
-
-const ValidatedScoreConfigSchema = z
-  .union([
-    ScoreConfigBase.merge(NumericScoreConfig),
-    ScoreConfigBase.merge(
-      z.object({
-        maxValue: z.undefined().nullish(),
-        minValue: z.undefined().nullish(),
-        dataType: z.literal("CATEGORICAL"),
-        categories: Categories.superRefine(validateCategories),
-      }),
-    ),
-    ScoreConfigBase.merge(BooleanScoreConfig),
-  ])
-  .superRefine((data, ctx) => {
-    if (data.dataType === "NUMERIC") {
-      if (
-        isPresent(data.maxValue) &&
-        isPresent(data.minValue) &&
-        data.maxValue <= data.minValue
-      ) {
-        ctx.addIssue({
-          code: z.ZodIssueCode.custom,
-          message: "Maximum value must be greater than Minimum value",
-        });
-      }
-    }
-  });
-
-/**
- * Use this function when pulling a list of score configs from the database before using in the application to ensure type safety.
- * All score configs are expected to pass the validation. If a score fails validation, it will be logged to Otel.
- * @param scoreConfigs
- * @returns list of validated score configs
- */
-export const filterAndValidateDbScoreConfigList = (
-  scoreConfigs: ScoreConfigDbType[],
-  onParseError?: (error: z.ZodError) => void, // eslint-disable-line no-unused-vars
-): ValidatedScoreConfig[] =>
-  scoreConfigs.reduce((acc, ts) => {
-    const result = ValidatedScoreConfigSchema.safeParse(ts);
-    if (result.success) {
-      acc.push(result.data);
-    } else {
-      onParseError?.(result.error);
-    }
-    return acc;
-  }, [] as ValidatedScoreConfig[]);
-
-/**
- * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
- * The score is expected to pass the validation. If a score fails validation, an error will be thrown.
- * @param scoreConfig
- * @returns validated score config
- * @throws error if score fails validation
- */
-export const validateDbScoreConfig = (
-  scoreConfig: ScoreConfigDbType,
-): ValidatedScoreConfig => ValidatedScoreConfigSchema.parse(scoreConfig);
-
-/**
- * Use this function when pulling a single score config from the database before using in the application to ensure type safety.
- * This function will NOT throw an error by default. The score is expected to pass the validation.
- * @param scoreConfig
- * @returns score config validation object:
- * - success: true if the score config passes validation
- * - data: the validated score config if success is true
- * - error: the error object if success is false
- */
-export const validateDbScoreConfigSafe = (scoreConfig: ScoreConfigDbType) =>
-  ValidatedScoreConfigSchema.safeParse(scoreConfig);
-
-/**
- * Endpoints
- */
-
-// GET /score-configs/{configId}
-export const GetScoreConfigQuery = z.object({
-  configId: z.string(),
-});
-
-export const GetScoreConfigResponse = ValidatedScoreConfigSchema;
-
-// POST /score-configs
-export const PostScoreConfigBody = z
-  .discriminatedUnion("dataType", [
-    ScoreConfigPostBase.merge(CategoricalScoreConfig),
-    ScoreConfigPostBase.merge(NumericScoreConfig),
-    ScoreConfigPostBase.merge(
-      z.object({
-        dataType: z.literal("BOOLEAN"),
-        categories: z.undefined().nullish(),
-      }),
-    ),
-  ])
-  .superRefine((data, ctx) => {
-    if (data.dataType === "NUMERIC") {
-      if (
-        isPresent(data.maxValue) &&
-        isPresent(data.minValue) &&
-        data.maxValue <= data.minValue
-      ) {
-        ctx.addIssue({
-          code: z.ZodIssueCode.custom,
-          message: "Maximum value must be greater than Minimum value",
-        });
-      }
-    }
-  });
-
-export const PostScoreConfigResponse = ValidatedScoreConfigSchema;
-
-// GET /score-configs
-export const GetScoreConfigsQuery = z.object({
-  ...publicApiPaginationZod,
-});
-
-export const GetScoreConfigsResponse = z.object({
-  data: z.array(ValidatedScoreConfigSchema),
-  meta: paginationMetaResponseZod,
-});

packages/shared/src/index.ts

@@ -19,6 +19,7 @@ export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
 export * from "./domain/webhooks";
 export * from "./domain/dataset-run-items";
+export * from "./domain/score-configs";
 
 // llm api
 export * from "./server/llm/types";
@@ -37,6 +38,9 @@ export * from "./features/annotation/types";
 // scores
 export * from "./features/scores";
 
+// score configs
+export * from "./features/scoreConfigs";
+
 // comments
 export * from "./features/comments/types";
 

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -13,26 +13,18 @@ export const CloudConfigSchema = z.object({
       customerId: z.string().optional(),
       activeSubscriptionId: z.string().optional(),
       activeProductId: z.string().optional(),
+      activeUsageProductId: z.string().optional(),
     })
+    .transform((data) => ({
+      ...data,
+      isLegacySubscription:
+        data?.activeProductId !== undefined &&
+        data?.activeUsageProductId === undefined,
+    }))
     .optional(),
 
   // custom rate limits for an organization
   rateLimitOverrides: CloudConfigRateLimit.optional(),
-
-  // billing alert configuration
-  usageAlerts: z
-    .object({
-      enabled: z.boolean().default(true),
-      type: z.enum(["STRIPE"]).default("STRIPE"),
-      threshold: z.number().int().positive(),
-      alertId: z.string(), // Alert ID for tracking
-      meterId: z.string(), // Meter ID for usage tracking
-      notifications: z.object({
-        email: z.boolean().default(true),
-        recipients: z.array(z.string().email()).default([]),
-      }),
-    })
-    .optional(),
 });
 
 export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/interfaces/parseDbOrg.ts

@@ -1,11 +1,11 @@
 import { type Organization } from "@prisma/client";
 import { CloudConfigSchema } from "./cloudConfigSchema";
 
-type parsedOrg = Omit<Organization, "cloudConfig"> & {
+export type ParsedOrganization = Omit<Organization, "cloudConfig"> & {
   cloudConfig: CloudConfigSchema | null;
 };
 
-export function parseDbOrg(dbOrg: Organization): parsedOrg {
+export function parseDbOrg(dbOrg: Organization): ParsedOrganization {
   const { cloudConfig, ...org } = dbOrg;
 
   const parsedCloudConfig = CloudConfigSchema.safeParse(cloudConfig);

packages/shared/src/server/auth/invalidateApiKeys.ts

@@ -0,0 +1,119 @@
+import { prisma } from "../../db";
+import { redis, safeMultiDel } from "..";
+import { logger } from "../logger";
+
+import { type ApiKey } from "../../db";
+
+/**
+ * Invalidate cached API keys from Redis cache
+ *
+ * Utility used by higher-level helpers to remove individual API keys from the cache,
+ * e.g. after key rotation, revocation, or entitlement/plan changes.
+ *
+ * Note: This only invalidates the Redis cache, not the API keys themselves in the database.
+ *
+ * Behavior:
+ * - Skips keys without a `fastHashedSecretKey`
+ * - No-ops when Redis is not configured
+ *
+ * @param apiKeys - List of API key records to invalidate from cache
+ * @param identifier - Context string for logging (e.g., org or project identifier)
+ */
+export async function invalidateCachedApiKeys(
+  apiKeys: ApiKey[],
+  identifier: string,
+) {
+  const hashKeys = apiKeys.map((key) => key.fastHashedSecretKey);
+
+  const filteredHashKeys = hashKeys.filter((hash): hash is string =>
+    Boolean(hash),
+  );
+  if (filteredHashKeys.length === 0) {
+    logger.info("No valid keys to invalidate");
+    return;
+  }
+
+  if (redis) {
+    logger.info(`Invalidating API keys in redis for ${identifier}`);
+    const keysToDelete = filteredHashKeys.map((hash) => `api-key:${hash}`);
+    await safeMultiDel(redis, keysToDelete);
+  }
+}
+
+/**
+ * Invalidate all cached API keys for an organization from Redis cache
+ *
+ * This function is used when organization-level changes occur that affect API key validity,
+ * such as:
+ * - Plan changes (subscription created/updated/deleted)
+ * - Usage threshold state changes (blocking/unblocking)
+ * - Billing cycle changes
+ *
+ * Note: This only invalidates the Redis cache, not the API keys themselves in the database.
+ *
+ * @param orgId - The organization ID whose API keys should be invalidated from cache
+ */
+export async function invalidateCachedOrgApiKeys(orgId: string): Promise<void> {
+  const apiKeys = await prisma.apiKey.findMany({
+    where: {
+      OR: [
+        {
+          project: {
+            orgId,
+          },
+        },
+        { orgId },
+      ],
+    },
+  });
+
+  const hashKeys = apiKeys
+    .map((key) => key.fastHashedSecretKey)
+    .filter((hash): hash is string => Boolean(hash));
+
+  if (hashKeys.length === 0) {
+    logger.info(`No valid API keys to invalidate for org ${orgId}`);
+    return;
+  }
+
+  if (redis) {
+    logger.info(`Invalidating API keys in redis for org ${orgId}`);
+    const keysToDelete = hashKeys.map((hash) => `api-key:${hash}`);
+    await safeMultiDel(redis, keysToDelete);
+  }
+}
+
+/**
+ * Invalidate all cached API keys for a project from Redis cache
+ *
+ * This function is used when project-level changes occur that affect API key validity.
+ *
+ * Note: This only invalidates the Redis cache, not the API keys themselves in the database.
+ *
+ * @param projectId - The project ID whose API keys should be invalidated from cache
+ */
+export async function invalidateCachedProjectApiKeys(
+  projectId: string,
+): Promise<void> {
+  const apiKeys = await prisma.apiKey.findMany({
+    where: {
+      projectId: projectId,
+      scope: "PROJECT",
+    },
+  });
+
+  const hashKeys = apiKeys
+    .map((key) => key.fastHashedSecretKey)
+    .filter((hash): hash is string => Boolean(hash));
+
+  if (hashKeys.length === 0) {
+    logger.info(`No valid API keys to invalidate for project ${projectId}`);
+    return;
+  }
+
+  if (redis) {
+    logger.info(`Invalidating API keys in redis for project ${projectId}`);
+    const keysToDelete = hashKeys.map((hash) => `api-key:${hash}`);
+    await safeMultiDel(redis, keysToDelete);
+  }
+}

packages/shared/src/server/auth/types.ts

@@ -16,6 +16,7 @@ const ApiKeyBaseSchema = z.object({
   orgId: z.string(),
   plan: z.enum(plans as unknown as [string, ...string[]]),
   rateLimitOverrides: CloudConfigRateLimit.nullish(),
+  isIngestionSuspended: z.boolean().nullish(),
 });
 
 export const OrgEnrichedApiKey = z.discriminatedUnion("scope", [
@@ -64,6 +65,7 @@ type ApiAccessScopeMetadata = {
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
   apiKeyId: string;
   publicKey: string;
+  isIngestionSuspended: boolean | null | undefined;
 };
 
 export type ApiAccessScopeIngestion = BaseApiAccessScope &

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -1,7 +1,5 @@
-import { instrumentAsync, recordDistribution } from "../instrumentation";
 import * as opentelemetry from "@opentelemetry/api";
-import { env } from "../../env";
-import { logger } from "../logger";
+import { instrumentAsync } from "../instrumentation";
 
 const executionWrapper = async <T, Y>(
   input: T,
@@ -20,16 +18,15 @@ const executionWrapper = async <T, Y>(
 };
 
 /**
- * Measures the execution time of two functions and returns the result based on the experiment configuration.
- * This is used to compare the execution of AggregatingMergeTrees with the existing ReplacingMergeTree execution.
+ * Measures the execution time of a query functions and returns the result based on the experiment configuration.
+ * Presently this is but a simple wrapper around single execution, but it is designed to be
+ * extended for A/B testing or canary releases.
  */
 export const measureAndReturn = async <T, Y>(args: {
   operationName: string;
   projectId: string;
   input: T;
-  existingExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
-  newExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
-  minStartTime?: Date;
+  fn: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
 }): Promise<Y> => {
   return instrumentAsync(
     {
@@ -37,112 +34,17 @@ export const measureAndReturn = async <T, Y>(args: {
       spanKind: opentelemetry.SpanKind.CLIENT,
     },
     async (currentSpan) => {
-      const { input, existingExecution, newExecution, minStartTime } = args;
+      const { input, fn } = args;
 
-      if (
-        env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES !==
-        "true"
-      ) {
-        currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
+      // When we want do to multiple executions for A/B testing or canary releases,
+      // or some other form of a more complex wrapper we used to try-catch
+      // using the wrapper function and fallback to a simple f(input) when it failed.
+      const [[existingResult, _existingDuration]] = // eslint-disable-line no-unused-vars
+        await Promise.all([
+          executionWrapper(input, fn, currentSpan, "existing"),
+        ]);
 
-        // Check for short-term new result experiment
-        if (
-          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
-          minStartTime
-        ) {
-          const thirtyDaysAgo = new Date();
-          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
-
-          if (minStartTime >= thirtyDaysAgo) {
-            currentSpan.setAttribute(
-              `langfuse.experiment.amts.short-term`,
-              "true",
-            );
-            return newExecution(input);
-          }
-        }
-
-        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
-          ? newExecution(input)
-          : existingExecution(input);
-      }
-
-      // If not whitelisted, apply sampling logic
-      if (
-        !env.LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS.includes(
-          args.projectId,
-        ) &&
-        Math.random() > env.LANGFUSE_EXPERIMENT_SAMPLING_RATE
-      ) {
-        currentSpan.setAttribute(`langfuse.experiment.amts.run`, "sampled-out");
-        return existingExecution(input);
-      }
-
-      currentSpan.setAttribute(`langfuse.experiment.amts.run`, "true");
-
-      try {
-        const [[existingResult, existingDuration], [newResult, newDuration]] =
-          await Promise.all([
-            executionWrapper(input, existingExecution, currentSpan, "existing"),
-            executionWrapper(input, newExecution, currentSpan, "new"),
-          ]);
-        // Positive duration difference means new is faster
-        const durationDifference = existingDuration - newDuration;
-        currentSpan?.setAttribute(
-          "langfuse.experiment.amts.execution-time-difference",
-          durationDifference,
-        );
-
-        recordDistribution(
-          "langfuse.experiment.amts.duration_difference_distribution",
-          durationDifference,
-          {
-            operation: args.operationName,
-          },
-        );
-
-        if (
-          env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
-            (p) => p === args.projectId,
-          )
-        ) {
-          currentSpan?.setAttribute(
-            "langfuse.experiment.amts.existing-result",
-            JSON.stringify(existingResult),
-          );
-          currentSpan?.setAttribute(
-            "langfuse.experiment.amts.new-result",
-            JSON.stringify(newResult),
-          );
-        }
-
-        // Check for short-term new result experiment
-        if (
-          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
-          minStartTime
-        ) {
-          const thirtyDaysAgo = new Date();
-          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
-
-          if (minStartTime >= thirtyDaysAgo) {
-            currentSpan.setAttribute(
-              `langfuse.experiment.amts.short-term`,
-              "true",
-            );
-            return newResult;
-          }
-        }
-
-        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
-          ? newResult
-          : existingResult;
-      } catch (e) {
-        logger.error(
-          "Failed to run experiment wrapper. Retrying existing query",
-          e,
-        );
-        return existingExecution(input);
-      }
+      return existingResult;
     },
   );
 };

packages/shared/src/server/index.ts

@@ -2,13 +2,16 @@ export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
-export * from "./services/email/billingAlert/sendBillingAlertEmail";
+export * from "./services/email/cloudSpendAlert/sendCloudSpendAlertEmail";
+export * from "./services/email/usageThresholdWarning/sendUsageThresholdWarningEmail";
+export * from "./services/email/usageThresholdSuspension/sendUsageThresholdSuspensionEmail";
 export * from "./services/PromptService";
 export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
 export * from "./services/InMemoryFilterService";
 export * from "./evalJobConfigCache";
 export * from "./auth/apiKeys";
+export * from "./auth/invalidateApiKeys";
 export * from "./auth/customSsoProvider";
 export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
@@ -18,6 +21,7 @@ export * from "./llm/compileChatMessages";
 export * from "./llm/testModelCall";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
+export * from "./utils/billingCycleHelpers";
 export * from "./clickhouse/client";
 export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
@@ -30,6 +34,8 @@ export * from "./redis/redis";
 export * from "./redis/traceUpsert";
 export * from "./redis/createEvalQueue";
 export * from "./redis/cloudUsageMeteringQueue";
+export * from "./redis/cloudSpendAlertQueue";
+export * from "./redis/cloudFreeTierUsageThresholdQueue";
 export * from "./redis/getQueue";
 export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
@@ -63,6 +69,7 @@ export * from "./logger";
 export * from "./headerPropagation";
 export * from "./queries";
 export * from "./repositories";
+export * from "./repositories/traces";
 export * from "./utils/rendering";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -1,7 +1,6 @@
 import { randomUUID } from "crypto";
 import { z } from "zod/v4";
 
-import { type Model } from "../../db";
 import { env } from "../../env";
 import {
   InvalidRequestError,
@@ -50,12 +49,6 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
   return s3StorageServiceClient;
 };
 
-// eslint-disable-next-line no-unused-vars
-export type TokenCountDelegate = (p: {
-  model: Model;
-  text: unknown;
-}) => number | undefined;
-
 /**
  * Get the delay for the event based on the event type. Uses delay if set, 0 if current UTC timestamp is not between
  * 23:45 and 00:15, and env.LANGFUSE_INGESTION_QUEUE_DELAY_MS otherwise.

packages/shared/src/server/ingestion/validateAndInflateScore.ts

@@ -1,23 +1,23 @@
 import {
   ScoreBodyWithoutConfig,
+  ScoreConfigDomain,
   ScoreDomain,
   ScorePropsAgainstConfig,
-  validateDbScoreConfigSafe,
-  ValidatedScoreConfig,
 } from "../../../src";
 import { prisma, ScoreDataType } from "../../db";
-
 import { InvalidRequestError, LangfuseNotFoundError } from "../../errors";
+import { validateDbScoreConfigSafe } from "../../features/scoreConfigs/validation";
+import { ScoreEventType } from "./types";
 
 type ValidateAndInflateScoreParams = {
   projectId: string;
   scoreId: string;
-  body: any;
+  body: ScoreEventType["body"];
 };
 
 export async function validateAndInflateScore(
   params: ValidateAndInflateScoreParams,
-): Promise<ScoreDomain> {
+) {
   const { body, projectId, scoreId } = params;
 
   if (body.configId) {
@@ -40,16 +40,17 @@ export async function validateAndInflateScore(
       name: config.name,
     };
 
-    validateConfigAgainstBody(
-      bodyWithConfigOverrides,
-      config as ValidatedScoreConfig,
-    );
+    validateConfigAgainstBody({
+      body: bodyWithConfigOverrides,
+      config: config as ScoreConfigDomain,
+      context: "INGESTION",
+    });
 
     return inflateScoreBody({
       projectId,
       scoreId,
       body: bodyWithConfigOverrides,
-      config: config as ValidatedScoreConfig,
+      config: config as ScoreConfigDomain,
     });
   }
 
@@ -74,7 +75,7 @@ function inferDataType(value: string | number): ScoreDataType {
 }
 
 function mapStringValueToNumericValue(
-  config: ValidatedScoreConfig,
+  config: ScoreConfigDomain,
   label: string,
 ): number | null {
   return (
@@ -84,12 +85,17 @@ function mapStringValueToNumericValue(
 }
 
 function inflateScoreBody(
-  params: ValidateAndInflateScoreParams & { config?: ValidatedScoreConfig },
-): ScoreDomain {
+  params: ValidateAndInflateScoreParams & { config?: ScoreConfigDomain },
+) {
   const { body, projectId, scoreId, config } = params;
 
   const relevantDataType = config?.dataType ?? body.dataType;
-  const scoreProps = { source: "API", ...body, id: scoreId, projectId };
+  const scoreProps = {
+    ...body,
+    source: body.source ?? "API",
+    id: scoreId,
+    projectId,
+  };
 
   if (typeof body.value === "number") {
     if (relevantDataType && relevantDataType === ScoreDataType.BOOLEAN) {
@@ -104,6 +110,7 @@ function inflateScoreBody(
     return {
       ...scoreProps,
       value: body.value,
+      stringValue: null,
       dataType: ScoreDataType.NUMERIC,
     };
   }
@@ -116,10 +123,43 @@ function inflateScoreBody(
   };
 }
 
-function validateConfigAgainstBody(
-  body: any,
-  config: ValidatedScoreConfig,
-): void {
+type ScoreBodyWithContext =
+  | {
+      body: ScoreEventType["body"];
+      context: "INGESTION";
+    }
+  | {
+      body: ScoreDomain;
+      context: "ANNOTATION";
+    };
+
+function resolveScoreValueIngestion(
+  body: ScoreEventType["body"],
+): string | number | null {
+  return body.value;
+}
+
+function resolveScoreValueAnnotation(
+  body: ScoreDomain,
+): string | number | null {
+  switch (body.dataType) {
+    case ScoreDataType.NUMERIC:
+    case ScoreDataType.BOOLEAN:
+      return body.value;
+    case ScoreDataType.CATEGORICAL:
+      return body.stringValue;
+  }
+}
+
+type ValidateConfigAgainstBodyParams = {
+  config: ScoreConfigDomain;
+} & ScoreBodyWithContext;
+
+export function validateConfigAgainstBody({
+  body,
+  config,
+  context,
+}: ValidateConfigAgainstBodyParams): void {
   const { maxValue, minValue, categories, dataType: configDataType } = config;
 
   if (body.dataType && body.dataType !== configDataType) {
@@ -141,20 +181,13 @@ function validateConfigAgainstBody(
   }
 
   const relevantDataType = configDataType ?? body.dataType;
-
-  const dataTypeValidation = ScoreBodyWithoutConfig.safeParse({
-    ...body,
-    dataType: relevantDataType,
-  });
-
-  if (!dataTypeValidation.success) {
-    throw new InvalidRequestError(
-      `Ingested score body not valid against provided config data type.`,
-    );
-  }
+  const scoreValue =
+    context === "INGESTION"
+      ? resolveScoreValueIngestion(body)
+      : resolveScoreValueAnnotation(body);
 
   const rangeValidation = ScorePropsAgainstConfig.safeParse({
-    value: body.value,
+    value: scoreValue,
     dataType: relevantDataType,
     ...(maxValue !== null && maxValue !== undefined && { maxValue }),
     ...(minValue !== null && minValue !== undefined && { minValue }),

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -26,8 +26,6 @@ import GCPServiceAccountKeySchema, {
   VertexAIConfigSchema,
   BEDROCK_USE_DEFAULT_CREDENTIALS,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-import { processEventBatch } from "../ingestion/processEventBatch";
-import { logger } from "../logger";
 import {
   ChatMessage,
   ChatMessageRole,
@@ -40,11 +38,13 @@ import {
   OpenAIModel,
   ToolCallResponse,
   ToolCallResponseSchema,
-  TraceParams,
+  TraceSinkParams,
 } from "./types";
-import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 import { HttpsProxyAgent } from "https-proxy-agent";
+import { getInternalTracingHandler } from "./getInternalTracingHandler";
+import { decrypt } from "../../encryption";
+import { decryptAndParseExtraHeaders } from "./utils";
 
 const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
 
@@ -68,15 +68,18 @@ type ProcessTracedEvents = () => Promise<void>;
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
+  llmConnection: {
+    secretKey: string;
+    extraHeaders?: string | null;
+    baseURL?: string | null;
+    config?: Record<string, string> | null;
+  };
   structuredOutputSchema?: ZodSchema | LLMJSONSchema;
   callbacks?: BaseCallbackHandler[];
-  baseURL?: string;
-  apiKey: string;
-  extraHeaders?: Record<string, string>;
   maxRetries?: number;
-  config?: Record<string, string> | null;
-  traceParams?: TraceParams;
+  traceSinkParams?: TraceSinkParams;
   throwOnError?: boolean; // default is true
+  shouldUseLangfuseAPIKey?: boolean;
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -136,47 +139,31 @@ export async function fetchLLMCompletion(
     | ToolCallResponse;
   processTracedEvents: ProcessTracedEvents;
 }> {
-  // the apiKey must never be printed to the console
   const {
     messages,
     tools,
     modelParams,
     streaming,
     callbacks,
-    apiKey,
-    baseURL,
+    llmConnection,
     maxRetries,
-    config,
-    traceParams,
-    extraHeaders,
+    traceSinkParams,
     throwOnError = true,
+    shouldUseLangfuseAPIKey = false,
   } = params;
 
+  const { baseURL, config } = llmConnection;
+  const apiKey = decrypt(llmConnection.secretKey); // the apiKey must never be printed to the console
+  const extraHeaders = decryptAndParseExtraHeaders(llmConnection.extraHeaders);
+
   let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
   let processTracedEvents: ProcessTracedEvents = () => Promise.resolve();
 
-  if (traceParams) {
-    const handler = new CallbackHandler({
-      _projectId: traceParams.projectId,
-      _isLocalEventExportEnabled: true,
-      environment: traceParams.environment,
-    });
-    finalCallbacks.push(handler);
+  if (traceSinkParams) {
+    const internalTracingHandler = getInternalTracingHandler(traceSinkParams);
+    processTracedEvents = internalTracingHandler.processTracedEvents;
 
-    processTracedEvents = async () => {
-      try {
-        const events = await handler.langfuse._exportLocalEvents(
-          traceParams.projectId,
-        );
-        await processEventBatch(
-          JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
-          traceParams.authCheck,
-          { isLangfuseInternal: true },
-        );
-      } catch (e) {
-        logger.error("Failed to process traced events", { error: e });
-      }
-    };
+    finalCallbacks.push(internalTracingHandler.handler);
   }
 
   finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
@@ -249,7 +236,7 @@ export async function fetchLLMCompletion(
   if (modelParams.adapter === LLMAdapter.Anthropic) {
     chatModel = new ChatAnthropic({
       anthropicApiKey: apiKey,
-      anthropicApiUrl: baseURL,
+      anthropicApiUrl: baseURL ?? undefined,
       modelName: modelParams.model,
       temperature: modelParams.temperature,
       maxTokens: modelParams.max_tokens,
@@ -285,7 +272,7 @@ export async function fetchLLMCompletion(
   } else if (modelParams.adapter === LLMAdapter.Azure) {
     chatModel = new AzureChatOpenAI({
       azureOpenAIApiKey: apiKey,
-      azureOpenAIBasePath: baseURL,
+      azureOpenAIBasePath: baseURL ?? undefined,
       azureOpenAIApiDeploymentName: modelParams.model,
       azureOpenAIApiVersion: "2025-02-01-preview",
       temperature: modelParams.temperature,
@@ -301,10 +288,16 @@ export async function fetchLLMCompletion(
       modelKwargs: modelParams.providerOptions,
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
-    const { region } = BedrockConfigSchema.parse(config);
+    const { region } = shouldUseLangfuseAPIKey
+      ? { region: env.LANGFUSE_AWS_BEDROCK_REGION }
+      : BedrockConfigSchema.parse(config);
+
     // Handle both explicit credentials and default provider chain
+    // Only allow default provider chain in self-hosted or internal AI features
+    const isSelfHosted = !isLangfuseCloud;
     const credentials =
-      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS && !isLangfuseCloud
+      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS &&
+      (isSelfHosted || shouldUseLangfuseAPIKey)
         ? undefined // undefined = use AWS SDK default credential provider chain
         : BedrockCredentialSchema.parse(JSON.parse(apiKey));
 
@@ -361,8 +354,9 @@ export async function fetchLLMCompletion(
 
   const runConfig = {
     callbacks: finalCallbacks,
-    runId: traceParams?.traceId,
-    runName: traceParams?.traceName,
+    runId: traceSinkParams?.traceId,
+    runName: traceSinkParams?.traceName,
+    metadata: traceSinkParams?.metadata,
   };
 
   try {

packages/shared/src/server/llm/getInternalTracingHandler.ts

@@ -0,0 +1,56 @@
+import CallbackHandler from "langfuse-langchain";
+import { TraceSinkParams } from "./types";
+import { processEventBatch } from "../ingestion/processEventBatch";
+import { logger } from "../logger";
+
+export function getInternalTracingHandler(traceSinkParams: TraceSinkParams): {
+  handler: CallbackHandler;
+  processTracedEvents: () => Promise<void>;
+} {
+  const { prompt, targetProjectId, environment, userId } = traceSinkParams;
+  const handler = new CallbackHandler({
+    _projectId: targetProjectId,
+    _isLocalEventExportEnabled: true,
+    environment: environment,
+    userId: userId,
+  });
+
+  const processTracedEvents = async () => {
+    try {
+      const events = await handler.langfuse._exportLocalEvents(
+        traceSinkParams.targetProjectId,
+      );
+      // to add the prompt name and version to only generation-type observations
+      const processedEvents = events.map((event: any) => {
+        if (event.type === "generation-create" && prompt) {
+          return {
+            ...event,
+            body: {
+              ...event.body,
+              ...{ promptName: prompt.name, promptVersion: prompt.version },
+            },
+          };
+        }
+        return event;
+      });
+
+      await processEventBatch(
+        JSON.parse(JSON.stringify(processedEvents)), // stringify to emulate network event batch from network call
+        {
+          validKey: true as const,
+          scope: {
+            projectId: traceSinkParams.targetProjectId, // Important: this controls into what project traces are ingested.
+            accessLevel: "project",
+          } as any,
+        },
+        {
+          isLangfuseInternal: true,
+        },
+      );
+    } catch (e) {
+      logger.error("Failed to process traced events", { error: e });
+    }
+  };
+
+  return { handler, processTracedEvents };
+}

packages/shared/src/server/llm/testModelCall.ts

@@ -5,9 +5,7 @@ import {
   LLMApiKeySchema,
   type ModelConfig,
 } from "./types";
-import { decrypt } from "../../encryption";
 import { fetchLLMCompletion } from "./fetchLLMCompletion";
-import { decryptAndParseExtraHeaders } from "./utils";
 import z from "zod/v4";
 
 export const testModelCall = async ({
@@ -26,9 +24,7 @@ export const testModelCall = async ({
   (
     await fetchLLMCompletion({
       streaming: false,
-      apiKey: decrypt(apiKey.secretKey), // decrypt the secret key
-      extraHeaders: decryptAndParseExtraHeaders(apiKey.extraHeaders),
-      baseURL: apiKey.baseURL ?? undefined,
+      llmConnection: apiKey,
       messages: [
         {
           role: ChatMessageRole.User,
@@ -46,7 +42,6 @@ export const testModelCall = async ({
         score: zodV3.string(),
         reasoning: zodV3.string(),
       }),
-      config: apiKey.config,
     })
   ).completion;
 };

packages/shared/src/server/llm/types.ts

@@ -4,14 +4,12 @@ import {
   BedrockConfigSchema,
   VertexAIConfigSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-import { TokenCountDelegate } from "../ingestion/processEventBatch";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
 import { JSONObjectSchema } from "../../utils/zod";
 
 /* eslint-disable no-unused-vars */
 // disable lint as this is exported and used in web/worker
 
-export const LLMJSONSchema = z.record(z.string(), z.unknown());
+export const LLMJSONSchema = z.record(z.string(), z.any());
 export type LLMJSONSchema = z.infer<typeof LLMJSONSchema>;
 
 export const JSONSchemaFormSchema = z
@@ -60,6 +58,13 @@ const AnthropicMessageContentWithToolUse = z.union([
   }),
 ]);
 
+const GoogleAIStudioMessageContentWithToolUse = z.object({
+  functionCall: z.object({
+    name: z.string(),
+    args: z.unknown(),
+  }),
+});
+
 export const LLMToolCallSchema = z.object({
   name: z.string(),
   id: z.string(),
@@ -104,7 +109,11 @@ export const OpenAIResponseFormatSchema = z.object({
 });
 
 export const ToolCallResponseSchema = z.object({
-  content: z.union([z.string(), z.array(AnthropicMessageContentWithToolUse)]),
+  content: z.union([
+    z.string(),
+    z.array(AnthropicMessageContentWithToolUse),
+    z.array(GoogleAIStudioMessageContentWithToolUse),
+  ]),
   tool_calls: z.array(LLMToolCallSchema),
 });
 export type ToolCallResponse = z.infer<typeof ToolCallResponseSchema>;
@@ -282,6 +291,9 @@ export const ExperimentMetadataSchema = z
     provider: z.string(),
     model: z.string(),
     model_params: ZodModelConfig,
+    structured_output_schema: LLMJSONSchema.optional(),
+    experiment_name: z.string().optional(),
+    experiment_run_name: z.string().optional(),
     error: z.string().optional(),
   })
   .strict();
@@ -389,6 +401,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 // NOTE: Update docs page when changing this! https://langfuse.com/docs/prompt-management/features/playground#openai-playground--anthropic-playground
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const anthropicModels = [
+  "claude-sonnet-4-5-20250929",
   "claude-sonnet-4-20250514",
   "claude-opus-4-1-20250805",
   "claude-opus-4-20250514",
@@ -480,17 +493,23 @@ export type LLMApiKey =
     ? z.infer<typeof LLMApiKeySchema>
     : never;
 
-// NOTE: This string is whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
-export const PROMPT_EXPERIMENT_ENVIRONMENT =
-  "langfuse-prompt-experiment" as const;
+export enum LangfuseInternalTraceEnvironment {
+  PromptExperiments = "langfuse-prompt-experiment",
+}
 
-type PromptExperimentEnvironment = typeof PROMPT_EXPERIMENT_ENVIRONMENT;
-
-export type TraceParams = {
-  traceName: string;
+export type TraceSinkParams = {
+  /**
+   * IMPORTANT: This controls into what project the resulting traces are ingested.
+   */
+  targetProjectId: string;
   traceId: string;
-  projectId: string;
-  environment: PromptExperimentEnvironment;
-  tokenCountDelegate: TokenCountDelegate;
-  authCheck: AuthHeaderValidVerificationResult;
+  traceName: string;
+  // NOTE: These strings must be whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
+  environment: string;
+  userId?: string;
+  metadata?: Record<string, unknown>;
+  prompt?: {
+    name: string;
+    version: number;
+  };
 };

packages/shared/src/server/otel/ObservationTypeMapper.ts

@@ -33,7 +33,8 @@ class SimpleAttributeMapper implements ObservationTypeMapper {
     _scopeData?: Record<string, unknown>,
   ): boolean {
     return (
-      this.attributeKey in attributes && attributes[this.attributeKey] != null
+      this.attributeKey in attributes &&
+      hasMeaningfulValue(attributes[this.attributeKey])
     );
   }
 
@@ -102,6 +103,20 @@ class CustomAttributeMapper implements ObservationTypeMapper {
   }
 }
 
+// value is not null, undefined, empty string, or empty object/array
+function hasMeaningfulValue(value: unknown): boolean {
+  if (value === null || value === undefined || value === "") {
+    return false;
+  }
+  if (Array.isArray(value)) {
+    return value.length > 0;
+  }
+  if (typeof value === "object" && value !== null) {
+    return Object.keys(value).length > 0;
+  }
+  return true;
+}
+
 /**
  * Registry to manage observation type mappers with a unified interface to map
  * span attributes to observation types.
@@ -240,7 +255,7 @@ export class ObservationTypeMapperRegistry {
           "llm.model_name",
           "model",
         ];
-        return modelKeys.some((key) => attributes[key] != null);
+        return modelKeys.some((key) => hasMeaningfulValue(attributes[key]));
       },
       () => "GENERATION",
     ),

packages/shared/src/server/otel/OtelIngestionProcessor.ts

@@ -15,12 +15,14 @@ import {
   traceException,
   getS3EventStorageClient,
   QueueJobs,
+  instrumentSync,
 } from "../";
 
 import { LangfuseOtelSpanAttributes } from "./attributes";
 import { ObservationTypeMapperRegistry } from "./ObservationTypeMapper";
 import { env } from "../../env";
 import { OtelIngestionQueue } from "../redis/otelIngestionQueue";
+import { isValidDateString } from "./utils";
 
 // Type definitions for internal processor state
 interface TraceState {
@@ -159,6 +161,199 @@ export class OtelIngestionProcessor {
       : Promise.reject("Failed to instantiate otel ingestion queue");
   }
 
+  /**
+   * Processes incoming resourceSpans and produces an event base record that can be enriched
+   * using the IngestionService.
+   * @param resourceSpans
+   */
+  processToEvent(resourceSpans: ResourceSpan[]): any[] {
+    return instrumentSync({ name: "otel-event-processor" }, (span) => {
+      try {
+        span.setAttribute("project_id", this.projectId);
+        span.setAttribute(
+          "total_span_count",
+          this.getTotalSpanCount(resourceSpans),
+        );
+
+        // Input validation
+        if (!Array.isArray(resourceSpans)) {
+          return [];
+        }
+        if (resourceSpans.length === 0) {
+          return [];
+        }
+
+        return resourceSpans
+          .filter((r) => Boolean(r))
+          .flatMap((resourceSpan) => {
+            const resourceAttributes =
+              this.extractResourceAttributes(resourceSpan);
+            const events: any[] = [];
+
+            for (const scopeSpan of resourceSpan?.scopeSpans ?? []) {
+              const scopeAttributes = this.extractScopeAttributes(scopeSpan);
+              for (const span of scopeSpan?.spans ?? []) {
+                const spanAttributes = this.extractSpanAttributes(span);
+                const traceId = this.parseId(span.traceId);
+                const spanId = this.parseId(span.spanId);
+                const parentSpanId = span?.parentSpanId
+                  ? this.parseId(span.parentSpanId)
+                  : null;
+                const name = span.name;
+                const startTimeISO =
+                  OtelIngestionProcessor.convertNanoTimestampToISO(
+                    span.startTimeUnixNano,
+                  );
+                const endTimeISO =
+                  OtelIngestionProcessor.convertNanoTimestampToISO(
+                    span.endTimeUnixNano,
+                  );
+
+                // Extract metadata from different sources
+                const spanMetadata = this.extractMetadata(
+                  spanAttributes,
+                  "observation",
+                );
+                const traceMetadata = this.extractMetadata(
+                  spanAttributes,
+                  "trace",
+                );
+
+                // Construct metadata object with the specified structure
+                const metadata = {
+                  // attributes: spanAttributes,
+                  resourceAttributes: resourceAttributes,
+                  scopeAttributes: scopeAttributes,
+                  ...spanMetadata,
+                  ...traceMetadata,
+                };
+
+                // Extract instrumentation metadata
+                const serviceName = resourceAttributes?.["service.name"] as
+                  | string
+                  | undefined;
+                const serviceVersion = resourceAttributes?.[
+                  "service.version"
+                ] as string | undefined;
+                const telemetrySdkLanguage = resourceAttributes?.[
+                  "telemetry.sdk.language"
+                ] as string | undefined;
+                const telemetrySdkName = resourceAttributes?.[
+                  "telemetry.sdk.name"
+                ] as string | undefined;
+                const telemetrySdkVersion = resourceAttributes?.[
+                  "telemetry.sdk.version"
+                ] as string | undefined;
+                const scopeName = scopeSpan?.scope?.name;
+                const scopeVersion = scopeSpan?.scope?.version;
+
+                const stringifiedSpan = JSON.stringify(span);
+
+                events.push({
+                  projectId: this.projectId,
+                  traceId,
+                  spanId,
+                  parentSpanId,
+
+                  name,
+                  type: observationTypeMapper.mapToObservationType(
+                    spanAttributes,
+                    resourceAttributes,
+                    scopeSpan?.scope,
+                  ),
+                  environment: this.extractEnvironment(
+                    spanAttributes,
+                    resourceAttributes,
+                  ),
+                  version:
+                    spanAttributes?.[LangfuseOtelSpanAttributes.VERSION] ??
+                    resourceAttributes?.["service.version"] ??
+                    null,
+
+                  startTimeISO,
+                  endTimeISO,
+
+                  level:
+                    spanAttributes[
+                      LangfuseOtelSpanAttributes.OBSERVATION_LEVEL
+                    ] ??
+                    (span.status?.code === 2
+                      ? ObservationLevel.ERROR
+                      : ObservationLevel.DEFAULT),
+                  statusMessage:
+                    spanAttributes[
+                      LangfuseOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE
+                    ] ??
+                    span.status?.message ??
+                    null,
+
+                  promptName:
+                    spanAttributes?.[
+                      LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_NAME
+                    ] ??
+                    spanAttributes["langfuse.prompt.name"] ??
+                    this.parseLangfusePromptFromAISDK(spanAttributes)?.name ??
+                    null,
+                  promptVersion:
+                    spanAttributes?.[
+                      LangfuseOtelSpanAttributes.OBSERVATION_PROMPT_VERSION
+                    ] ??
+                    spanAttributes["langfuse.prompt.version"] ??
+                    this.parseLangfusePromptFromAISDK(spanAttributes)
+                      ?.version ??
+                    null,
+
+                  modelParameters: this.extractModelParameters(
+                    spanAttributes,
+                    scopeSpan?.scope?.name ?? "",
+                  ),
+                  modelName: this.extractModelName(spanAttributes),
+                  completionStartTime: this.extractCompletionStartTime(
+                    spanAttributes,
+                    startTimeISO,
+                  ),
+
+                  // TODO: Usage details
+
+                  userId: this.extractUserId(spanAttributes),
+                  sessionId: this.extractSessionId(spanAttributes),
+
+                  ...this.extractInputAndOutput({
+                    events: span?.events ?? [],
+                    attributes: spanAttributes,
+                    instrumentationScopeName: scopeSpan?.scope?.name ?? "",
+                  }),
+
+                  // Metadata
+                  metadata,
+
+                  // Instrumentation metadata
+                  source: "otel",
+                  serviceName,
+                  serviceVersion,
+                  scopeName,
+                  scopeVersion,
+                  telemetrySdkLanguage,
+                  telemetrySdkName,
+                  telemetrySdkVersion,
+
+                  // Source data
+                  // eventRaw: stringifiedSpan,
+                  eventBytes: Buffer.byteLength(stringifiedSpan, "utf8"),
+                });
+              }
+            }
+
+            return events;
+          });
+      } catch (error) {
+        logger.error("Error processing OTEL spans to events:", error);
+        traceException(error, span);
+        throw error;
+      }
+    });
+  }
+
   /**
    * Process resource spans and convert them to Langfuse ingestion events.
    * Handles trace deduplication automatically using internal state.
@@ -551,9 +746,10 @@ export class OtelIngestionProcessor {
         metadata: {
           ...resourceAttributeMetadata,
           ...this.extractMetadata(attributes, "trace"),
-          ...(isLangfuseSDKSpans
-            ? {}
-            : { attributes: spanAttributesInMetadata }),
+          // removed to not remove trace metadata->attributes through subsequent observations
+          // ...(isLangfuseSDKSpans
+          //   ? {}
+          //   : { attributes: spanAttributesInMetadata }),
           resourceAttributes,
           scope: {
             ...(scopeSpan.scope || {}),
@@ -1025,6 +1221,14 @@ export class OtelIngestionProcessor {
       return { input, output };
     }
 
+    // LiveKit
+    input = attributes["lk.input_text"];
+    output =
+      attributes["lk.function_tool.output"] || attributes["lk.response.text"];
+    if (input || output) {
+      return { input, output };
+    }
+
     // Logfire uses single `events` array for GenAI events
     const eventsArray = attributes["events"];
     if (typeof eventsArray === "string" || Array.isArray(eventsArray)) {
@@ -1109,13 +1313,20 @@ export class OtelIngestionProcessor {
       };
     }
 
-    // OpenTelemetry (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans)
+    // OpenTelemetry messages (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans)
     input = attributes["gen_ai.input.messages"];
     output = attributes["gen_ai.output.messages"];
     if (input || output) {
       return { input, output };
     }
 
+    // OpenTelemetry tools (https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans)
+    input = attributes["gen_ai.tool.call.arguments"];
+    output = attributes["gen_ai.tool.call.result"];
+    if (input || output) {
+      return { input, output };
+    }
+
     return { input: null, output: null };
   }
 
@@ -1130,12 +1341,12 @@ export class OtelIngestionProcessor {
     ];
 
     for (const key of environmentAttributeKeys) {
-      if (resourceAttributes[key]) {
-        return resourceAttributes[key] as string;
-      }
       if (attributes[key]) {
         return attributes[key] as string;
       }
+      if (resourceAttributes[key]) {
+        return resourceAttributes[key] as string;
+      }
     }
 
     return "default";
@@ -1396,6 +1607,7 @@ export class OtelIngestionProcessor {
       LangfuseOtelSpanAttributes.OBSERVATION_MODEL,
       "gen_ai.request.model",
       "gen_ai.response.model",
+      "llm.response.model",
       "llm.model_name",
       "model",
     ];
@@ -1567,11 +1779,16 @@ export class OtelIngestionProcessor {
     startTimeISO?: string,
   ): string | null {
     try {
-      return JSON.parse(
-        attributes[
-          LangfuseOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME
-        ] as string,
-      );
+      const value = attributes[
+        LangfuseOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME
+      ] as any;
+
+      if (isValidDateString(value)) return value;
+
+      // Older SDKs have double stringified timestamps that need JSON parsing
+      // "\"2025-10-01T08:45:26.112648Z\""
+      const parsed = JSON.parse(value);
+      if (isValidDateString(parsed)) return parsed;
     } catch {
       // Fallthrough
     }

packages/shared/src/server/otel/utils.ts

@@ -0,0 +1,3 @@
+export function isValidDateString(dateString: string): boolean {
+  return !isNaN(new Date(dateString).getTime());
+}

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -6,7 +6,6 @@ export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
   tablePrefix?: string,
-  useTracesAmtCompatMode: boolean = false,
 ) => {
   const prefix = tablePrefix ? `${tablePrefix}.` : "";
 
@@ -15,11 +14,9 @@ export const clickhouseSearchCondition = (
     !searchType || searchType.includes("id")
       ? `${prefix}id ILIKE {searchString: String} OR t.user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
-    searchType && searchType.includes("content") && !useTracesAmtCompatMode
+    searchType && searchType.includes("content")
       ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
-      : searchType && searchType.includes("content") && useTracesAmtCompatMode
-        ? `finalizeAggregation(${prefix}input) ILIKE {searchString: String} OR finalizeAggregation(${prefix}output) ILIKE {searchString: String}`
-        : null,
+      : null,
   ].filter(Boolean);
 
   return {

packages/shared/src/server/queues.ts

@@ -42,6 +42,9 @@ export const BatchExportJobSchema = z.object({
   projectId: z.string(),
   batchExportId: z.string(),
 });
+export const CloudSpendAlertJobSchema = z.object({
+  orgId: z.string(),
+});
 export const TraceQueueEventSchema = z.object({
   projectId: z.string(),
   traceId: z.string(),
@@ -209,6 +212,7 @@ export type CreateEvalQueueEventType = z.infer<
   typeof CreateEvalQueueEventSchema
 >;
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
+export type CloudSpendAlertJobType = z.infer<typeof CloudSpendAlertJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
@@ -259,6 +263,8 @@ export enum QueueName {
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
   IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
+  CloudSpendAlertQueue = "cloud-spend-alert-queue",
+  CloudFreeTierUsageThresholdQueue = "cloud-free-tier-usage-threshold-queue",
   ExperimentCreate = "experiment-create-queue",
   PostHogIntegrationQueue = "posthog-integration-queue",
   PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
@@ -285,6 +291,8 @@ export enum QueueJobs {
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
+  CloudSpendAlertJob = "cloud-spend-alert-job",
+  CloudFreeTierUsageThresholdJob = "cloud-free-tier-usage-threshold-job",
   OtelIngestionJob = "otel-ingestion-job",
   IngestionJob = "ingestion-job",
   IngestionSecondaryJob = "secondary-ingestion-job",
@@ -429,4 +437,15 @@ export type TQueueJobTypes = {
     payload: EntityChangeEventType;
     name: QueueJobs.EntityChangeJob;
   };
+  [QueueName.CloudSpendAlertQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: CloudSpendAlertJobType;
+    name: QueueJobs.CloudSpendAlertJob;
+  };
+  [QueueName.CloudFreeTierUsageThresholdQueue]: {
+    timestamp: Date;
+    id: string;
+    name: QueueJobs.CloudFreeTierUsageThresholdJob;
+  };
 };

packages/shared/src/server/redis/cloudFreeTierUsageThresholdQueue.ts

@@ -0,0 +1,94 @@
+import { Queue } from "bullmq";
+import { env } from "../../env";
+import { QueueName, QueueJobs } from "../queues";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class CloudFreeTierUsageThresholdQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    // Only enable in cloud deployments with Stripe configured
+    if (!env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION) {
+      return null;
+    }
+
+    if (CloudFreeTierUsageThresholdQueue.instance) {
+      return CloudFreeTierUsageThresholdQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudFreeTierUsageThresholdQueue.instance = newRedis
+      ? new Queue(QueueName.CloudFreeTierUsageThresholdQueue, {
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.CloudFreeTierUsageThresholdQueue),
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudFreeTierUsageThresholdQueue.instance?.on("error", (err) => {
+      logger.error("[CloudFreeTierUsageThresholdQueue] error", err);
+    });
+
+    if (CloudFreeTierUsageThresholdQueue.instance) {
+      // Schedule recurring job - runs every hour at minute 35 (30 minutes after cloudUsageMetering at :05)
+      logger.info(
+        "[CloudFreeTierUsageThresholdQueue] Scheduling recurring job",
+        {
+          pattern: "35 * * * *",
+          jobId: "free-tier-usage-threshold-hourly",
+          description: "Every hour at minute 35",
+          timestamp: new Date().toISOString(),
+        },
+      );
+
+      CloudFreeTierUsageThresholdQueue.instance.add(
+        QueueJobs.CloudFreeTierUsageThresholdJob,
+        { type: "recurring" },
+        {
+          repeat: { pattern: "35 * * * *" },
+          // jobId: "free-tier-usage-threshold-hourly", // CRITICAL: Unique ID prevents duplicates across containers
+        },
+      );
+
+      // Optional: Bootstrap job for immediate execution on startup
+      // This ensures usage thresholds are processed immediately when service starts
+      logger.info(
+        "[CloudFreeTierUsageThresholdQueue] Scheduling bootstrap job (commented out for now)",
+        {
+          jobId: "free-tier-usage-threshold-bootstrap",
+          description: "Immediate execution on startup",
+          timestamp: new Date().toISOString(),
+        },
+      );
+
+      // Note: disabled for now
+      // ------------------------------------------------------------
+      // CloudFreeTierUsageThresholdQueue.instance.add(
+      //   QueueJobs.CloudFreeTierUsageThresholdJob,
+      //   { type: "bootstrap" },
+      //   {
+      //     jobId: "free-tier-usage-threshold-bootstrap", // CRITICAL: Unique ID prevents duplicates across containers
+      //   },
+      // );
+    }
+
+    return CloudFreeTierUsageThresholdQueue.instance;
+  }
+}

packages/shared/src/server/redis/cloudSpendAlertQueue.ts

@@ -0,0 +1,53 @@
+import { Queue } from "bullmq";
+import { env } from "../../env";
+import { QueueName } from "../queues";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class CloudSpendAlertQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (!env.STRIPE_SECRET_KEY) {
+      return null;
+    }
+
+    if (CloudSpendAlertQueue.instance) {
+      return CloudSpendAlertQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CloudSpendAlertQueue.instance = newRedis
+      ? new Queue(QueueName.CloudSpendAlertQueue, {
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.CloudSpendAlertQueue),
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CloudSpendAlertQueue.instance?.on("error", (err) => {
+      logger.error("CloudSpendAlertQueue error", err);
+    });
+
+    // Note: Jobs are triggered by the metering job with 5-minute delays
+    // No automatic scheduling needed
+
+    return CloudSpendAlertQueue.instance;
+  }
+}

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -46,6 +46,11 @@ export class CloudUsageMeteringQueue {
     });
 
     if (CloudUsageMeteringQueue.instance) {
+      logger.info("[CloudUsageMeteringQueue] Scheduling recurring job", {
+        pattern: "5 * * * *",
+        jobId: "cloud-usage-metering-recurring",
+        timestamp: new Date().toISOString(),
+      });
       CloudUsageMeteringQueue.instance.add(
         QueueJobs.CloudUsageMeteringJob,
         {},
@@ -55,11 +60,12 @@ export class CloudUsageMeteringQueue {
         },
       );
 
-      CloudUsageMeteringQueue.instance.add(
-        QueueJobs.CloudUsageMeteringJob,
-        {},
-        {},
-      );
+      logger.info("[CloudUsageMeteringQueue] Scheduling bootstrap job", {
+        jobId: "cloud-usage-metering-bootstrap",
+        timestamp: new Date().toISOString(),
+      });
+      // Bootstrap job to run immediately on startup
+      CloudUsageMeteringQueue.instance.add(QueueJobs.CloudUsageMeteringJob, {});
     }
 
     return CloudUsageMeteringQueue.instance;

packages/shared/src/server/redis/getQueue.ts

@@ -2,6 +2,8 @@ import { Queue } from "bullmq";
 import { QueueName } from "../queues";
 import { BatchExportQueue } from "./batchExport";
 import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
+import { CloudSpendAlertQueue } from "./cloudSpendAlertQueue";
+import { CloudFreeTierUsageThresholdQueue } from "./cloudFreeTierUsageThresholdQueue";
 import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
@@ -39,6 +41,10 @@ export function getQueue(
       return BatchExportQueue.getInstance();
     case QueueName.CloudUsageMeteringQueue:
       return CloudUsageMeteringQueue.getInstance();
+    case QueueName.CloudSpendAlertQueue:
+      return CloudSpendAlertQueue.getInstance();
+    case QueueName.CloudFreeTierUsageThresholdQueue:
+      return CloudFreeTierUsageThresholdQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
     case QueueName.DatasetDelete:

packages/shared/src/server/repositories/clickhouse.ts

@@ -15,6 +15,71 @@ import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { ClickHouseSettings } from "@clickhouse/client";
+
+/**
+ * Custom error class for ClickHouse resource-related errors
+ */
+// Error type configuration map
+const ERROR_TYPE_CONFIG: Record<
+  "MEMORY_LIMIT" | "OVERCOMMIT" | "TIMEOUT",
+  {
+    discriminators: string[];
+  }
+> = {
+  MEMORY_LIMIT: {
+    discriminators: ["memory limit exceeded"],
+  },
+  OVERCOMMIT: {
+    discriminators: ["OvercommitTracker"],
+  },
+  TIMEOUT: {
+    discriminators: ["Timeout", "timeout", "timed out"],
+  },
+};
+
+type ErrorType = keyof typeof ERROR_TYPE_CONFIG;
+
+export class ClickHouseResourceError extends Error {
+  static ERROR_ADVICE_MESSAGE = [
+    "Database resource limit exceeded.",
+    "Please use more specific filters or a shorter time range.",
+    "We are continuously improving our API performance.",
+  ].join(" ");
+
+  public readonly errorType: ErrorType;
+
+  constructor(errType: ErrorType, originalError: Error) {
+    super(originalError.message, { cause: originalError });
+    this.name = "ClickHouseResourceError";
+    this.errorType = errType;
+    // Preserve the original stack trace if available
+    if (originalError.stack) {
+      this.stack = originalError.stack;
+    }
+  }
+
+  static wrapIfResourceError(originalError: Error): Error {
+    const errorMessage = originalError.message || "";
+
+    for (const [type, config] of Object.entries(ERROR_TYPE_CONFIG) as Array<
+      [
+        keyof typeof ERROR_TYPE_CONFIG,
+        (typeof ERROR_TYPE_CONFIG)[keyof typeof ERROR_TYPE_CONFIG],
+      ]
+    >) {
+      const hasDiscriminator = config.discriminators.some((discriminator) =>
+        errorMessage.includes(discriminator),
+      );
+
+      if (hasDiscriminator) {
+        return new ClickHouseResourceError(type, originalError);
+      }
+    }
+
+    return originalError;
+  }
+}
 
 let s3StorageServiceClient: StorageService;
 
@@ -146,6 +211,7 @@ export async function* queryClickhouseStream<T>(opts: {
   clickhouseConfigs?: NodeClickHouseClientConfigOptions;
   tags?: Record<string, string>;
   preferredClickhouseService?: PreferredClickhouseService;
+  clickhouseSettings?: ClickHouseSettings;
 }): AsyncGenerator<T> {
   const tracer = getTracer("clickhouse-query-stream");
   const span = tracer.startSpan("clickhouse-query-stream", {
@@ -153,9 +219,8 @@ export async function* queryClickhouseStream<T>(opts: {
   });
 
   try {
-    const res = await context.with(
-      trace.setSpan(context.active(), span),
-      async () => {
+    const res = await context
+      .with(trace.setSpan(context.active(), span), async () => {
         // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
         span.setAttribute("ch.query.text", opts.query);
         span.setAttribute("db.system", "clickhouse");
@@ -170,6 +235,7 @@ export async function* queryClickhouseStream<T>(opts: {
           format: "JSONEachRow",
           query_params: opts.params,
           clickhouse_settings: {
+            ...opts.clickhouseSettings,
             log_comment: JSON.stringify(opts.tags ?? {}),
           },
         });
@@ -198,19 +264,58 @@ export async function* queryClickhouseStream<T>(opts: {
           }
         }
         return res;
-      },
-    );
+      })
+      .catch((error) => {
+        // Transform resource errors to provide actionable advice
+        throw ClickHouseResourceError.wrapIfResourceError(error as Error);
+      });
 
     for await (const rows of res.stream<T>()) {
       for (const row of rows) {
-        yield row.json();
+        yield handleExceptionRow(row.json());
       }
     }
+  } catch (error) {
+    // Also catch errors during streaming
+    throw ClickHouseResourceError.wrapIfResourceError(error as Error);
   } finally {
     span.end();
   }
 }
 
+/**
+ * ClickHouse has a quirk when it comes to handling exceptions mid response.
+ * It will simply output a row with "exception" key inside, which is indistinguishable from
+ * a query like `SELECT "my lovely string" AS exception;` may return.
+ *
+ * E.g.:
+ * ```
+ * {"exception":"Code: 395. DB::Exception: memory limit exceeded: would use l0.23 GiB"}
+ * ```
+ *
+ * This function makes the best effort to convert such rows into errors and throws them.
+ *
+ * See:
+ * - https://github.com/ClickHouse/clickhouse-js/issues/332
+ * - https://github.com/ClickHouse/ClickHouse/issues/75175
+ *
+ * Ideally this should get fixed in the future versions of ClickHouse.
+ */
+function handleExceptionRow<T>(parsedRow: T): T {
+  if (
+    typeof parsedRow === "object" &&
+    parsedRow !== null &&
+    Object.keys(parsedRow).length === 1 &&
+    "exception" in parsedRow
+  ) {
+    const potentialException = (parsedRow as { exception: string }).exception;
+    if (potentialException.match(/^Code: (\d+)/)) {
+      throw new Error(potentialException);
+    }
+  }
+  return parsedRow;
+}
+
 /**
  * Determines if an error is retryable (socket hang up, connection reset, etc.)
  */
@@ -229,6 +334,7 @@ export async function queryClickhouse<T>(opts: {
   clickhouseConfigs?: NodeClickHouseClientConfigOptions;
   tags?: Record<string, string>;
   preferredClickhouseService?: PreferredClickhouseService;
+  clickhouseSettings?: ClickHouseSettings;
 }): Promise<T[]> {
   return await instrumentAsync(
     { name: "clickhouse-query", spanKind: SpanKind.CLIENT },
@@ -250,6 +356,7 @@ export async function queryClickhouse<T>(opts: {
             format: "JSONEachRow",
             query_params: opts.params,
             clickhouse_settings: {
+              ...opts.clickhouseSettings,
               log_comment: JSON.stringify(opts.tags ?? {}),
             },
           });
@@ -279,7 +386,7 @@ export async function queryClickhouse<T>(opts: {
             }
           }
 
-          return await res.json<T>();
+          return (await res.json<T>()).map(handleExceptionRow);
         },
         {
           numOfAttempts: env.LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS,
@@ -313,7 +420,10 @@ export async function queryClickhouse<T>(opts: {
           timeMultiple: 1,
           maxDelay: 100,
         },
-      );
+      ).catch((error) => {
+        // Transform resource errors to provide actionable advice
+        throw ClickHouseResourceError.wrapIfResourceError(error as Error);
+      });
     },
   );
 }

packages/shared/src/server/repositories/dataset-run-items-converters.ts

@@ -2,7 +2,10 @@ import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
-import { DatasetRunItemRecordReadType } from "./definitions";
+import {
+  DatasetRunItemRecordReadType,
+  DatasetRunItemRecord,
+} from "./definitions";
 
 export const convertToDatasetRunMetrics = (row: any) => {
   return {
@@ -57,10 +60,19 @@ export const convertDatasetRunItemDomainToClickhouse = (
   };
 };
 
-export const convertDatasetRunItemClickhouseToDomain = (
-  row: DatasetRunItemRecordReadType,
-): DatasetRunItemDomain => {
-  return {
+// Function overloads for clean type discrimination
+export function convertDatasetRunItemClickhouseToDomain(
+  // eslint-disable-next-line no-unused-vars
+  row: DatasetRunItemRecord<true>,
+): DatasetRunItemDomain<true>;
+export function convertDatasetRunItemClickhouseToDomain(
+  // eslint-disable-next-line no-unused-vars
+  row: DatasetRunItemRecord<false>,
+): DatasetRunItemDomain<false>;
+export function convertDatasetRunItemClickhouseToDomain<
+  WithIO extends boolean = true,
+>(row: DatasetRunItemRecord<WithIO>): DatasetRunItemDomain<WithIO> {
+  const baseConversion = {
     id: row.id,
     projectId: row.project_id,
     traceId: row.trace_id,
@@ -71,17 +83,27 @@ export const convertDatasetRunItemClickhouseToDomain = (
     datasetRunCreatedAt: parseClickhouseUTCDateTimeFormat(
       row.dataset_run_created_at,
     ),
-    datasetRunMetadata:
-      parseMetadataCHRecordToDomain(row.dataset_run_metadata) ?? null,
     datasetItemId: row.dataset_item_id,
-    datasetItemInput: row.dataset_item_input,
-    datasetItemExpectedOutput: row.dataset_item_expected_output,
-    datasetItemMetadata: parseMetadataCHRecordToDomain(
-      row.dataset_item_metadata,
-    ),
     createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
     datasetId: row.dataset_id,
     error: row.error ?? null,
   };
-};
+
+  // Check if row has IO fields at runtime, typescript does not support conditional types without runtime checks
+  if ("dataset_item_input" in row) {
+    return {
+      ...baseConversion,
+      datasetRunMetadata:
+        parseMetadataCHRecordToDomain((row as any).dataset_run_metadata) ??
+        null,
+      datasetItemInput: (row as any).dataset_item_input,
+      datasetItemExpectedOutput: (row as any).dataset_item_expected_output,
+      datasetItemMetadata: parseMetadataCHRecordToDomain(
+        (row as any).dataset_item_metadata,
+      ),
+    } as DatasetRunItemDomain<WithIO>;
+  } else {
+    return baseConversion as DatasetRunItemDomain<WithIO>;
+  }
+}

packages/shared/src/server/repositories/dataset-run-items.ts

@@ -15,12 +15,13 @@ import {
   queryClickhouse,
 } from "./clickhouse";
 import { convertDatasetRunItemClickhouseToDomain } from "./dataset-run-items-converters";
-import { DatasetRunItemRecordReadType } from "./definitions";
+import { DatasetRunItemRecord } from "./definitions";
 import { env } from "../../env";
 import { commandClickhouse } from "./clickhouse";
 import Decimal from "decimal.js";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { ScoreAggregate } from "../../features/scores";
 
 type DatasetItemIdsByTraceIdQuery = {
   projectId: string;
@@ -39,6 +40,23 @@ type DatasetRunItemsTableQuery = {
   clickhouseConfigs?: ClickHouseClientConfigOptions;
 };
 
+type BaseDatasetItemWithRunDataQuery = {
+  projectId: string;
+  datasetId: string;
+  runIds: string[];
+  filterByRun: {
+    runId: string;
+    filters: FilterState;
+  }[];
+};
+
+type DatasetItemIdsWithRunDataQuery = BaseDatasetItemWithRunDataQuery & {
+  limit?: number;
+  offset?: number;
+};
+
+type DatasetItemsWithRunDataCountQuery = BaseDatasetItemWithRunDataQuery;
+
 type DatasetRunItemsByDatasetIdQuery = Omit<
   DatasetRunItemsTableQuery,
   "datasetId"
@@ -57,6 +75,17 @@ type DatasetRunsMetricsTableQuery = {
   offset?: number;
 };
 
+type BaseDatasetRunItemsWithoutIOQuery = {
+  projectId: string;
+  datasetId: string;
+  runIds: string[];
+};
+
+type DatasetRunItemsByItemIdsWithoutIOQuery =
+  BaseDatasetRunItemsWithoutIOQuery & {
+    datasetItemIds: string[];
+  };
+
 export type DatasetRunsMetrics = {
   id: string;
   name: string;
@@ -103,6 +132,27 @@ type DatasetRunsRowsRecordType = {
   dataset_run_metadata: string;
 };
 
+export type EnrichedDatasetRunItem = {
+  id: string;
+  createdAt: Date;
+  datasetItemId: string;
+  datasetRunId: string;
+  datasetRunName: string;
+  observation:
+    | {
+        id: string;
+        latency: number;
+        calculatedTotalCost: Decimal;
+      }
+    | undefined;
+  trace: {
+    id: string;
+    duration: number;
+    totalCost: number;
+  };
+  scores: ScoreAggregate;
+};
+
 const convertDatasetRunsMetricsRecord = (
   record: DatasetRunsMetricsRecordType,
 ): DatasetRunsMetrics => {
@@ -469,13 +519,195 @@ export const getDatasetRunsTableCountCh = async (
   return Number(rows[0]?.count);
 };
 
-const getDatasetRunItemsTableInternal = async <T>(
-  opts: DatasetRunItemsTableQuery & {
+type GetDatasetRunItemsTableOpts<IncludeIO extends boolean> =
+  DatasetRunItemsTableQuery & {
     select: "count" | "rows";
     tags: Record<string, string>;
-  },
+    includeIO?: IncludeIO;
+  };
+
+// Phase 1: Find dataset item IDs or count that satisfy conditions across ALL runs
+const getQualifyingDatasetItems = async <T>(opts: {
+  select: "count" | "rows";
+  projectId: string;
+  datasetId: string;
+  runIds: string[];
+  runFilters: {
+    runId: string;
+    filters: FilterState;
+  }[];
+  limit?: number;
+  offset?: number;
+}): Promise<Array<T>> => {
+  const { select, projectId, datasetId, runIds, runFilters, limit, offset } =
+    opts;
+
+  // Build base filter (project + dataset only)
+  const { datasetRunItemsFilter: baseDatasetRunItemsFilter } =
+    getProjectDatasetIdDefaultFilter(projectId, datasetId);
+  const baseFilter = baseDatasetRunItemsFilter.apply();
+
+  // Build run-specific conditions for the intersection query
+  const runFilterResults = runFilters.map((runFilter) => {
+    const { runId, filters: filterState } = runFilter;
+
+    // Create run ID condition
+    const runConditionFilter = new StringFilter({
+      clickhouseTable: "dataset_run_items_rmt",
+      field: "dataset_run_id",
+      operator: "=",
+      value: runId,
+    });
+
+    // Create user filters for this run
+    const userFilters = createFilterFromFilterState(
+      filterState,
+      datasetRunItemsTableUiColumnDefinitions,
+    );
+
+    // Combine run condition with user filters using AND and apply immediately
+    const runFilterList = new FilterList([runConditionFilter, ...userFilters]);
+    return runFilterList.apply();
+  });
+
+  // add empty filters for the runs that have no filters
+  runIds.forEach((runId) => {
+    if (runFilters.find((runFilter) => runFilter.runId === runId)) {
+      return;
+    }
+    // Create run ID condition
+    const runConditionFilter = new FilterList([
+      new StringFilter({
+        clickhouseTable: "dataset_run_items_rmt",
+        field: "dataset_run_id",
+        operator: "=",
+        value: runId,
+      }),
+    ]);
+    runFilterResults.push(runConditionFilter.apply());
+  });
+
+  const combinedQuery = `(${runFilterResults.map((result) => `(${result.query})`).join(" OR ")})`;
+
+  const intersectionQuery =
+    runFilters.length > 0
+      ? `HAVING COUNT(DISTINCT dataset_run_id) = {totalRunCount: UInt32}`
+      : "";
+
+  // Check if any run has score filters for CTE
+  const hasScoresFilter = runFilters
+    .flatMap((f) => f.filters)
+    .some((f) => f.column.toLowerCase().includes("score"));
+
+  // Build scores filter
+  const scoresFilter = new FilterList([
+    new StringFilter({
+      clickhouseTable: "scores",
+      field: "project_id",
+      operator: "=",
+      value: projectId,
+    }),
+  ]);
+  const appliedScoresFilter = scoresFilter.apply();
+
+  const selectString =
+    select === "count"
+      ? "COUNT(DISTINCT dataset_item_id) as count"
+      : "dataset_item_id";
+
+  // Build the intersection query
+  const scoresCte = hasScoresFilter
+    ? `
+  WITH scores_aggregated AS (
+     SELECT
+       dri.dataset_run_id,
+       dri.project_id,
+       dri.trace_id,
+       -- For numeric scores, use tuples of (name, avg_value)
+       groupArrayIf(
+         tuple(s.name, s.avg_value),
+         s.data_type IN ('NUMERIC', 'BOOLEAN')
+       ) AS scores_avg,
+       -- For categorical scores, use name:value format for improved query performance
+       groupArrayIf(
+         concat(s.name, ':', s.string_value),
+         s.data_type = 'CATEGORICAL' AND notEmpty(s.string_value)
+       ) AS score_categories
+     FROM dataset_run_items_rmt dri
+     LEFT JOIN (
+       SELECT
+         project_id,
+         trace_id,
+         name,
+         data_type,
+         string_value,
+         avg(value) as avg_value
+       FROM scores s FINAL
+       WHERE ${appliedScoresFilter.query}
+       GROUP BY
+         project_id,
+         trace_id,
+         name,
+         data_type,
+         string_value
+     ) s ON s.project_id = dri.project_id AND s.trace_id = dri.trace_id
+     WHERE ${baseFilter.query}
+     GROUP BY dri.project_id, dri.dataset_id, dri.dataset_run_id, dri.trace_id
+   ),
+   `
+    : "WITH ";
+
+  const query = `
+    ${scoresCte}
+    run_qualified_items AS (
+      SELECT DISTINCT dri.dataset_item_id, dri.dataset_run_id
+      FROM dataset_run_items_rmt dri
+      ${hasScoresFilter ? `LEFT JOIN scores_aggregated sa ON dri.dataset_run_id = sa.dataset_run_id AND dri.project_id = sa.project_id AND dri.trace_id = sa.trace_id` : ""}
+      WHERE ${baseFilter.query}
+      AND ${combinedQuery}
+    ),
+    intersection_items AS (
+      SELECT dataset_item_id
+      FROM run_qualified_items
+      GROUP BY dataset_item_id
+      ${intersectionQuery}
+    )
+    SELECT 
+      ${selectString}
+    FROM intersection_items
+    ${select === "count" ? "" : "ORDER BY dataset_item_id -- for consistent pagination"}
+    ${limit !== undefined && offset !== undefined ? `LIMIT ${limit} OFFSET ${offset}` : ""};`;
+
+  const res = await queryClickhouse<T>({
+    query,
+    params: {
+      ...baseFilter.params,
+      ...(hasScoresFilter ? appliedScoresFilter.params : {}),
+      totalRunCount: runIds.length,
+      ...runFilterResults.reduce((acc, result) => {
+        return { ...acc, ...result.params };
+      }, {}),
+      ...(limit !== undefined && offset !== undefined ? { limit, offset } : {}),
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      projectId,
+      datasetId,
+    },
+  });
+
+  return res;
+};
+
+const getDatasetRunItemsTableInternal = async <
+  T,
+  IncludeIO extends boolean = true,
+>(
+  opts: GetDatasetRunItemsTableOpts<IncludeIO>,
 ): Promise<Array<T>> => {
-  const { projectId, datasetId, filter, orderBy, limit, offset } = opts;
+  const { projectId, datasetId, filter, orderBy, limit, offset, includeIO } =
+    opts;
 
   let selectString = "";
 
@@ -498,11 +730,11 @@ const getDatasetRunItemsTableInternal = async <T>(
       dri.updated_at as updated_at,
       dri.dataset_run_name as dataset_run_name,
       dri.dataset_run_description as dataset_run_description,
-      dri.dataset_run_metadata as dataset_run_metadata,
       dri.dataset_run_created_at as dataset_run_created_at,
-      dri.dataset_item_input as dataset_item_input,
-      dri.dataset_item_expected_output as dataset_item_expected_output,
-      dri.dataset_item_metadata as dataset_item_metadata,
+      ${includeIO ? "dri.dataset_run_metadata as dataset_run_metadata, " : ""}
+      ${includeIO ? "dri.dataset_item_input as dataset_item_input, " : ""}
+      ${includeIO ? "dri.dataset_item_expected_output as dataset_item_expected_output, " : ""}
+      ${includeIO ? "dri.dataset_item_metadata as dataset_item_metadata, " : ""}
       dri.is_deleted as is_deleted,
       dri.event_ts as event_ts`;
       break;
@@ -651,27 +883,87 @@ const getDatasetRunItemsTableInternal = async <T>(
 export const getDatasetRunItemsCh = async (
   opts: DatasetRunItemsTableQuery,
 ): Promise<DatasetRunItemDomain[]> => {
-  const rows =
-    await getDatasetRunItemsTableInternal<DatasetRunItemRecordReadType>({
-      ...opts,
-      select: "rows",
-      tags: { kind: "list" },
-    });
+  const rows = await getDatasetRunItemsTableInternal<DatasetRunItemRecord>({
+    ...opts,
+    select: "rows",
+    tags: { kind: "list" },
+  });
 
-  return rows.map(convertDatasetRunItemClickhouseToDomain);
+  return rows.map((row) => convertDatasetRunItemClickhouseToDomain(row));
 };
 
 export const getDatasetRunItemsByDatasetIdCh = async (
   opts: DatasetRunItemsByDatasetIdQuery,
 ): Promise<DatasetRunItemDomain[]> => {
-  const rows =
-    await getDatasetRunItemsTableInternal<DatasetRunItemRecordReadType>({
-      ...opts,
-      select: "rows",
-      tags: { kind: "list" },
-    });
+  const rows = await getDatasetRunItemsTableInternal<DatasetRunItemRecord>({
+    ...opts,
+    select: "rows",
+    tags: { kind: "list" },
+  });
 
-  return rows.map(convertDatasetRunItemClickhouseToDomain);
+  return rows.map((row) => convertDatasetRunItemClickhouseToDomain(row));
+};
+
+export const getDatasetItemsWithRunDataCount = async (
+  opts: DatasetItemsWithRunDataCountQuery,
+): Promise<number> => {
+  const { projectId, datasetId, runIds, filterByRun } = opts;
+
+  const rows = await getQualifyingDatasetItems<{ count: string }>({
+    select: "count",
+    projectId,
+    datasetId,
+    runIds,
+    runFilters: filterByRun,
+  });
+
+  return Number(rows[0]?.count);
+};
+
+export const getDatasetItemIdsWithRunData = async (
+  opts: DatasetItemIdsWithRunDataQuery,
+): Promise<string[]> => {
+  const rows = await getQualifyingDatasetItems<{ dataset_item_id: string }>({
+    select: "rows",
+    runFilters: opts.filterByRun,
+    ...opts,
+  });
+
+  return rows.map((row) => row.dataset_item_id);
+};
+
+export const getDatasetRunItemsWithoutIOByItemIds = async (
+  opts: DatasetRunItemsByItemIdsWithoutIOQuery,
+): Promise<DatasetRunItemDomain<false>[]> => {
+  // Step 1: Get DRI data matching [datasetId, runId, datasetItemId]
+  const { datasetItemIds, runIds, ...rest } = opts;
+
+  const filter: FilterState = [
+    {
+      column: "datasetItemId",
+      operator: "any of",
+      value: datasetItemIds,
+      type: "stringOptions" as const,
+    },
+    {
+      column: "datasetRunId",
+      operator: "any of",
+      value: runIds,
+      type: "stringOptions" as const,
+    },
+  ];
+  const rows = await getDatasetRunItemsTableInternal<
+    DatasetRunItemRecord<false>,
+    false
+  >({
+    ...rest,
+    filter,
+    select: "rows",
+    tags: { kind: "list" },
+  });
+
+  // Step 2: Convert to domain
+  return rows.map((row) => convertDatasetRunItemClickhouseToDomain(row));
 };
 
 export const getDatasetItemIdsByTraceIdCh = async (

packages/shared/src/server/repositories/definitions.ts

@@ -225,6 +225,17 @@ const datasetRunItemRecordReadSchema = datasetRunItemRecordBaseSchema.extend({
 export type DatasetRunItemRecordReadType = z.infer<
   typeof datasetRunItemRecordReadSchema
 >;
+// Conditional type for dataset run item records with optional IO
+export type DatasetRunItemRecord<WithIO extends boolean = true> =
+  WithIO extends true
+    ? DatasetRunItemRecordReadType
+    : Omit<
+        DatasetRunItemRecordReadType,
+        | "dataset_run_metadata"
+        | "dataset_item_input"
+        | "dataset_item_expected_output"
+        | "dataset_item_metadata"
+      >;
 
 export const datasetRunItemRecordInsertSchema =
   datasetRunItemRecordBaseSchema.extend({
@@ -496,3 +507,94 @@ export const convertPostgresScoreToInsert = (
     is_deleted: 0,
   };
 };
+
+export const eventRecordBaseSchema = z.object({
+  // Identifiers
+  org_id: z.string().nullish(),
+  project_id: z.string(),
+  trace_id: z.string(),
+  span_id: z.string(),
+  // We mainly use the id for compatibility with old events that always had a `id` column.
+  id: z.string(), // same as span_id. Needs to be set manually.
+  parent_span_id: z.string().nullish(),
+
+  // Core properties
+  name: z.string(),
+  type: z.string(),
+  environment: z.string().default("default"),
+  version: z.string().nullish(),
+
+  user_id: z.string().nullish(),
+  session_id: z.string().nullish(),
+
+  level: z.string(),
+  status_message: z.string().nullish(),
+
+  // Prompt
+  prompt_id: z.string().nullish(),
+  prompt_name: z.string().nullish(),
+  prompt_version: z.string().nullish(),
+
+  // Model
+  model_id: z.string().nullish(),
+  provided_model_name: z.string().nullish(),
+  model_parameters: z.string().nullish(),
+
+  // Usage & Cost
+  provided_usage_details: UsageCostSchema,
+  usage_details: UsageCostSchema,
+  provided_cost_details: UsageCostSchema,
+  cost_details: UsageCostSchema,
+  total_cost: z.number().nullish(),
+
+  // I/O
+  input: z.string().nullish(),
+  output: z.string().nullish(),
+
+  // Metadata - multiple approaches supported
+  metadata: z.record(z.string(), z.string()),
+  metadata_names: z.array(z.string()).default([]),
+  metadata_values: z.array(z.any()).default([]),
+  metadata_string_names: z.array(z.string()).default([]),
+  metadata_string_values: z.array(z.string()).default([]),
+  metadata_number_names: z.array(z.string()).default([]),
+  metadata_number_values: z.array(z.number()).default([]),
+  metadata_bool_names: z.array(z.string()).default([]),
+  metadata_bool_values: z.array(z.number()).default([]),
+
+  // Source metadata (Instrumentation)
+  source: z.string(),
+  service_name: z.string().nullish(),
+  service_version: z.string().nullish(),
+  scope_name: z.string().nullish(),
+  scope_version: z.string().nullish(),
+  telemetry_sdk_language: z.string().nullish(),
+  telemetry_sdk_name: z.string().nullish(),
+  telemetry_sdk_version: z.string().nullish(),
+
+  // Generic props
+  blob_storage_file_path: z.string(),
+  event_raw: z.string(),
+  event_bytes: z.number(),
+  is_deleted: z.number(),
+});
+
+export const eventRecordReadSchema = eventRecordBaseSchema.extend({
+  start_time: clickhouseStringDateSchema,
+  end_time: clickhouseStringDateSchema.nullish(),
+  completion_start_time: clickhouseStringDateSchema.nullish(),
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+  event_ts: clickhouseStringDateSchema,
+});
+export type EventRecordReadType = z.infer<typeof eventRecordReadSchema>;
+
+export const eventRecordInsertSchema = eventRecordBaseSchema.extend({
+  start_time: z.number(),
+  end_time: z.number().nullish(),
+  completion_start_time: z.number().nullish(),
+  created_at: z.number(),
+  updated_at: z.number(),
+  event_ts: z.number(),
+});
+export type EventRecordInsertType = z.infer<typeof eventRecordInsertSchema>;

packages/shared/src/server/repositories/observations.ts

@@ -23,7 +23,7 @@ import {
   observationsTableUiColumnDefinitions,
 } from "../tableMappings";
 import { OrderByState } from "../../interfaces/orderBy";
-import { getTimeframesTracesAMT, getTracesByIds } from "./traces";
+import { getTracesByIds } from "./traces";
 import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import {
   convertDateToClickhouseDateTime,
@@ -167,7 +167,7 @@ export const getObservationsForTrace = async <IncludeIO extends boolean>(
     created_at,
     updated_at,
     event_ts
-  FROM observations 
+  FROM observations
   WHERE trace_id = {traceId: String}
   AND project_id = {projectId: String}
    ${timestamp ? `AND start_time >= {traceTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
@@ -282,7 +282,7 @@ export const getObservationForTraceIdByName = async ({
     created_at,
     updated_at,
     event_ts
-  FROM observations 
+  FROM observations
   WHERE trace_id = {traceId: String}
   AND project_id = {projectId: String}
   AND name = {name: String}
@@ -753,7 +753,7 @@ const getObservationsTableInternal = async <T>(
         trace_id
       ) tmp
     GROUP BY
-      trace_id, 
+      trace_id,
       observation_id
   )`;
 
@@ -781,11 +781,11 @@ const getObservationsTableInternal = async <T>(
       ${scoresCte}
       SELECT
        ${selectString}
-      FROM observations o 
+      FROM observations o
         ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN __TRACE_TABLE__ t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
         ${hasScoresFilter ? `LEFT JOIN scores_agg AS s ON s.trace_id = o.trace_id and s.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
-        
+
         ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
       ${chOrderBy}
@@ -795,7 +795,6 @@ const getObservationsTableInternal = async <T>(
   return measureAndReturn({
     operationName: "getObservationsTableInternal",
     projectId,
-    minStartTime: (timeFilter?.value as Date) || undefined,
     input: {
       params: {
         ...appliedScoresFilter.params,
@@ -818,22 +817,11 @@ const getObservationsTableInternal = async <T>(
         operation_name: "getObservationsTableInternal",
       },
     },
-    existingExecution: async (input) => {
+    fn: async (input) => {
       return queryClickhouse<T>({
         query: query.replace("__TRACE_TABLE__", "traces"),
         params: input.params,
-        tags: { ...input.tags, experiment_amt: "original" },
-        clickhouseConfigs,
-      });
-    },
-    newExecution: async (input) => {
-      const traceAmt = getTimeframesTracesAMT(
-        (timeFilter?.value as Date) || undefined,
-      );
-      return queryClickhouse<T>({
-        query: query.replace("__TRACE_TABLE__", traceAmt),
-        params: input.params,
-        tags: { ...input.tags, experiment_amt: "new" },
+        tags: input.tags,
         clickhouseConfigs,
       });
     },
@@ -1225,9 +1213,9 @@ export const getObservationMetricsForPrompts = async (
                   dateDiff('millisecond', start_time, end_time) AS latency_ms
               FROM observations
               FINAL
-              WHERE (type = 'GENERATION') 
-              AND (prompt_name IS NOT NULL) 
-              AND project_id={projectId: String} 
+              WHERE (type = 'GENERATION')
+              AND (prompt_name IS NOT NULL)
+              AND project_id={projectId: String}
               AND prompt_id IN ({promptIds: Array(String)})
           )
       SELECT
@@ -1293,9 +1281,9 @@ export const getLatencyAndTotalCostForObservations = async (
         id,
         cost_details['total'] AS total_cost,
         dateDiff('millisecond', start_time, end_time) AS latency_ms
-    FROM observations FINAL 
-    WHERE project_id = {projectId: String} 
-    AND id IN ({observationIds: Array(String)}) 
+    FROM observations FINAL
+    WHERE project_id = {projectId: String}
+    AND id IN ({observationIds: Array(String)})
     ${timestamp ? `AND start_time >= {timestamp: DateTime64(3)}` : ""}
 `;
   const rows = await queryClickhouse<{
@@ -1337,7 +1325,7 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
         sumMap(cost_details)['total'] AS total_cost,
         dateDiff('millisecond', min(start_time), max(end_time)) AS latency_ms
     FROM observations FINAL
-    WHERE project_id = {projectId: String} 
+    WHERE project_id = {projectId: String}
     AND trace_id IN ({traceIds: Array(String)})
     ${timestamp ? `AND start_time >= {timestamp: DateTime64(3)}` : ""}
     GROUP BY trace_id
@@ -1378,7 +1366,7 @@ export const getObservationCountsByProjectInCreationInterval = async ({
   end: Date;
 }) => {
   const query = `
-    SELECT 
+    SELECT
       project_id,
       count(*) as count
     FROM observations
@@ -1414,7 +1402,7 @@ export const getObservationCountOfProjectsSinceCreationDate = async ({
   start: Date;
 }) => {
   const query = `
-    SELECT 
+    SELECT
       count(*) as count
     FROM observations
     WHERE project_id IN ({projectIds: Array(String)})
@@ -1442,7 +1430,7 @@ export const getTraceIdsForObservations = async (
   observationIds: string[],
 ) => {
   const query = `
-    SELECT 
+    SELECT
       trace_id,
       id
     FROM observations
@@ -1531,14 +1519,7 @@ export const getGenerationsForPostHog = async function* (
   minTimestamp: Date,
   maxTimestamp: Date,
 ) {
-  // Determine which trace table to use based on experiment flag
-  const useAMT = env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true";
-  // Subtract 7d from minTimestamp to account for shift in query
-  const traceTable = useAMT
-    ? getTimeframesTracesAMT(
-        new Date(minTimestamp.getTime() - 7 * 24 * 60 * 60 * 1000),
-      )
-    : "traces";
+  const traceTable = "traces";
 
   const query = `
     SELECT
@@ -1586,7 +1567,6 @@ export const getGenerationsForPostHog = async function* (
       type: "observation",
       kind: "analytic",
       projectId,
-      experiment_amt: useAMT ? "new" : "original",
     },
     clickhouseConfigs: {
       request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
@@ -1635,3 +1615,66 @@ export const getGenerationsForPostHog = async function* (
     };
   }
 };
+
+/**
+ * Get observation counts grouped by project and day within a date range.
+ *
+ * Returns one row per project per day with the count of observations started on that day.
+ * Uses half-open interval [startDate, endDate) for filtering based on start_time.
+ *
+ * @param startDate - Start of date range (inclusive)
+ * @param endDate - End of date range (exclusive)
+ * @returns Array of { count, projectId, date } objects
+ *
+ * @example
+ * // Get observation counts for March 1-2, 2024
+ * const counts = await getObservationCountsByProjectAndDay({
+ *   startDate: new Date('2024-03-01T00:00:00Z'),
+ *   endDate: new Date('2024-03-03T00:00:00Z')
+ * });
+ *
+ * Note: Skips using FINAL (double counting risk) for faster and cheaper
+ * queries against clickhouse. Generous 4x overcompensation before blocking allows
+ * for usage aggregation to be meaningful.
+ */
+export const getObservationCountsByProjectAndDay = async ({
+  startDate,
+  endDate,
+}: {
+  startDate: Date;
+  endDate: Date;
+}) => {
+  const query = `
+    SELECT
+      count(*) as count,
+      project_id,
+      toDate(start_time) as date
+    FROM observations
+    WHERE start_time >= {startDate: DateTime64(3)}
+    AND start_time < {endDate: DateTime64(3)}
+    GROUP BY project_id, toDate(start_time)
+  `;
+
+  const rows = await queryClickhouse<{
+    count: string;
+    project_id: string;
+    date: string;
+  }>({
+    query,
+    params: {
+      startDate: convertDateToClickhouseDateTime(startDate),
+      endDate: convertDateToClickhouseDateTime(endDate),
+    },
+    tags: {
+      feature: "tracing",
+      type: "observation",
+      kind: "analytic",
+    },
+  });
+
+  return rows.map((row) => ({
+    count: Number(row.count),
+    projectId: row.project_id,
+    date: row.date,
+  }));
+};

packages/shared/src/server/repositories/scores.ts

@@ -36,7 +36,6 @@ import { ClickHouseClientConfigOptions } from "@clickhouse/client";
 import { recordDistribution } from "../instrumentation";
 import { prisma } from "../../db";
 import { measureAndReturn } from "../clickhouse/measureAndReturn";
-import { getTimeframesTracesAMT } from "./traces";
 import { scoresColumnsTableUiColumnDefinitions } from "../tableMappings/mapScoresColumnsTable";
 
 export const searchExistingAnnotationScore = async (
@@ -217,11 +216,11 @@ export const getScoresForSessions = async <
   const select = formatMetadataSelect(excludeMetadata, includeHasMetadata);
 
   const query = `
-      select 
+      select
         ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
-      AND s.session_id IN ({sessionIds: Array(String)}) 
+      AND s.session_id IN ({sessionIds: Array(String)})
       ORDER BY s.event_ts DESC
       LIMIT 1 BY s.id, s.project_id
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
@@ -266,11 +265,11 @@ export const getScoresForDatasetRuns = async <
   const select = formatMetadataSelect(excludeMetadata, includeHasMetadata);
 
   const query = `
-      select 
+      select
         ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
-      AND s.dataset_run_id IN ({runIds: Array(String)}) 
+      AND s.dataset_run_id IN ({runIds: Array(String)})
       ORDER BY s.event_ts DESC
       LIMIT 1 BY s.id, s.project_id
       ${limit && offset ? `limit {limit: Int32} offset {offset: Int32}` : ""}
@@ -303,7 +302,7 @@ export const getTraceScoresForDatasetRuns = async (
   if (datasetRunIds.length === 0) return [];
 
   const query = `
-    SELECT 
+    SELECT
       s.id as id,
       s.timestamp as timestamp,
       s.project_id as project_id,
@@ -324,11 +323,11 @@ export const getTraceScoresForDatasetRuns = async (
       s.created_at as created_at,
       s.updated_at as updated_at,
       s.event_ts as event_ts,
-      s.is_deleted as is_deleted, 
+      s.is_deleted as is_deleted,
       length(mapKeys(s.metadata)) > 0 AS has_metadata,
       dri.dataset_run_id as run_id
-    FROM dataset_run_items_rmt dri 
-    JOIN scores s FINAL ON dri.trace_id = s.trace_id 
+    FROM dataset_run_items_rmt dri
+    JOIN scores s FINAL ON dri.trace_id = s.trace_id
       AND dri.project_id = s.project_id
     WHERE dri.project_id = {projectId: String}
       AND dri.dataset_run_id IN {datasetRunIds: Array(String)}
@@ -389,7 +388,7 @@ export const getScoresForTraces = async <
         ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
-      AND s.trace_id IN ({traceIds: Array(String)}) 
+      AND s.trace_id IN ({traceIds: Array(String)})
       ${timestamp ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
       ORDER BY s.event_ts DESC
       LIMIT 1 BY s.id, s.project_id
@@ -487,7 +486,7 @@ export const getScoresForObservations = async <
     .join(", ");
 
   const query = `
-      select 
+      select
         ${select}
       from scores s
       WHERE s.project_id = {projectId: String}
@@ -562,7 +561,7 @@ export const getScoresGroupedByNameSourceType = async ({
   // Therefore, we can skip final as some inaccuracy in count is acceptable.
 
   const query = `
-    select 
+    select
       s.name as name,
       s.source as source,
       s.data_type as data_type
@@ -630,7 +629,7 @@ export const getNumericScoresGroupedByName = async (
   // We mainly use queries like this to retrieve filter options.
   // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-      select 
+      select
         name as name
       from scores s
       WHERE s.project_id = {projectId: String}
@@ -949,10 +948,10 @@ const getScoresUiGeneric = async <T>(props: {
     scoresFilter.some((f) => f.clickhouseTable === "traces");
 
   const query = `
-      SELECT 
+      SELECT
           ${select}
       FROM scores s final
-      ${performTracesJoin ? "LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
+      ${performTracesJoin ? "LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id" : ""}
       WHERE s.project_id = {projectId: String}
       ${scoresFilterRes?.query ? `AND ${scoresFilterRes.query}` : ""}
       ${orderByToClickhouseSql(orderBy ?? null, scoresTableUiColumnDefinitions)}
@@ -978,19 +977,11 @@ const getScoresUiGeneric = async <T>(props: {
         operation_name: "getScoresUiGeneric",
       },
     },
-    existingExecution: async (input) => {
+    fn: async (input) => {
       return queryClickhouse<T>({
-        query: query.replace("__TRACE_TABLE__", "traces"),
+        query,
         params: input.params,
-        tags: { ...input.tags, experiment_amt: "original" },
-        clickhouseConfigs,
-      });
-    },
-    newExecution: async (input) => {
-      return queryClickhouse<T>({
-        query: query.replace("__TRACE_TABLE__", "traces_all_amt"),
-        params: input.params,
-        tags: { ...input.tags, experiment_amt: "new" },
+        tags: input.tags,
         clickhouseConfigs,
       });
     },
@@ -1012,7 +1003,7 @@ export const getScoreNames = async (
   // We mainly use queries like this to retrieve filter options.
   // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-      select 
+      select
         name,
         count(*) as count
       from scores s
@@ -1161,7 +1152,7 @@ export const getNumericScoreHistogram = async (
   const query = `
     select s.value
     from scores s
-    ${traceFilter ? `LEFT JOIN __TRACE_TABLE__ t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
+    ${traceFilter ? `LEFT JOIN traces t ON s.trace_id = t.id AND t.project_id = s.project_id` : ""}
     WHERE s.project_id = {projectId: String}
     ${traceFilter ? `AND t.project_id = {projectId: String}` : ""}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
@@ -1170,16 +1161,9 @@ export const getNumericScoreHistogram = async (
     ${limit !== undefined ? `limit {limit: Int32}` : ""}
   `;
 
-  // Extract timestamp from filter for AMT table selection
-  const timestampFilter = chFilter.find(
-    (f) => f.clickhouseTable === "traces" && f.field === "timestamp",
-  ) as TimeFilter | undefined;
-  const timestamp = timestampFilter?.value;
-
   return measureAndReturn({
     operationName: "getNumericScoreHistogram",
     projectId,
-    minStartTime: timestamp,
     input: {
       params: {
         projectId,
@@ -1193,21 +1177,12 @@ export const getNumericScoreHistogram = async (
         projectId,
         operation_name: "getNumericScoreHistogram",
       },
-      timestamp,
     },
-    existingExecution: async (input) => {
+    fn: async (input) => {
       return queryClickhouse<{ value: number }>({
-        query: query.replace("__TRACE_TABLE__", "traces"),
+        query,
         params: input.params,
-        tags: { ...input.tags, experiment_amt: "original" },
-      });
-    },
-    newExecution: async (input) => {
-      const traceAmt = getTimeframesTracesAMT(input.timestamp);
-      return queryClickhouse<{ value: number }>({
-        query: query.replace("__TRACE_TABLE__", traceAmt),
-        params: input.params,
-        tags: { ...input.tags, experiment_amt: "new" },
+        tags: input.tags,
       });
     },
   });
@@ -1219,7 +1194,7 @@ export const getAggregatedScoresForPrompts = async (
   fetchScoreRelation: "observation" | "trace",
 ) => {
   const query = `
-    SELECT 
+    SELECT
       prompt_id,
       s.id,
       s.name,
@@ -1229,9 +1204,9 @@ export const getAggregatedScoresForPrompts = async (
       s.data_type,
       s.comment,
       length(mapKeys(s.metadata)) > 0 AS has_metadata
-    FROM scores s FINAL LEFT JOIN observations o FINAL 
-      ON o.trace_id = s.trace_id 
-      AND o.project_id = s.project_id 
+    FROM scores s FINAL LEFT JOIN observations o FINAL
+      ON o.trace_id = s.trace_id
+      AND o.project_id = s.project_id
       ${fetchScoreRelation === "observation" ? "AND o.id = s.observation_id" : ""}
     WHERE o.project_id = {projectId: String}
     AND s.project_id = {projectId: String}
@@ -1276,7 +1251,7 @@ export const getScoreCountsByProjectInCreationInterval = async ({
   end: Date;
 }) => {
   const query = `
-    SELECT 
+    SELECT
       project_id,
       count(*) as count
     FROM scores
@@ -1312,7 +1287,7 @@ export const getScoreCountOfProjectsSinceCreationDate = async ({
   start: Date;
 }) => {
   const query = `
-    SELECT 
+    SELECT
       count(*) as count
     FROM scores
     WHERE project_id IN ({projectIds: Array(String)})
@@ -1353,7 +1328,7 @@ export const getDistinctScoreNames = async (p: {
 
   const query = `    SELECT DISTINCT
       name
-    FROM scores s 
+    FROM scores s
     WHERE s.project_id = {projectId: String}
     AND s.created_at <= {cutoffCreatedAt: DateTime64(3)}
     ${scoreTimestampFilter ? `AND s.timestamp >= {filterTimestamp: DateTime64(3)}` : ""}
@@ -1435,14 +1410,8 @@ export const getScoresForPostHog = async function* (
   minTimestamp: Date,
   maxTimestamp: Date,
 ) {
-  // Determine which trace table to use based on experiment flag
-  const useAMT = env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true";
   // Subtract 7d from minTimestamp to account for shift in query
-  const traceTable = useAMT
-    ? getTimeframesTracesAMT(
-        new Date(minTimestamp.getTime() - 7 * 24 * 60 * 60 * 1000),
-      )
-    : "traces";
+  const traceTable = "traces";
 
   const query = `    SELECT
       s.id as id,
@@ -1470,12 +1439,12 @@ export const getScoresForPostHog = async function* (
     AND s.timestamp >= {minTimestamp: DateTime64(3)}
     AND s.timestamp <= {maxTimestamp: DateTime64(3)}
     AND (
-      s.trace_id IS NOT NULL 
-      OR s.session_id IS NOT NULL 
+      s.trace_id IS NOT NULL
+      OR s.session_id IS NOT NULL
       OR s.dataset_run_id IS NOT NULL
     )
     AND (
-      t.project_id IS NULL 
+      t.project_id = '' -- use the default value for the string type to filter for absence
       OR (
         t.project_id = {projectId: String}
         AND t.timestamp >= {minTimestamp: DateTime64(3)} - INTERVAL 7 DAY
@@ -1496,7 +1465,6 @@ export const getScoresForPostHog = async function* (
       type: "score",
       kind: "analytic",
       projectId,
-      experiment_amt: useAMT ? "new" : "original",
     },
     clickhouseConfigs: {
       request_timeout: env.LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS,
@@ -1584,7 +1552,7 @@ export const getScoreMetadataById = async (
   id: string,
   source?: ScoreSourceType,
 ) => {
-  const query = `    SELECT 
+  const query = `    SELECT
       metadata
     FROM scores s
     WHERE s.project_id = {projectId: String}
@@ -1616,3 +1584,67 @@ export const getScoreMetadataById = async (
     )
     .shift();
 };
+
+/**
+ * Get score counts grouped by project and day within a date range.
+ *
+ * Returns one row per project per day with the count of scores created on that day.
+ * Uses half-open interval [startDate, endDate) for filtering based on timestamp.
+ *
+ * @param startDate - Start of date range (inclusive)
+ * @param endDate - End of date range (exclusive)
+ * @returns Array of { count, projectId, date } objects
+ *
+ * @example
+ * // Get score counts for March 1-2, 2024
+ * const counts = await getScoreCountsByProjectAndDay({
+ *   startDate: new Date('2024-03-01T00:00:00Z'),
+ *   endDate: new Date('2024-03-03T00:00:00Z')
+ * });
+ *
+ * Note: Skips using FINAL (double counting risk) for faster and cheaper
+ * queries against clickhouse. Generous 4x overcompensation before blocking allows
+ * for usage aggregation to be meaningful.
+ *
+ */
+export const getScoreCountsByProjectAndDay = async ({
+  startDate,
+  endDate,
+}: {
+  startDate: Date;
+  endDate: Date;
+}) => {
+  const query = `
+    SELECT
+      count(*) as count,
+      project_id,
+      toDate(timestamp) as date
+    FROM scores
+    WHERE timestamp >= {startDate: DateTime64(3)}
+    AND timestamp < {endDate: DateTime64(3)}
+    GROUP BY project_id, toDate(timestamp)
+  `;
+
+  const rows = await queryClickhouse<{
+    count: string;
+    project_id: string;
+    date: string;
+  }>({
+    query,
+    params: {
+      startDate: convertDateToClickhouseDateTime(startDate),
+      endDate: convertDateToClickhouseDateTime(endDate),
+    },
+    tags: {
+      feature: "tracing",
+      type: "score",
+      kind: "analytic",
+    },
+  });
+
+  return rows.map((row) => ({
+    count: Number(row.count),
+    projectId: row.project_id,
+    date: row.date,
+  }));
+};