chore: release v3.26.0

fix: fix cost calculation discrepancies in dashboards (#5448 )
* fix: fix charts * security: upgrade jsonpath-plus * security: upgrade jsonpath-plus * fix * fix * fix * fix * push * push
2025-02-10 22:44:15 +01:00 · 2025-02-10 21:13:05 +00:00 · 2025-02-10 17:22:44 +00:00 · 2025-02-10 15:25:15 +00:00 · 2025-02-10 14:28:03 +00:00 · 2025-02-10 09:45:45 +00:00
405 changed files with 35278 additions and 19914 deletions
@@ -1,86 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-
-# Clickhouse
-CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
-CLICKHOUSE_URL="http://localhost:8123"
-CLICKHOUSE_USER="clickhouse"
-CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_CLUSTER_ENABLED="false"
-
-# Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-# NEXTAUTH_SECRET=""
-NEXTAUTH_URL="http://localhost:3000"
-NEXTAUTH_SECRET="secret"
-
-# Langfuse Cloud Environment
-NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
-
-# Langfuse experimental features
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
-
-# Salt for API key hashing
-SALT="salt"
-
-# Email
-EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
-SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
-
-# S3 Batch Exports
-LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
-LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
-LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
-LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
-LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
-LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
-
-# # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
-
-# S3 Event Bucket Upload
-## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
-
-# Set during docker build of application
-# Used to disable environment verification at build time
-# DOCKER_BUILD=1
-
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
-
-# speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=true
-LANGFUSE_RETURN_FROM_CLICKHOUSE=false
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false
@@ -103,6 +103,7 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
 # AUTH_CUSTOM_CLIENT_AUTH_METHOD="client_secret_basic" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true

 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -245,13 +246,12 @@ OTEL_SERVICE_NAME="langfuse"
 # CLICKHOUSE_URL=
 # CLICKHOUSE_USER=
 # CLICKHOUSE_PASSWORD=
+# CLICKHOUSE_DB=

 # Ingestion
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
-# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"

-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion
@@ -3,9 +3,13 @@ name: Codespell

 on:
  push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
  pull_request:
-    branches: [main]
+    branches:
+      - "**"
  merge_group:

 permissions:
@@ -17,8 +17,22 @@ concurrency:
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

 jobs:
+  pre-job:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    timeout-minutes: 15
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          do_not_skip: '["workflow_dispatch"]'
+
  lint:
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
@@ -41,6 +55,9 @@ jobs:
  test-docker-build:
    timeout-minutes: 20
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
      - name: Checkout
        uses: actions/checkout@v4
@@ -74,6 +91,9 @@ jobs:
  tests-web-sync:
    timeout-minutes: 20
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
    strategy:
      matrix:
@@ -87,7 +107,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - uses: pnpm/action-setup@v3
@@ -111,8 +131,9 @@ jobs:
      - name: Load default env
        run: |
          cp .env.dev.example .env
-          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
-          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
      - name: Run + migrate
        run: |
          docker compose -f docker-compose.dev.yml up -d
@@ -146,6 +167,9 @@ jobs:
  tests-web-async:
    timeout-minutes: 20
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
    strategy:
      matrix:
@@ -160,7 +184,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - uses: pnpm/action-setup@v3
@@ -184,7 +208,9 @@ jobs:
      - name: Load default env
        run: |
          cp .env.dev${{ matrix.blob-provider }}.example .env
-          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
      - name: Run + migrate
        run: |
          docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -216,6 +242,9 @@ jobs:
  tests-worker:
    timeout-minutes: 20
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
    strategy:
      matrix:
@@ -248,7 +277,7 @@ jobs:
          pnpm install
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Load default env
@@ -281,6 +310,9 @@ jobs:

  e2e-tests:
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v3
@@ -322,6 +354,9 @@ jobs:

  e2e-server-tests:
    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
    steps:
      - uses: actions/checkout@v4
      - name: Login to Docker Hub
@@ -343,7 +378,7 @@ jobs:
          pnpm install
      - name: Install golang-migrate for Clickhouse migrations
        run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
          sudo mv migrate /usr/bin/migrate
          which migrate
      - name: Load default env
@@ -6,7 +6,7 @@ on:
  push:
    branches:
      - main
-  merge_group:
+  # Snyk cannot upload results in merge group. Hence, we only run on PRs and when pushingon the main branch https://github.com/github/codeql-action/issues/1572

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -39,7 +39,6 @@ yarn-error.log*
 !.env.dev.example
 !.env.dev-azure.example
 !.env.prod.example
-!.env.dev.legacy.example

 # vercel
 .vercel
@@ -121,36 +121,40 @@ flowchart TB
        end
    end

-    DB[Postgres Database]
-    Redis[Redis Cache/Queue]
-    Clickhouse[Clickhouse Database]
+    subgraph s9 ["VPC (US and EU separated)"]
+        DB[Postgres Database]
+        Redis[Redis Cache/Queue]
+        Clickhouse[Clickhouse Database]
+
+        subgraph s1["Application (langfuse/langfuse/web)"]
+            API[Public HTTP API]
+            G[TRPC API]
+            I[NextAuth]
+            H[React Frontend]
+            ORM
+            H --> G
+            H --> I
+            G --> I
+            G --- ORM
+            API --- ORM
+            I --- ORM
+        end
+
+        subgraph s5["Application (langfuse/langfuse/worker)"]
+            Worker
+        end
+
+        Worker --- DB
+        Worker --- Redis
+        Worker --- Clickhouse
+
+        ORM --- DB
+        ORM --- Redis
+        ORM --- Clickhouse
+

-    subgraph s1["Application (langfuse/langfuse/web)"]
-        API[Public HTTP API]
-        G[TRPC API]
-        I[NextAuth]
-        H[React Frontend]
-        ORM
-        H --> G
-        H --> I
-        G --> I
-        G --- ORM
-        API --- ORM
-        I --- ORM
    end

-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker
-    end
-
-    Worker --- DB
-    Worker --- Redis
-    Worker --- Clickhouse
-
-    ORM --- DB
-    ORM --- Redis
-    ORM --- Clickhouse
-
    JS --- API
    Python --- API
 ```
@@ -397,7 +401,7 @@ The background color of the following component will be `hsl(var(--primary))` an
 | --primary-accent         | Primary accent color used for branding                             | Layout                           |
 | --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
 | --muted-green            | Muted green for Event label                                        | ObservationTree                  |
-| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-magenta          | Muted magenta for Generation label                                 | ObservationTree                  |
 | --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
 | --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
 | --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
@@ -439,7 +443,7 @@ You can update the default AI models and prices by adding or updating an entry i
 Please note that

 - prices are in USD
- the list is ordered by ID, so make sure to keep this order
+- the list is ordered by ID, so make sure to keep this order and insert new models at the end of the list
 - the `updated_at` field must be updated with the current date in ISO 8601 format. Otherwise, the change will be ignored.

 ### Transition period until V3 release
@@ -1,13 +1,19 @@
 _We are Hiring_
-Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+Join us in building out Langfuse in Berlin, Germany. Langfuse is the open source LLM engineering platform: we build tooling to help developers [build & improve LLM applications](https://langfuse.com/docs). 
+
+We are an open source company, we hire in person (4+ days a week), we only hire excellent technical talent. Find more information on our [careers page](https://langfuse.com/careers)

 _Open Roles_

- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Product Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Backend Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/1bO16H6-backend-engineer
+- Design Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mDquP95-design-engineer
+- Developer Advocate, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+

 _More Info_

 - https://langfuse.com/careers
 - https://langfuse.com/docs
+- https://langfuse.com/why
 - https://langfuse.com/changelog
@@ -38,6 +38,8 @@ services:
      LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
      LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
      LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
+      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
      REDIS_HOST: ${REDIS_HOST:-redis}
      REDIS_PORT: ${REDIS_PORT:-6379}
      REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
@@ -61,10 +63,6 @@ services:
      LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
      LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
      LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
-      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
-      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
-      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}

  clickhouse:
    image: clickhouse/clickhouse-server
@@ -27,6 +27,7 @@
    "@langfuse/shared": "workspace:*",
    "@opentelemetry/api": ">=1.0.0 <1.10.0",
    "axios": "^1.7.7",
+    "https-proxy-agent": "^7.0.6",
    "next": "^14.2.21",
    "next-auth": "^4.24.11",
    "zod": "^3.23.8"
@@ -47,7 +48,8 @@
  },
  "pnpm": {
    "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8"
    }
  }
 }
@@ -12,7 +12,7 @@ groups:
      #     namespaceExport: Langfuse
      #     allowCustomFetcher: true
      - name: fernapi/fern-openapi
-        version: 0.0.26
+        version: 0.1.7
        output:
          location: local-file-system
          path: ../../../web/public/generated/api-client
@@ -378,10 +378,10 @@ types:
        type: string
      startDate:
        docs: Apply only to generations which are newer than this ISO date.
-        type: optional<date>
+        type: optional<datetime>
      unit:
        docs: Unit used by this model.
-        type: ModelUsageUnit
+        type: optional<ModelUsageUnit>
      inputPrice:
        docs: Price (USD) per input unit
        type: optional<double>
@@ -128,7 +128,7 @@ types:
      modelParameters: optional<map<string, commons.MapValue>>
      usage: optional<IngestionUsage>
      usageDetails: optional<UsageDetails>
-      costDetails: optional<map<string, float>>
+      costDetails: optional<map<string, double>>
      promptName: optional<string>
      promptVersion: optional<integer>

@@ -141,7 +141,7 @@ types:
      usage: optional<IngestionUsage>
      promptName: optional<string>
      usageDetails: optional<UsageDetails>
-      costDetails: optional<map<string, float>>
+      costDetails: optional<map<string, double>>
      promptVersion: optional<integer>

  ObservationBody:
@@ -58,7 +58,7 @@ types:
        type: optional<datetime>
      unit:
        docs: Unit used by this model.
-        type: commons.ModelUsageUnit
+        type: optional<commons.ModelUsageUnit>
      inputPrice:
        docs: Price (USD) per input unit
        type: optional<double>
@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  prompts: ./prompts.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public/v2
+  endpoints:
+    update:
+      docs: Update labels for a specific prompt version
+      method: PATCH
+      path: /prompts/{name}/versions/{version}
+      path-parameters:
+        name:
+          type: string
+          docs: The name of the prompt
+        version:
+          type: integer
+          docs: Version of the prompt to update
+      request:
+        name: UpdatePromptRequest
+        body:
+          properties:
+            newLabels:
+              type: list<string>
+              docs: New labels for the prompt version. Labels are unique across versions. The "latest" label is reserved and managed by Langfuse.
+      response: prompts.Prompt
@@ -90,6 +90,9 @@ types:
      tags:
        type: optional<list<string>>
        docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.

  CreateTextPromptRequest:
    properties:
@@ -102,6 +105,9 @@ types:
      tags:
        type: optional<list<string>>
        docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.

  Prompt:
    union:
@@ -119,6 +125,9 @@ types:
      tags:
        type: list<string>
        docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.

  ChatMessage:
    properties:
@@ -59,7 +59,7 @@ service:
            type: optional<commons.ScoreDataType>
            docs: Retrieve only scores with a specific dataType.
          traceTags:
-            type: optional<list<string>>
+            type: optional<string>
            allow-multiple: true
            docs: Only scores linked to traces that include all of these tags will be returned.
      response: GetScoresResponse
@@ -3,7 +3,7 @@ groups:
  local:
    generators:
      - name: fernapi/fern-openapi
-        version: 0.0.31
+        version: 0.1.7
        output:
          location: local-file-system
          path: ../../../web/public/generated/api
@@ -1,4 +1,5 @@
 {
-  "organization": "finto",
+  "organization": "langfuse",
  "version": "0.43.7"
-}
+}
+
@@ -1,6 +1,6 @@
 {
  "name": "langfuse",
-  "version": "3.5.2",
+  "version": "3.26.0",
  "author": "engineering@langfuse.com",
  "license": "MIT",
  "private": true,
@@ -25,7 +25,6 @@
    "dev": "turbo run dev",
    "lint": "turbo run lint",
    "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate",
    "release": "dotenv -e ../.env -- release-it",
    "prepare": "husky"
  },
@@ -84,7 +83,12 @@
  "packageManager": "pnpm@9.5.0",
  "pnpm": {
    "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
    }
  }
 }
@@ -18,21 +18,31 @@ then
    exit 1
 fi

+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
  fi

  # Execute the up command
  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
 else
  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
  fi

  # Execute the up command
@@ -12,11 +12,16 @@ then
    exit 1
 fi

+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
 fi
 # Execute the drop command
 migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop
@@ -18,21 +18,31 @@ then
    exit 1
 fi

+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
  if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
  fi

  # Execute the up command
  migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
 else
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
  else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
  fi

  # Execute the up command
@@ -33,7 +33,7 @@
  "scripts": {
    "build": "tsc",
    "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 110",
    "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
    "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
    "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -61,25 +61,26 @@
    "@aws-sdk/s3-request-presigner": "^3.679.0",
    "@azure/storage-blob": "^12.26.0",
    "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.8",
-    "@langchain/aws": "^0.1.2",
-    "@langchain/core": "^0.3.18",
-    "@langchain/google-vertexai": "^0.1.3",
-    "@langchain/openai": "^0.3.14",
+    "@langchain/anthropic": "^0.3.12",
+    "@langchain/aws": "^0.1.3",
+    "@langchain/core": "^0.3.37",
+    "@langchain/google-vertexai": "^0.1.8",
+    "@langchain/openai": "^0.3.17",
    "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^5.22.0",
+    "@prisma/client": "^6.3.0",
    "@react-email/components": "^0.0.19",
    "@react-email/render": "^0.0.15",
    "@types/bcryptjs": "^2.4.6",
    "axios": "^1.7.7",
    "bcryptjs": "^2.4.3",
-    "bullmq": "^5.12.10",
+    "bullmq": "^5.34.10",
    "dd-trace": "^5.23.1",
    "decimal.js": "^10.4.3",
    "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
    "ioredis": "^5.4.1",
    "kysely": "^0.27.4",
-    "langchain": "^0.3.6",
+    "langchain": "^0.3.15",
    "langfuse-langchain": "3.30.3",
    "lodash": "^4.17.21",
    "next-auth": "^4.24.11",
@@ -106,7 +107,7 @@
    "kysely-codegen": "^0.16.8",
    "nodemon": "^3.1.7",
    "prettier": "^3.3.3",
-    "prisma": "^5.22.0",
+    "prisma": "^6.3.0",
    "prisma-erd-generator": "^1.11.2",
    "prisma-kysely": "^1.8.0",
    "ts-node": "^10.9.2",
@@ -121,7 +122,8 @@
  },
  "pnpm": {
    "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8"
    }
  }
 }
@@ -170,6 +170,17 @@ export type BatchExport = {
    url: string | null;
    log: string | null;
 };
+export type BillingMeterBackup = {
+    stripe_customer_id: string;
+    meter_id: string;
+    start_time: Timestamp;
+    end_time: Timestamp;
+    aggregated_value: number;
+    event_name: string;
+    org_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Comment = {
    id: string;
    project_id: string;
@@ -242,6 +253,18 @@ export type EvalTemplate = {
    vars: Generated<string[]>;
    output_schema: unknown;
 };
+export type EventLog = {
+    id: string;
+    bucket_name: string;
+    bucket_path: string;
+    project_id: string;
+    entity_type: string;
+    entity_id: string;
+    event_id: string;
+    trace_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Events = {
    id: string;
    created_at: Generated<Timestamp>;
@@ -293,6 +316,8 @@ export type LlmApiKeys = {
    base_url: string | null;
    custom_models: Generated<string[]>;
    with_default_models: Generated<boolean>;
+    extra_headers: string | null;
+    extra_header_keys: Generated<string[]>;
    config: unknown | null;
    project_id: string;
 };
@@ -455,6 +480,7 @@ export type Project = {
    updated_at: Generated<Timestamp>;
    deleted_at: Timestamp | null;
    name: string;
+    retention_days: number | null;
 };
 export type ProjectMembership = {
    org_membership_id: string;
@@ -478,6 +504,14 @@ export type Prompt = {
    config: Generated<unknown>;
    tags: Generated<string[]>;
    labels: Generated<string[]>;
+    commit_message: string | null;
+};
+export type QueueBackUp = {
+    id: string;
+    project_id: string | null;
+    queue_name: string;
+    content: unknown;
+    created_at: Generated<Timestamp>;
 };
 export type Score = {
    id: string;
@@ -604,6 +638,7 @@ export type DB = {
    audit_logs: AuditLog;
    background_migrations: BackgroundMigration;
    batch_exports: BatchExport;
+    billing_meter_backups: BillingMeterBackup;
    comments: Comment;
    cron_jobs: CronJobs;
    dataset_items: DatasetItem;
@@ -611,6 +646,7 @@ export type DB = {
    dataset_runs: DatasetRuns;
    datasets: Dataset;
    eval_templates: EvalTemplate;
+    event_log: EventLog;
    events: Events;
    job_configurations: JobConfiguration;
    job_executions: JobExecution;
@@ -628,6 +664,7 @@ export type DB = {
    project_memberships: ProjectMembership;
    projects: Project;
    prompts: Prompt;
+    queue_backups: QueueBackUp;
    score_configs: ScoreConfig;
    scores: Score;
    Session: Session;
@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "queue_backups" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT,
+    "queue_name" TEXT NOT NULL,
+    "content" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "queue_backups_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_project_id_fkey" FOREIGN KEY ("session_id", "project_id") REFERENCES "trace_sessions"("id", "project_id") ON DELETE RESTRICT ON UPDATE CASCADE;
@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";
@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "billing_meter_backups" (
+    "stripe_customer_id" TEXT NOT NULL,
+    "meter_id" TEXT NOT NULL,
+    "start_time" TIMESTAMP(3) NOT NULL,
+    "end_time" TIMESTAMP(3) NOT NULL,
+    
+    "aggregated_value" INTEGER NOT NULL,
+    
+    "event_name" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- CreateIndex
+CREATE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_idx" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_key" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");
@@ -0,0 +1,3 @@
+ALTER TABLE "llm_api_keys"
+    ADD COLUMN "extra_headers" TEXT,
+    ADD COLUMN "extra_header_keys" TEXT[] NOT NULL DEFAULT '{}'::TEXT[];
@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "projects"
+    ADD COLUMN "retention_days" INTEGER;
@@ -0,0 +1 @@
+UPDATE "llm_api_keys" SET adapter = 'google-vertex-ai' WHERE adapter = 'vertex-ai';
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "commit_message" TEXT;
@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "event_log" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "bucket_name" TEXT NOT NULL,
+    "bucket_path" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "event_id" TEXT NOT NULL,
+
+    "trace_id" TEXT,
+
+    CONSTRAINT "event_log_pkey" PRIMARY KEY ("id"),
+    CONSTRAINT "event_log_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE INDEX "event_log_project_id_entity_type_entity_id_idx" ON "event_log"("project_id", "entity_type", "entity_id");
@@ -115,6 +115,7 @@ model Project {
  updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
  deletedAt           DateTime?              @map("deleted_at")
  name                String
+  retentionDays       Int?                   @map("retention_days")
  projectMembers      ProjectMembership[]
  organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
  traces              Trace[]
@@ -140,6 +141,7 @@ model Project {
  TraceMedia          TraceMedia[]
  Media               Media[]
  ObservationMedia    ObservationMedia[]
+  EventLog            EventLog[]

  @@index([orgId])
  @@map("projects")
@@ -192,6 +194,8 @@ model LlmApiKeys {
  baseURL           String?  @map("base_url")
  customModels      String[] @default([]) @map("custom_models")
  withDefaultModels Boolean  @default(true) @map("with_default_models")
+  extraHeaders      String?  @map("extra_headers")
+  extraHeaderKeys   String[] @default([]) @map("extra_header_keys")
  config            Json?

  projectId String  @map("project_id")
@@ -272,7 +276,6 @@ model TraceSession {
  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
  bookmarked Boolean  @default(false)
  public     Boolean  @default(false)
-  traces     Trace[]

  @@id([id, projectId])
  @@index([projectId])
@@ -284,25 +287,24 @@ model TraceSession {
 // Update TraceView below when making changes to this model!

 model Trace {
-  id         String        @id @default(cuid())
-  externalId String?       @map("external_id")
-  timestamp  DateTime      @default(now())
+  id         String   @id @default(cuid())
+  externalId String?  @map("external_id")
+  timestamp  DateTime @default(now())
  name       String?
-  userId     String?       @map("user_id")
+  userId     String?  @map("user_id")
  metadata   Json?
  release    String?
  version    String?
-  projectId  String        @map("project_id")
-  project    Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  public     Boolean       @default(false)
-  bookmarked Boolean       @default(false)
-  tags       String[]      @default([])
+  projectId  String   @map("project_id")
+  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  public     Boolean  @default(false)
+  bookmarked Boolean  @default(false)
+  tags       String[] @default([])
  input      Json?
  output     Json?
-  sessionId  String?       @map("session_id")
-  session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
-  createdAt  DateTime      @default(now()) @map("created_at")
-  updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
+  sessionId  String?  @map("session_id")
+  createdAt  DateTime @default(now()) @map("created_at")
+  updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")

  @@index([projectId, timestamp])
  @@index([sessionId])
@@ -705,6 +707,29 @@ model Events {
  @@map("events")
 }

+model EventLog {
+  id String @id @default(cuid())
+
+  bucketName String  @map("bucket_name")
+  bucketPath String  @map("bucket_path")
+  projectId  String  @map("project_id")
+  project    Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  entityType String  @map("entity_type")
+  entityId   String  @map("entity_id")
+  eventId    String  @map("event_id")
+
+  // traceId is only optional, because we cannot guarantee its presence for observations.
+  // We use it to support deletions on traces as we cannot cascade it down to scores and observations otherwise.
+  // We expect a traceId to be present in most cases.
+  traceId String? @map("trace_id")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, entityType, entityId])
+  @@map("event_log")
+}
+
 model Comment {
  id           String            @id @default(cuid())
  projectId    String            @map("project_id")
@@ -737,14 +762,15 @@ model Prompt {

  createdBy String @map("created_by")

-  prompt   Json
-  name     String
-  version  Int
-  type     String   @default("text")
-  isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-  config   Json     @default("{}") @db.Json
-  tags     String[] @default([])
-  labels   String[] @default([])
+  prompt        Json
+  name          String
+  version       Int
+  type          String   @default("text")
+  isActive      Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+  config        Json     @default("{}") @db.Json
+  tags          String[] @default([])
+  labels        String[] @default([])
+  commitMessage String?  @map("commit_message")

  @@unique([projectId, name, version])
  @@index([projectId, id])
@@ -1017,3 +1043,37 @@ model ObservationMedia {
  @@index([projectId, observationId])
  @@map("observation_media")
 }
+
+model QueueBackUp {
+  id        String  @id @default(cuid())
+  projectId String? @map("project_id")
+  queueName String  @map("queue_name")
+  content   Json
+
+  createdAt DateTime @default(now()) @map("created_at")
+
+  @@map("queue_backups")
+}
+
+model BillingMeterBackup {
+  // unique
+  stripeCustomerId String   @map("stripe_customer_id")
+  meterId          String   @map("meter_id")
+  startTime        DateTime @map("start_time")
+  endTime          DateTime @map("end_time")
+
+  // value
+  aggregatedValue Int @map("aggregated_value")
+
+  // labels
+  eventName String @map("event_name")
+  orgId     String @map("org_id")
+
+  // ts
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([stripeCustomerId, meterId, startTime, endTime])
+  @@index([stripeCustomerId, meterId, startTime, endTime])
+  @@map("billing_meter_backups")
+}
@@ -97,19 +97,31 @@ export const prepareClickhouse = async (
      repeat('input', toInt64(randExponential(1 / 100))) AS input,
      repeat('output', toInt64(randExponential(1 / 100))) AS output,
      case
-        when number % 2 = 0 then 'claude-3-haiku-20240307'
+        when number % 2 = 0 then 'clause-3-haiku-20230407'
        else 'gpt-4'
      end as provided_model_name,
      case
-        when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
-        else 'clrntkjgy000f08jx79v9g1xj'
+        when number % 2 = 0 then 'cltra4wbs0000k1407g0ya3'
+        else '1cmtk9y0000y3y79x9jgxj'
      end as internal_model_id,
-      '{"temperature": 0.7, "max_tokens": 150}' AS model_parameters,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS provided_usage_details,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS usage_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
-      toDecimal64(randUniform(0, 2000), 12) AS total_cost,
+      if("type" = 'GENERATION',
+        '{"temperature": 0.7, "max_tokens": 150}',
+        '{}') AS model_parameters,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS provided_usage_details,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS usage_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS provided_cost_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS cost_details,
+      if("type" = 'GENERATION',
+        toDecimal64(randUniform(0, 2000), 12),
+        NULL) AS total_cost,
      addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
      array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
        ",",
@@ -29,17 +29,10 @@ const EnvSchema = z.object({
  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
  CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
+  CLICKHOUSE_DB: z.string().default("default"),
  CLICKHOUSE_USER: z.string(),
  CLICKHOUSE_PASSWORD: z.string(),
-  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
-    .enum(["true", "false"])
-    .default("true"),
-  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
-    .enum(["true", "false"])
-    .default("false"),

  LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
    .number()
@@ -53,6 +46,9 @@ const EnvSchema = z.object({
  ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
    .enum(["true", "false"])
    .default("false"),
+  LANGFUSE_S3_EVENT_UPLOAD_POSTGRES_LOG_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
  LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
    required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
  }),
@@ -70,6 +66,9 @@ const EnvSchema = z.object({
  LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
    .enum(["true", "false"])
    .default("false"),
+  LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
 });

 export const env: z.infer<typeof EnvSchema> =
@@ -1,7 +1,7 @@
 import { BaseError } from "./BaseError";

 export class ApiError extends BaseError {
-  constructor(description = "Api call failed") {
-    super("ApiError", 500, description, true);
+  constructor(description = "Api call failed", status = 500) {
+    super("ApiError", status, description, true);
  }
 }
@@ -0,0 +1,38 @@
+import z from "zod";
+import { singleFilter } from "../../interfaces/filters";
+import { orderBy } from "../../interfaces/orderBy";
+import { BatchExportTableName } from "../batchExport/types";
+
+/* eslint-disable no-unused-vars */
+export enum BatchActionType {
+  Create = "create",
+  Delete = "delete",
+}
+
+const ActionIdSchema = z.enum([
+  "trace-delete",
+  "trace-add-to-annotation-queue",
+]);
+
+export type ActionId = z.infer<typeof ActionIdSchema>;
+
+export const BatchActionQuerySchema = z.object({
+  filter: z.array(singleFilter).nullable(),
+  orderBy,
+});
+
+export type BatchActionQuery = z.infer<typeof BatchActionQuerySchema>;
+
+export const CreateBatchActionSchema = z.object({
+  projectId: z.string(),
+  actionId: ActionIdSchema,
+  targetId: z.string().optional(),
+  query: BatchActionQuerySchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+});
+
+export const GetIsBatchActionInProgressSchema = z.object({
+  projectId: z.string(),
+  actionId: ActionIdSchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+});
@@ -17,6 +17,7 @@ export const variableMapping = z
    objectName: z.string().nullish(),
    langfuseObject: z.enum(langfuseObjects),
    selectedColumnId: z.string(),
+    jsonSelector: z.string().nullish(),
  })
  .refine(
    (value) => value.langfuseObject === "trace" || value.objectName !== null,
@@ -32,6 +33,7 @@ export const wipVariableMapping = z.object({
  objectName: z.string().nullish(),
  langfuseObject: z.enum(langfuseObjects),
  selectedColumnId: z.string().nullish(),
+  jsonSelector: z.string().nullish(),
 });

 const observationCols = [
@@ -21,7 +21,10 @@ export * from "./server/llm/types";

 // evals
 export * from "./features/evals/types";
+
+// table actions
 export * from "./features/batchExport/types";
+export * from "./features/batchAction/types";

 // annotation
 export * from "./features/annotation/types";
@@ -20,6 +20,13 @@ export const observationsTableCols: ColumnDefinition[] = [
    options: [], // to be added at runtime
    nullable: true,
  },
+  {
+    name: "type",
+    id: "type",
+    type: "stringOptions",
+    options: [],
+    internal: 'o."type"',
+  },
  { name: "Trace ID", id: "traceId", type: "string", internal: 't."id"' },
  {
    name: "Trace Name",
@@ -200,6 +207,7 @@ export type ObservationOptions = {
  scores_avg: Array<string>;
  promptName: Array<OptionsDefinition>;
  tags: Array<OptionsDefinition>;
+  type: Array<OptionsDefinition>;
 };

 export function observationsTableColsWithOptions(
@@ -227,6 +235,9 @@ export function observationsTableColsWithOptions(
    if (col.id === "tags") {
      return { ...col, options: options?.tags ?? [] };
    }
+    if (col.id === "type") {
+      return { ...col, options: options?.type ?? [] };
+    }
    return col;
  });
 }
@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
    wellKnown: `${options.issuer}/.well-known/openid-configuration`,
    authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
    checks: ["pkce", "state"],
-    idToken: true,
    profile(profile) {
      return {
        id: profile.sub,
@@ -2,64 +2,64 @@ import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
 import type { GithubProfile, GithubEmail } from "next-auth/providers/github";

 export function GitHubEnterpriseProvider<P extends GithubProfile>(
-    options: OAuthUserConfig<P> & {
-        enterprise?: {
-            baseUrl?: string;
-        };
-    }
+  options: OAuthUserConfig<P> & {
+    enterprise?: {
+      baseUrl?: string;
+    };
+  },
 ): OAuthConfig<P> {
-    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
-    const apiBaseUrl = options?.enterprise?.baseUrl
-        ? `${options?.enterprise?.baseUrl}/api/v3`
-        : "https://api.github.com"
+  const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com";
+  const apiBaseUrl = options?.enterprise?.baseUrl
+    ? `${options?.enterprise?.baseUrl}/api/v3`
+    : "https://api.github.com";

-    return {
-        id: "github-enterprise",
-        name: "GitHub Enterprise",
-        type: "oauth",
-        authorization: {
-            url: `${baseUrl}/login/oauth/authorize`,
-            params: { scope: "read:user user:email" },
-        },
-        token: `${baseUrl}/login/oauth/access_token`,
-        userinfo: {
-            url: `${apiBaseUrl}/user`,
-            async request({ client, tokens }) {
-                const profile = await client.userinfo(tokens.access_token!)
+  return {
+    id: "github-enterprise",
+    name: "GitHub Enterprise",
+    type: "oauth",
+    authorization: {
+      url: `${baseUrl}/login/oauth/authorize`,
+      params: { scope: "read:user user:email" },
+    },
+    token: `${baseUrl}/login/oauth/access_token`,
+    userinfo: {
+      url: `${apiBaseUrl}/user`,
+      async request({ client, tokens }) {
+        const profile = await client.userinfo(tokens.access_token!);

-                if (!profile.email) {
-                    // If the user does not have a public email, get another via the GitHub API
-                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
-                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
-                        headers: { Authorization: `token ${tokens.access_token}` },
-                    })
+        if (!profile.email) {
+          // If the user does not have a public email, get another via the GitHub API
+          // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+          const res = await fetch(`${apiBaseUrl}/user/emails`, {
+            headers: { Authorization: `token ${tokens.access_token}` },
+          });

-                    if (res.ok) {
-                        const emails: GithubEmail[] = await res.json()
-                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
-                    }
-                }
+          if (res.ok) {
+            const emails = (await res.json()) as GithubEmail[];
+            profile.email = (emails.find((e) => e.primary) ?? emails[0]).email;
+          }
+        }

-                return profile
-            },
-        },
-        profile(profile) {
-            return {
-                id: profile.id.toString(),
-                name: profile.name ?? profile.login,
-                email: profile.email,
-                image: profile.avatar_url,
-            }
-        },
-        style: {
-            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
-            logoDark:
-                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
-            bg: "#fff",
-            bgDark: "#000",
-            text: "#000",
-            textDark: "#fff",
-        },
-        options,
-    }
-}
+        return profile;
+      },
+    },
+    profile(profile) {
+      return {
+        id: profile.id.toString(),
+        name: profile.name ?? profile.login,
+        email: profile.email,
+        image: profile.avatar_url,
+      };
+    },
+    style: {
+      logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+      logoDark:
+        "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+      bg: "#fff",
+      bgDark: "#000",
+      text: "#000",
+      textDark: "#fff",
+    },
+    options,
+  };
+}
@@ -10,7 +10,7 @@ export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
    url: env.CLICKHOUSE_URL,
    username: env.CLICKHOUSE_USER,
    password: env.CLICKHOUSE_PASSWORD,
-    database: "default",
+    database: env.CLICKHOUSE_DB,
    clickhouse_settings: {
      async_insert: 1,
      wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse
@@ -8,9 +8,11 @@ export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
 export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
+export * from "./llm/utils";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
+export * from "./utils/eventLog";
 export * from "./clickhouse/client";
 export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
@@ -28,16 +30,17 @@ export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
-export * from "./redis/legacyIngestion";
+export * from "./redis/batchActionQueue";
 export * from "./redis/ingestionQueue";
 export * from "./redis/postHogIntegrationQueue";
 export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/dataRetentionQueue";
+export * from "./redis/dataRetentionProcessingQueue";
 export * from "./redis/coreDataS3ExportQueue";
+export * from "./redis/meteringDataPostgresExportQueue";
 export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
-export * from "./ingestion/legacy/index";
 export * from "./queues";
-export * from "./ingestion/legacy/EventProcessor";
 export * from "./orderByToPrisma";
 export * from "./filterToPrisma";
 export * from "./instrumentation";
@@ -45,6 +48,7 @@ export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
+export * from "./services/sessions-ui-table-service";

 // test utils
 export * from "./test-utils";
@@ -1,796 +0,0 @@
-import { v4 } from "uuid";
-import { type z } from "zod";
-import Decimal from "decimal.js";
-import { findModel } from "../modelMatch";
-import {
-  ObservationEvent,
-  eventTypes,
-  legacyObservationCreateEvent,
-  generationCreateEvent,
-  traceEvent,
-  scoreEvent,
-  sdkLogEvent,
-  ingestionEvent,
-} from "../types";
-import { validateAndInflateScore } from "../validateAndInflateScore";
-import { Trace, Observation, Score, Prisma, Model } from "@prisma/client";
-import { ForbiddenError, LangfuseNotFoundError } from "../../../errors";
-import { mergeJson } from "../../../utils/json";
-import { jsonSchema } from "../../../utils/zod";
-import { prisma } from "../../../db";
-import { LegacyIngestionAccessScope } from ".";
-import { logger } from "../../logger";
-import { env } from "../../../env";
-import { upsertTrace } from "../../repositories";
-import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
-
-export interface EventProcessor {
-  auth(apiScope: LegacyIngestionAccessScope): void;
-
-  process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> | undefined;
-}
-
-export const getProcessorForEvent = (
-  event: z.infer<typeof ingestionEvent>,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-): EventProcessor => {
-  switch (event.type) {
-    case eventTypes.TRACE_CREATE:
-      return new TraceProcessor(event);
-    case eventTypes.OBSERVATION_CREATE:
-    case eventTypes.OBSERVATION_UPDATE:
-    case eventTypes.EVENT_CREATE:
-    case eventTypes.SPAN_CREATE:
-    case eventTypes.SPAN_UPDATE:
-    case eventTypes.GENERATION_CREATE:
-    case eventTypes.GENERATION_UPDATE:
-      return new ObservationProcessor(event, calculateTokenDelegate);
-    case eventTypes.SCORE_CREATE: {
-      return new ScoreProcessor(event);
-    }
-    case eventTypes.SDK_LOG:
-      return new SdkLogProcessor(event);
-  }
-};
-
-export class ObservationProcessor implements EventProcessor {
-  event: ObservationEvent;
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined;
-
-  constructor(
-    event: ObservationEvent,
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-  ) {
-    this.event = event;
-    this.calculateTokenDelegate = calculateTokenDelegate;
-  }
-
-  async convertToObservation(
-    apiScope: LegacyIngestionAccessScope,
-    existingObservation: Omit<Observation, "input" | "output"> | null,
-  ): Promise<{
-    id: string;
-    create: Prisma.ObservationUncheckedCreateInput;
-    update: Prisma.ObservationUncheckedUpdateInput;
-  }> {
-    let type: "EVENT" | "SPAN" | "GENERATION";
-    switch (this.event.type) {
-      case eventTypes.OBSERVATION_CREATE:
-      case eventTypes.OBSERVATION_UPDATE:
-        type = this.event.body.type;
-        break;
-      case eventTypes.EVENT_CREATE:
-        type = "EVENT" as const;
-        break;
-      case eventTypes.SPAN_CREATE:
-      case eventTypes.SPAN_UPDATE:
-        type = "SPAN" as const;
-        break;
-      case eventTypes.GENERATION_CREATE:
-      case eventTypes.GENERATION_UPDATE:
-        type = "GENERATION" as const;
-        break;
-    }
-
-    if (
-      this.event.type === eventTypes.OBSERVATION_UPDATE &&
-      !existingObservation
-    ) {
-      throw new LangfuseNotFoundError(
-        `Observation with id ${this.event.id} not found`,
-      );
-    }
-
-    // find matching model definition based on event and existing observation in db
-    const internalModel: Model | undefined | null =
-      type === "GENERATION"
-        ? await findModel({
-            event: {
-              projectId: apiScope.projectId,
-              model:
-                "model" in this.event.body
-                  ? (this.event.body.model ?? undefined)
-                  : undefined,
-              unit:
-                "usage" in this.event.body
-                  ? (this.event.body.usage?.unit ?? undefined)
-                  : undefined,
-              startTime: this.event.body.startTime
-                ? new Date(this.event.body.startTime)
-                : undefined,
-            },
-            existingDbObservation: existingObservation ?? undefined,
-          })
-        : undefined;
-
-    // Token counts
-    const [newInputCount, newOutputCount] =
-      "usage" in this.event.body || "usageDetails" in this.event.body
-        ? await this.calculateTokenCounts(
-            apiScope.projectId,
-            this.event.body,
-            this.calculateTokenDelegate,
-            internalModel ?? undefined,
-            existingObservation ?? undefined,
-          )
-        : [undefined, undefined];
-
-    const newTotalCount =
-      "usage" in this.event.body || "usageDetails" in this.event.body
-        ? (this.event.body.usage?.total ??
-          this.event.body.usageDetails?.total ??
-          (newInputCount != null || newOutputCount != null
-            ? (newInputCount ?? 0) + (newOutputCount ?? 0)
-            : undefined))
-        : undefined;
-
-    const userProvidedTokenCosts = {
-      inputCost:
-        "usage" in this.event.body && this.event.body.usage?.inputCost != null // inputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.inputCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.input != null
-            ? new Decimal(this.event.body.costDetails?.input)
-            : existingObservation?.inputCost,
-      outputCost:
-        "usage" in this.event.body && this.event.body.usage?.outputCost != null // outputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.outputCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.output != null
-            ? new Decimal(this.event.body.costDetails?.output)
-            : existingObservation?.outputCost,
-      totalCost:
-        "usage" in this.event.body && this.event.body.usage?.totalCost != null // totalCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.totalCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.total != null
-            ? new Decimal(this.event.body.costDetails?.total)
-            : existingObservation?.totalCost,
-    };
-
-    const tokenCounts = {
-      input: newInputCount ?? existingObservation?.promptTokens,
-      output: newOutputCount ?? existingObservation?.completionTokens,
-      total: newTotalCount || existingObservation?.totalTokens,
-    };
-
-    const calculatedCosts = ObservationProcessor.calculateTokenCosts(
-      internalModel,
-      userProvidedTokenCosts,
-      tokenCounts,
-    );
-
-    // merge metadata from existingObservation.metadata and metadata
-    const mergedMetadata = mergeJson(
-      existingObservation?.metadata
-        ? jsonSchema.parse(existingObservation.metadata)
-        : undefined,
-      this.event.body.metadata ?? undefined,
-    );
-
-    const prompt =
-      "promptName" in this.event.body &&
-      typeof this.event.body.promptName === "string" &&
-      "promptVersion" in this.event.body &&
-      typeof this.event.body.promptVersion === "number"
-        ? await prisma.prompt.findUnique({
-            where: {
-              projectId_name_version: {
-                projectId: apiScope.projectId,
-                name: this.event.body.promptName,
-                version: this.event.body.promptVersion,
-              },
-            },
-          })
-        : undefined;
-
-    // Only null if promptName and promptVersion are set but prompt is not found
-    if (prompt === null) {
-      logger.warn("Prompt not found for observation", this.event.body);
-    }
-
-    const observationId =
-      this.event.body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `observation.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    let traceId = this.event.body?.traceId;
-    if (!this.event.body.traceId && !existingObservation) {
-      // Create trace if no traceId
-      traceId = observationId;
-
-      // Insert trace into postgres
-      await prisma.trace.upsert({
-        where: {
-          id: observationId,
-        },
-        create: {
-          projectId: apiScope.projectId,
-          name: this.event.body.name,
-          id: observationId,
-          timestamp: this.event.body.startTime || new Date(),
-        },
-        update: {},
-      });
-
-      if (env.CLICKHOUSE_URL) {
-        // Insert trace into clickhouse if enabled
-        await upsertTrace({
-          id: observationId,
-          project_id: apiScope.projectId,
-          timestamp: convertDateToClickhouseDateTime(
-            this.event.body.startTime
-              ? new Date(this.event.body.startTime)
-              : new Date(),
-          ),
-          created_at: convertDateToClickhouseDateTime(new Date()),
-          updated_at: convertDateToClickhouseDateTime(new Date()),
-        });
-      }
-    }
-
-    return {
-      id: observationId,
-      create: {
-        id: observationId,
-        traceId,
-        type: type,
-        name: this.event.body.name,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        projectId: apiScope.projectId,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-      update: {
-        name: this.event.body.name ?? undefined,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-    };
-  }
-
-  async calculateTokenCounts(
-    projectId: string,
-    body:
-      | z.infer<typeof legacyObservationCreateEvent>["body"]
-      | z.infer<typeof generationCreateEvent>["body"],
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-    model?: Model,
-    existingObservation?: Omit<Observation, "input" | "output">,
-  ) {
-    let newPromptTokens = body.usage?.input ?? body.usageDetails?.input;
-    if (newPromptTokens === undefined && model && model.tokenizerId) {
-      if (body.input) {
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: body.input,
-        });
-      } else {
-        logger.debug(
-          `No input provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationInput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            input: true,
-          },
-        });
-
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: observationInput?.input,
-        });
-      }
-    }
-
-    let newCompletionTokens = body.usage?.output ?? body.usageDetails?.output;
-
-    if (newCompletionTokens === undefined && model && model.tokenizerId) {
-      if (body.output) {
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: body.output,
-        });
-      } else {
-        logger.debug(
-          `No output provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationOutput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            output: true,
-          },
-        });
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: observationOutput?.output,
-        });
-      }
-    }
-
-    return [newPromptTokens ?? undefined, newCompletionTokens ?? undefined];
-  }
-
-  static calculateTokenCosts(
-    model: Model | null | undefined,
-    userProvidedCosts: {
-      inputCost?: Decimal | null;
-      outputCost?: Decimal | null;
-      totalCost?: Decimal | null;
-    },
-    tokenCounts: { input?: number; output?: number; total?: number },
-  ): {
-    inputCost?: Decimal | null;
-    outputCost?: Decimal | null;
-    totalCost?: Decimal | null;
-  } {
-    // If user has provided any cost point, do not calculate anything else
-    if (
-      userProvidedCosts.inputCost ||
-      userProvidedCosts.outputCost ||
-      userProvidedCosts.totalCost
-    ) {
-      return {
-        ...userProvidedCosts,
-        totalCost:
-          userProvidedCosts.totalCost ??
-          (userProvidedCosts.inputCost ?? new Decimal(0)).add(
-            userProvidedCosts.outputCost ?? new Decimal(0),
-          ),
-      };
-    }
-
-    const finalInputCost =
-      tokenCounts.input !== undefined && model?.inputPrice
-        ? model.inputPrice.mul(tokenCounts.input)
-        : undefined;
-
-    const finalOutputCost =
-      tokenCounts.output !== undefined && model?.outputPrice
-        ? model.outputPrice.mul(tokenCounts.output)
-        : finalInputCost
-          ? new Decimal(0)
-          : undefined;
-
-    const finalTotalCost =
-      tokenCounts.total !== undefined && model?.totalPrice
-        ? model.totalPrice.mul(tokenCounts.total)
-        : (finalInputCost ?? finalOutputCost)
-          ? new Decimal(finalInputCost ?? 0).add(finalOutputCost ?? 0)
-          : undefined;
-
-    return {
-      inputCost: finalInputCost,
-      outputCost: finalOutputCost,
-      totalCost: finalTotalCost,
-    };
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for observation creation");
-  }
-
-  async process(apiScope: LegacyIngestionAccessScope): Promise<Observation> {
-    this.auth(apiScope);
-
-    const existingObservation = this.event.body.id
-      ? await prisma.observation.findFirst({
-          select: {
-            // do not select I/O to spare our db
-            input: false,
-            output: false,
-
-            id: true,
-            traceId: true,
-            projectId: true,
-            type: true,
-            startTime: true,
-            endTime: true,
-            name: true,
-            metadata: true,
-            parentObservationId: true,
-            level: true,
-            statusMessage: true,
-            version: true,
-            createdAt: true,
-            updatedAt: true,
-            model: true,
-            internalModelId: true,
-            modelParameters: true,
-            promptTokens: true,
-            completionTokens: true,
-            totalTokens: true,
-            unit: true,
-            inputCost: true,
-            outputCost: true,
-            totalCost: true,
-            calculatedInputCost: true,
-            calculatedOutputCost: true,
-            calculatedTotalCost: true,
-            completionStartTime: true,
-            promptId: true,
-            internalModel: true,
-          },
-          where: { id: this.event.body.id, projectId: apiScope.projectId },
-        })
-      : null;
-
-    if (
-      existingObservation &&
-      existingObservation.projectId !== apiScope.projectId
-    ) {
-      throw new ForbiddenError(
-        `Access denied for observation creation ${existingObservation.projectId} `,
-      );
-    }
-
-    const obs = await this.convertToObservation(apiScope, existingObservation);
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    return await prisma.observation.upsert({
-      where: {
-        id: obs.id,
-      },
-      create: obs.create,
-      update: obs.update,
-    });
-  }
-}
-
-export class TraceProcessor implements EventProcessor {
-  event: z.infer<typeof traceEvent>;
-
-  constructor(event: z.infer<typeof traceEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for trace creation");
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const internalId =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `trace.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    logger.debug(
-      `Trying to create trace, project ${apiScope.projectId}, id: ${internalId}`,
-    );
-
-    const existingTrace = await prisma.trace.findFirst({
-      where: {
-        id: internalId,
-      },
-    });
-
-    if (existingTrace && existingTrace.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for trace creation ${existingTrace.projectId}`,
-      );
-    }
-
-    const mergedMetadata = mergeJson(
-      existingTrace?.metadata
-        ? jsonSchema.parse(existingTrace.metadata)
-        : undefined,
-      body.metadata ?? undefined,
-    );
-
-    const mergedTags =
-      existingTrace?.tags && body.tags
-        ? Array.from(new Set(existingTrace.tags.concat(body.tags ?? []))).sort()
-        : body.tags
-          ? Array.from(new Set(body.tags)).sort()
-          : undefined;
-
-    if (body.sessionId) {
-      try {
-        await prisma.traceSession.upsert({
-          where: {
-            id_projectId: {
-              id: body.sessionId,
-              projectId: apiScope.projectId,
-            },
-          },
-          create: {
-            id: body.sessionId,
-            projectId: apiScope.projectId,
-          },
-          update: {},
-        });
-      } catch (e) {
-        if (
-          e instanceof Prisma.PrismaClientKnownRequestError &&
-          e.code === "P2002"
-        ) {
-          logger.warn(
-            `Failed to upsert session. Session ${body.sessionId} in project ${apiScope.projectId} already exists`,
-          );
-        } else {
-          throw e;
-        }
-      }
-    }
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    const upsertedTrace = await prisma.trace.upsert({
-      where: {
-        id: internalId,
-      },
-      create: {
-        id: internalId,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        name: body.name ?? undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        projectId: apiScope.projectId,
-        tags: mergedTags ?? undefined,
-      },
-      update: {
-        name: body.name ?? undefined,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        tags: mergedTags ?? undefined,
-      },
-    });
-    return upsertedTrace;
-  }
-}
-
-export class ScoreProcessor implements EventProcessor {
-  event: z.infer<typeof scoreEvent>;
-
-  constructor(event: z.infer<typeof scoreEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
-      throw new ForbiddenError(
-        `Access denied for score creation, ${apiScope.accessLevel}`,
-      );
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const id =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `score.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    const existingScore = await prisma.score.findFirst({
-      where: {
-        id: id,
-      },
-      select: {
-        projectId: true,
-      },
-    });
-    if (existingScore && existingScore.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for score creation ${existingScore.projectId}`,
-      );
-    }
-
-    const validatedScore = await validateAndInflateScore({
-      body,
-      scoreId: id,
-      projectId: apiScope.projectId,
-    });
-
-    return await prisma.score.upsert({
-      where: {
-        id_projectId: {
-          id,
-          projectId: apiScope.projectId,
-        },
-      },
-      create: {
-        ...validatedScore,
-      },
-      update: {
-        ...validatedScore,
-      },
-    });
-  }
-}
-
-export class SdkLogProcessor implements EventProcessor {
-  event: z.infer<typeof sdkLogEvent>;
-
-  constructor(event: z.infer<typeof sdkLogEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    return;
-  }
-
-  process() {
-    try {
-      logger.info("SDK Log", this.event);
-      return undefined;
-    } catch (error) {
-      return undefined;
-    }
-  }
-}
@@ -1,161 +0,0 @@
-import { env } from "node:process";
-import z from "zod";
-import { ForbiddenError, UnauthorizedError } from "../../../errors";
-import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
-import { getProcessorForEvent } from "./EventProcessor";
-import { ApiAccessScope } from "../../auth/types";
-import { backOff } from "exponential-backoff";
-import { Model } from "../../..";
-import { logger } from "../../logger";
-
-export type BatchResult = {
-  result: unknown;
-  id: string;
-  type: string;
-};
-
-type TokenCountInput = {
-  model: Model;
-  text: unknown;
-};
-
-export type LegacyIngestionAccessScope = Omit<
-  ApiAccessScope,
-  "orgId" | "plan" | "rateLimitOverrides"
->;
-
-type LegacyIngestionAuthHeaderVerificationResult =
-  | {
-      validKey: true;
-      scope: LegacyIngestionAccessScope;
-    }
-  | {
-      validKey: false;
-      error: string;
-    };
-
-export const handleBatch = async (
-  events: z.infer<typeof ingestionApiSchema>["batch"],
-  authCheck: LegacyIngestionAuthHeaderVerificationResult,
-  calculateTokenDelegate: (p: TokenCountInput) => number | undefined,
-) => {
-  logger.debug(`handling ingestion ${events.length} events`);
-
-  if (!authCheck.validKey) throw new UnauthorizedError(authCheck.error);
-
-  const results: BatchResult[] = []; // Array to store the results
-
-  const errors: {
-    error: unknown;
-    id: string;
-    type: string;
-  }[] = []; // Array to store the errors
-
-  for (const singleEvent of events) {
-    try {
-      const result = await retry(async () => {
-        return await handleSingleEvent(
-          singleEvent,
-          authCheck.scope,
-          calculateTokenDelegate,
-        );
-      });
-      results.push({
-        result: result,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      }); // Push each result into the array
-    } catch (error) {
-      // Handle or log the error if `handleSingleEvent` fails
-      logger.error("Error handling event:", error);
-      // Decide how to handle the error: rethrow, continue, or push an error object to results
-      // For example, push an error object:
-      errors.push({
-        error,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      });
-    }
-  }
-
-  return { results, errors };
-};
-
-async function retry<T>(request: () => Promise<T>): Promise<T> {
-  return await backOff(request, {
-    numOfAttempts: 5,
-    retry: (e: Error, attemptNumber: number) => {
-      if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
-        logger.info("not retrying auth error");
-        return false;
-      }
-      logger.info(`retrying processing events ${attemptNumber}`);
-      return true;
-    },
-  });
-}
-
-const handleSingleEvent = async (
-  event: IngestionEventType,
-  apiScope: LegacyIngestionAccessScope,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-) => {
-  const { body } = event;
-  let restEvent = body;
-  if ("input" in body) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { input, ...rest } = body;
-    restEvent = rest;
-  }
-  if ("output" in restEvent) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { output, ...rest } = restEvent;
-    restEvent = rest;
-  }
-
-  logger.debug(
-    `handling single event ${event.id} of type ${event.type}:  ${JSON.stringify({ body: restEvent })}`,
-  );
-
-  const cleanedEvent = cleanEvent(event) as IngestionEventType;
-
-  // Deny access to non-score events if the access level is not "all"
-  // This is an additional safeguard to auth checks in EventProcessor
-  if (
-    apiScope.accessLevel !== "all" &&
-    cleanedEvent.type !== eventTypes.SCORE_CREATE
-  ) {
-    throw new ForbiddenError("Access denied. Event type not allowed.");
-  }
-
-  return getProcessorForEvent(cleanedEvent, calculateTokenDelegate).process(
-    apiScope,
-  );
-};
-
-// cleans NULL characters from the event
-export function cleanEvent(obj: unknown): unknown {
-  if (typeof obj === "string") {
-    return obj.replace(/\u0000/g, "");
-  } else if (typeof obj === "object" && obj !== null) {
-    if (Array.isArray(obj)) {
-      return obj.map(cleanEvent);
-    } else {
-      // Here we assert that obj is a Record<string, unknown>
-      const objAsRecord = obj as Record<string, unknown>;
-      const newObj: Record<string, unknown> = {};
-      for (const key in objAsRecord) {
-        newObj[key] = cleanEvent(objAsRecord[key]);
-      }
-      return newObj;
-    }
-  } else {
-    return obj;
-  }
-}
-
-export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
-  val === undefined || val === null;
@@ -18,43 +18,51 @@ import {
  traceException,
 } from "../instrumentation";
 import { logger } from "../logger";
-import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
-import { LegacyIngestionQueue } from "../redis/legacyIngestion";
 import { redis } from "../redis/redis";
-import { handleBatch } from "./legacy";
-import {
-  StorageService,
-  StorageServiceFactory,
-} from "../services/StorageService";
-import { getProcessorForEvent } from "./legacy/EventProcessor";
 import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+import { uploadEventToS3 } from "../utils/eventLog";

 export type TokenCountDelegate = (p: {
  model: Model;
  text: unknown;
 }) => number | undefined;

-let s3StorageServiceClient: StorageService;
-
-const getS3StorageServiceClient = (bucketName: string): StorageService => {
-  if (!s3StorageServiceClient) {
-    s3StorageServiceClient = StorageServiceFactory.getInstance({
-      bucketName,
-      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
-      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
-      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
-      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
-      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
-    });
+/**
+ * Get the delay for the event based on the event type. Uses delay if set, 0 if current UTC timestamp is not between
+ * 23:45 and 00:15, and env.LANGFUSE_INGESTION_QUEUE_DELAY_MS otherwise.
+ * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
+ * @param delay - Delay overwrite. Used if non-null.
+ */
+const getDelay = (delay: number | null) => {
+  if (delay !== null) {
+    return delay;
  }
-  return s3StorageServiceClient;
+  const now = new Date();
+  const hours = now.getUTCHours();
+  const minutes = now.getUTCMinutes();
+
+  if ((hours === 23 && minutes >= 45) || (hours === 0 && minutes <= 15)) {
+    return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
+  }
+
+  // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
+  // we use this instead.
+  // Values should be revisited based on a cost/performance trade-off.
+  return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
 };

+/**
+ * Processes a batch of events.
+ * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
+ * @param authCheck - AuthHeaderValidVerificationResult
+ * @param delay - (Optional) Delay in ms to wait before processing events in the batch.
+ */
 export const processEventBatch = async (
  input: unknown[],
  authCheck: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
+  delay: number | null = null,
 ): Promise<{
  successes: { id: string; status: number }[];
  errors: {
@@ -99,7 +107,7 @@ export const processEventBatch = async (
        });
        return [];
      }
-      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+      if (!isAuthorized(parsed.data, authCheck)) {
        authenticationErrors.push({
          id: parsed.data.id,
          error: new UnauthorizedError("Access Scope Denied"),
@@ -157,9 +165,6 @@ export const processEventBatch = async (
   ********************/
  let s3UploadErrored = false;
  await instrumentAsync({ name: "s3-upload-events" }, async () => {
-    const s3Client = getS3StorageServiceClient(
-      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-    );
    // S3 Event Upload is blocking, but non-failing.
    // If a promise rejects, we log it below, but do not throw an error.
    // In this case, we upload the full batch into the Redis queue.
@@ -169,8 +174,21 @@ export const processEventBatch = async (
        // That way we batch updates from the same invocation into a single file and reduce
        // write operations on S3.
        const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-        return s3Client.uploadJson(
-          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+        return uploadEventToS3(
+          {
+            projectId: authCheck.scope.projectId,
+            entityType: getClickhouseEntityType(type),
+            entityId: eventBodyId,
+            eventId: key,
+            traceId:
+              data // Use the first truthy traceId for the event log.
+                .flatMap((event) =>
+                  "traceId" in event.body && event.body.traceId
+                    ? [event.body.traceId]
+                    : [],
+                )
+                .shift() ?? null,
+          },
          data,
        );
      }),
@@ -185,154 +203,65 @@ export const processEventBatch = async (
    });
  });

-  // This is a workaround to allow us to disable async ingestion processing for SDK CI testing
-  // TODO: remove this block after SDKs are ready for V3 async ingestion processing
-  if (env.LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED === "true") {
-    const result = await handleBatch(
-      sortedBatch,
-      authCheck,
-      tokenCountDelegate,
-    );
-
-    //  in case we did not return early, we return the result here
-    return aggregateBatchResult(
-      [...validationErrors, ...authenticationErrors, ...result.errors],
-      result.results,
-      authCheck.scope.projectId,
-    );
-  }
-
  // Send each event individually to IngestionQueue for ClickHouse processing
-  if (env.LANGFUSE_CLICKHOUSE_INGESTION_ENABLED === "true") {
-    if (s3UploadErrored) {
-      throw new Error(
-        "Failed to upload events to blob storage, aborting event processing",
-      );
-    }
-
-    if (redis) {
-      const queue = IngestionQueue.getInstance();
-      await Promise.all(
-        Object.keys(sortedBatchByEventBodyId).map(async (id) =>
-          queue
-            ? queue.add(
-                QueueJobs.IngestionJob,
-                {
-                  id: randomUUID(),
-                  timestamp: new Date(),
-                  name: QueueJobs.IngestionJob as const,
-                  payload: {
-                    data: {
-                      type: sortedBatchByEventBodyId[id].type,
-                      eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
-                    },
-                    authCheck,
-                  },
-                },
-                {
-                  delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
-                },
-              )
-            : Promise.reject("Failed to instantiate queue"),
-        ),
-      );
-      if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED !== "true") {
-        // If postgres ingestion is disabled, we return early
-        return aggregateBatchResult(
-          [...validationErrors, ...authenticationErrors],
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
-          authCheck.scope.projectId,
-        );
-      }
-    }
-  }
-
-  if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED === "true") {
-    // As part of the legacy processing we sent the entire batch to the worker.
-    if (redis) {
-      const queue = LegacyIngestionQueue.getInstance();
-
-      if (queue) {
-        let addToQueueFailed = false;
-
-        const queuePayload: LegacyIngestionEventType = !s3UploadErrored
-          ? {
-              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
-                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-                return {
-                  type,
-                  eventBodyId,
-                  eventId: key,
-                };
-              }),
-              authCheck,
-              useS3EventStore: true,
-            }
-          : { data: sortedBatch, authCheck, useS3EventStore: false };
-
-        try {
-          await queue.add(QueueJobs.LegacyIngestionJob, {
-            payload: queuePayload,
-            id: randomUUID(),
-            timestamp: new Date(),
-            name: QueueJobs.LegacyIngestionJob as const,
-          });
-        } catch (e: unknown) {
-          logger.warn(
-            "Failed to add batch to queue, falling back to sync processing",
-            e,
-          );
-          addToQueueFailed = true;
-        }
-
-        if (!addToQueueFailed) {
-          return aggregateBatchResult(
-            // we are not sending additional server errors to the client in case of early return
-            [...validationErrors, ...authenticationErrors],
-            sortedBatch.map((event) => ({ id: event.id, result: event })),
-            authCheck.scope.projectId,
-          );
-        }
-      } else {
-        logger.warn(
-          "Ingestion queue not initialized, falling back to sync processing",
-        );
-      }
-    }
-
-    /*******************
-     * SYNC PROCESSING *
-     *******************/
-    const result = await handleBatch(
-      sortedBatch,
-      authCheck,
-      tokenCountDelegate,
-    );
-
-    //  in case we did not return early, we return the result here
-    return aggregateBatchResult(
-      [...validationErrors, ...authenticationErrors, ...result.errors],
-      result.results,
-      authCheck.scope.projectId,
+  if (s3UploadErrored) {
+    throw new Error(
+      "Failed to upload events to blob storage, aborting event processing",
    );
  }

-  throw new Error(
-    "Either Clickhouse or Postgres ingestion (or both) must be enabled",
+  if (!redis) {
+    throw new Error("Redis not initialized, aborting event processing");
+  }
+
+  const queue = IngestionQueue.getInstance();
+  await Promise.all(
+    Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+      queue
+        ? queue.add(
+            QueueJobs.IngestionJob,
+            {
+              id: randomUUID(),
+              timestamp: new Date(),
+              name: QueueJobs.IngestionJob as const,
+              payload: {
+                data: {
+                  type: sortedBatchByEventBodyId[id].type,
+                  eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                  fileKey: sortedBatchByEventBodyId[id].key,
+                },
+                authCheck,
+              },
+            },
+            { delay: getDelay(delay) },
+          )
+        : Promise.reject("Failed to instantiate queue"),
+    ),
+  );
+
+  return aggregateBatchResult(
+    [...validationErrors, ...authenticationErrors],
+    sortedBatch.map((event) => ({ id: event.id, result: event })),
+    authCheck.scope.projectId,
  );
 };

 const isAuthorized = (
  event: IngestionEventType,
  authScope: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
 ): boolean => {
-  try {
-    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+  if (event.type === eventTypes.SDK_LOG) {
    return true;
-  } catch (error) {
-    return false;
  }
+
+  if (event.type === eventTypes.SCORE_CREATE) {
+    return (
+      authScope.scope.accessLevel === "scores" ||
+      authScope.scope.accessLevel === "all"
+    );
+  }
+
+  return authScope.scope.accessLevel === "all";
 };

 /**
@@ -66,8 +66,12 @@ const OpenAIUsageSchema = z
    prompt_tokens: z.number().nonnegative(),
    completion_tokens: z.number().nonnegative(),
    total_tokens: z.number().nonnegative(),
-    prompt_tokens_details: z.record(z.string(), z.number().nonnegative()),
-    completion_tokens_details: z.record(z.string(), z.number().nonnegative()),
+    prompt_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
+    completion_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
  })
  .strict()
  .transform((v) => {
@@ -112,13 +116,16 @@ export const UsageOrCostDetails = z
  .union([OpenAIUsageSchema, RawUsageOrCostDetails])
  .nullish();

+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 export const TraceBody = z.object({
  id: z.string().nullish(),
  timestamp: stringDateTime,
  name: z.string().max(1000).nullish(),
  externalId: z.string().nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
  sessionId: z.string().nullish(),
  userId: z.string().nullish(),
  metadata: jsonSchema.nullish(),
@@ -133,8 +140,8 @@ export const OptionalObservationBody = z.object({
  name: z.string().nullish(),
  startTime: stringDateTime,
  metadata: jsonSchema.nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
  level: z.nativeEnum(ObservationLevel).nullish(),
  statusMessage: z.string().nullish(),
  parentObservationId: z.string().nullish(),
@@ -226,6 +226,14 @@ export const recordHistogram = (
  dd.dogstatsd.histogram(stat, value, tags);
 };

+export const recordDistribution = (
+  stat: string,
+  value?: number | undefined,
+  tags?: { [tag: string]: string | number } | undefined,
+) => {
+  dd.dogstatsd.distribution(stat, value, tags);
+};
+
 /**
 * Converts a queue name to the matching datadog metric name.
 * Consumer only needs to append the relevant suffix.
@@ -1,7 +1,5 @@
 import type { ZodSchema } from "zod";

-import { CallbackHandler } from "langfuse-langchain";
-
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
@@ -21,27 +19,20 @@ import GCPServiceAccountKeySchema, {
  BedrockConfigSchema,
  BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
-import {
-  processEventBatch,
-  type TokenCountDelegate,
-} from "../ingestion/processEventBatch";
+import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
-import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
-
+import {
+  ChatMessage,
+  ChatMessageRole,
+  LLMAdapter,
+  ModelParams,
+  TraceParams,
+} from "./types";
+import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";

 type ProcessTracedEvents = () => Promise<void>;

-export type TraceParams = {
-  traceName: string;
-  traceId: string;
-  projectId: string;
-  tags: string[];
-  tokenCountDelegate: TokenCountDelegate;
-  authCheck: AuthHeaderValidVerificationResult;
-};
-
 type LLMCompletionParams = {
  messages: ChatMessage[];
  modelParams: ModelParams;
@@ -49,9 +40,11 @@ type LLMCompletionParams = {
  callbacks?: BaseCallbackHandler[];
  baseURL?: string;
  apiKey: string;
+  extraHeaders?: Record<string, string>;
  maxRetries?: number;
  config?: Record<string, string> | null;
  traceParams?: TraceParams;
+  throwOnError?: boolean; // default is true
 };

 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -100,6 +93,8 @@ export async function fetchLLMCompletion(
    maxRetries,
    config,
    traceParams,
+    extraHeaders,
+    throwOnError = true,
  } = params;

  let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
@@ -111,7 +106,6 @@ export async function fetchLLMCompletion(
      _isLocalEventExportEnabled: true,
      tags: traceParams.tags,
    });
-
    finalCallbacks.push(handler);

    processTracedEvents = async () => {
@@ -122,7 +116,6 @@ export async function fetchLLMCompletion(
        await processEventBatch(
          JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
          traceParams.authCheck,
-          traceParams.tokenCountDelegate,
        );
      } catch (e) {
        logger.error("Failed to process traced events", { error: e });
@@ -177,6 +170,7 @@ export async function fetchLLMCompletion(
      maxRetries,
      configuration: {
        baseURL,
+        defaultHeaders: extraHeaders,
      },
      timeout: 1000 * 60 * 2, // 2 minutes timeout
    });
@@ -237,17 +231,18 @@ export async function fetchLLMCompletion(
    runName: traceParams?.traceName,
  };

-  if (params.structuredOutputSchema) {
-    return {
-      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-        .withStructuredOutput(params.structuredOutputSchema)
-        .invoke(finalMessages, runConfig),
-      processTracedEvents,
-    };
-  }
+  try {
+    if (params.structuredOutputSchema) {
+      return {
+        completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+          .withStructuredOutput(params.structuredOutputSchema)
+          .invoke(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }

-  /*
-  Workaround OpenAI o1 while in beta:
+    /*
+  Workaround OpenAI reasoning models:
  
  This is a temporary workaround to avoid sending system messages to OpenAI's O1 models.
  O1 models do not support in beta:
@@ -258,43 +253,55 @@ export async function fetchLLMCompletion(

  Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
  */
-  if (modelParams.model.startsWith("o1-")) {
-    return {
-      completion: await new ChatOpenAI({
-        openAIApiKey: apiKey,
-        modelName: modelParams.model,
-        temperature: 1,
-        maxTokens: undefined,
-        topP: undefined,
-        callbacks,
-        maxRetries,
-        configuration: {
-          baseURL,
-        },
-        timeout: 1000 * 60 * 2, // 2 minutes timeout
-      })
-        .pipe(new StringOutputParser())
-        .invoke(
-          finalMessages.filter((message) => message._getType() !== "system"),
-          runConfig,
-        ),
-      processTracedEvents,
-    };
-  }
+    if (
+      modelParams.model.startsWith("o1-") ||
+      modelParams.model.startsWith("o3-")
+    ) {
+      return {
+        completion: await new ChatOpenAI({
+          openAIApiKey: apiKey,
+          modelName: modelParams.model,
+          temperature: 1,
+          maxTokens: undefined,
+          topP: undefined,
+          callbacks,
+          maxRetries,
+          modelKwargs: {
+            max_completion_tokens: modelParams.max_tokens,
+          },
+          configuration: {
+            baseURL,
+          },
+          timeout: 1000 * 60 * 2, // 2 minutes timeout
+        })
+          .pipe(new StringOutputParser())
+          .invoke(
+            finalMessages.filter((message) => message._getType() !== "system"),
+            runConfig,
+          ),
+        processTracedEvents,
+      };
+    }
+
+    if (streaming) {
+      return {
+        completion: await chatModel
+          .pipe(new BytesOutputParser())
+          .stream(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }

-  if (streaming) {
    return {
      completion: await chatModel
-        .pipe(new BytesOutputParser())
-        .stream(finalMessages, runConfig),
+        .pipe(new StringOutputParser())
+        .invoke(finalMessages, runConfig),
      processTracedEvents,
    };
+  } catch (error) {
+    if (throwOnError) {
+      throw error;
+    }
+    return { completion: null, processTracedEvents };
  }
-
-  return {
-    completion: await chatModel
-      .pipe(new StringOutputParser())
-      .invoke(finalMessages, runConfig),
-    processTracedEvents,
-  };
 }
@@ -1,6 +1,8 @@
 import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
+import { TokenCountDelegate } from "../ingestion/processEventBatch";
+import { AuthHeaderValidVerificationResult } from "../auth/types";

 export type PromptVariable = { name: string; value: string; isUsed: boolean };

@@ -16,7 +18,7 @@ export enum LLMAdapter {
  OpenAI = "openai",
  Azure = "azure",
  Bedrock = "bedrock",
-  VertexAI = "vertex-ai",
+  VertexAI = "google-vertex-ai",
 }

 export enum ChatMessageRole {
@@ -71,6 +73,7 @@ export const ExperimentMetadataSchema = z
    provider: z.string(),
    model: z.string(),
    model_params: ZodModelConfig,
+    error: z.string().optional(),
  })
  .strict();
 export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
@@ -82,6 +85,8 @@ export const openAIModels = [
  "gpt-4o-2024-05-13",
  "gpt-4o-mini",
  "gpt-4o-mini-2024-07-18",
+  "o3-mini",
+  "o3-mini-2025-01-31",
  "o1-preview",
  "o1-preview-2024-09-12",
  "o1-mini",
@@ -116,6 +121,10 @@ export const anthropicModels = [
 ] as const;

 export const vertexAIModels = [
+  "gemini-2.0-pro-exp-02-05",
+  "gemini-2.0-flash-001",
+  "gemini-2.0-flash-lite-preview-02-05",
+  "gemini-2.0-flash-exp",
  "gemini-1.5-pro",
  "gemini-1.5-flash",
  "gemini-1.0-pro",
@@ -147,6 +156,8 @@ export const LLMApiKeySchema = z
    provider: z.string(),
    displaySecretKey: z.string(),
    secretKey: z.string(),
+    extraHeaders: z.string().nullish(),
+    extraHeaderKeys: z.array(z.string()),
    baseURL: z.string().nullable(),
    customModels: z.array(z.string()),
    withDefaultModels: z.boolean(),
@@ -160,3 +171,12 @@ export type LLMApiKey =
  z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
    ? z.infer<typeof LLMApiKeySchema>
    : never;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};
@@ -0,0 +1,13 @@
+import { z } from "zod";
+
+import { decrypt } from "../../encryption";
+
+const ExtraHeaderSchema = z.record(z.string(), z.string());
+
+export function decryptAndParseExtraHeaders(
+  extraHeaders: string | null | undefined,
+) {
+  if (!extraHeaders) return;
+
+  return ExtraHeaderSchema.parse(JSON.parse(decrypt(extraHeaders)));
+}
@@ -383,6 +383,10 @@ export class FilterList {
    return this.filters.find(predicate);
  }

+  some(predicate: (filter: Filter) => boolean) {
+    return this.filters.some(predicate);
+  }
+
  length() {
    return this.filters.length;
  }
@@ -1,11 +1,4 @@
-import { ObservationView, Prisma } from "@prisma/client";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { observationsTableCols } from "../../observationsTable";
-import { TableFilters } from "./types";
+import { ObservationView } from "@prisma/client";

 type AdditionalObservationFields = {
  traceName: string | null;
@@ -26,147 +19,3 @@ export type IOAndMetadataOmittedObservations = Array<
  Omit<ObservationView, "input" | "output" | "metadata"> &
    AdditionalObservationFields
 >;
-
-export function parseGetAllGenerationsInput(filters: TableFilters) {
-  const searchCondition = filters.searchQuery
-    ? Prisma.sql`AND (
-        o."id" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."name" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."model" ILIKE ${`%${filters.searchQuery}%`} OR
-        t."name" ILIKE ${`%${filters.searchQuery}%`}
-      )`
-    : Prisma.empty;
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filters.filter ?? [],
-    observationsTableCols,
-    "observations",
-  );
-
-  const orderByCondition = orderByToPrismaSql(
-    filters.orderBy,
-    observationsTableCols,
-  );
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const startTimeFilter = filters.filter?.find(
-    (f) => f.column === "Start Time" && f.type === "datetime",
-  );
-
-  const datetimeFilter =
-    startTimeFilter && startTimeFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          startTimeFilter.operator,
-          startTimeFilter.value,
-        )
-      : Prisma.empty;
-
-  return {
-    searchCondition,
-    filterCondition,
-    orderByCondition,
-    datetimeFilter,
-  };
-}
-
-export function createGenerationsQuery({
-  projectId,
-  datetimeFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectIOAndMetadata = false,
-  selectScoreValues = false,
-}: {
-  projectId: string;
-  datetimeFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectIOAndMetadata?: boolean;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  WITH scores_avg AS (
-        SELECT
-          trace_id,
-          observation_id,
-          ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-          jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-        FROM (
-          SELECT
-            trace_id,
-            observation_id,
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            avg(value) avg_value,
-            comment
-          FROM
-            scores
-          WHERE
-            project_id = ${projectId}
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-          GROUP BY
-            trace_id,
-            observation_id,
-            name,
-            comment
-          ORDER BY
-            trace_id
-          ) tmp
-        GROUP BY
-          trace_id, 
-          observation_id
-      )
-      SELECT
-        ${selectScoreValues ? Prisma.sql`s_avg."scores_values" AS "scores",` : Prisma.empty}
-        o.id,
-        o.name,
-        o.model,
-        o."modelParameters",
-        o.start_time as "startTime",
-        o.end_time as "endTime",
-        ${selectIOAndMetadata ? Prisma.sql`o.input, o.output, o.metadata,` : Prisma.empty} 
-        o.trace_id as "traceId",
-        t.name as "traceName",
-        o.completion_start_time as "completionStartTime",
-        o.time_to_first_token as "timeToFirstToken",
-        o.prompt_tokens as "promptTokens",
-        o.completion_tokens as "completionTokens",
-        o.total_tokens as "totalTokens",
-        o.unit,
-        o.level,
-        o.status_message as "statusMessage",
-        o.version,
-        o.model_id as "modelId",
-        o.input_price as "inputPrice",
-        o.output_price as "outputPrice",
-        o.total_price as "totalPrice",
-        o.calculated_input_cost as "calculatedInputCost",
-        o.calculated_output_cost as "calculatedOutputCost",
-        o.calculated_total_cost as "calculatedTotalCost",
-        o."latency",
-        o.prompt_id as "promptId",
-        p.name as "promptName",
-        p.version as "promptVersion",
-        t.tags as "traceTags"
-      FROM observations_view o
-      JOIN traces t ON t.id = o.trace_id AND t.project_id = ${projectId}
-      LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
-      LEFT JOIN prompts p ON p.id = o.prompt_id AND p.project_id = ${projectId}
-      WHERE
-        o.project_id = ${projectId}
-        AND o.type = 'GENERATION'
-        ${datetimeFilter}
-        ${searchCondition}
-        ${filterCondition}
-        ${orderByCondition}
-      ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-      ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-    `;
-}
@@ -1,135 +0,0 @@
-import { Prisma } from "@prisma/client";
-import { TableFilters } from "./types";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { tracesTableCols } from "../../tableDefinitions/tracesTable";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-
-export function parseTraceAllFilters(input: TableFilters) {
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    input.filter ?? [],
-    tracesTableCols,
-    "traces",
-  );
-  const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime",
-  );
-
-  const observationTimeseriesFilter =
-    timeseriesFilter && timeseriesFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          timeseriesFilter.operator,
-          timeseriesFilter.value,
-        )
-      : Prisma.empty;
-
-  const searchCondition = input.searchQuery
-    ? Prisma.sql`AND (
-    t."id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."external_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."user_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."name" ILIKE ${`%${input.searchQuery}%`}
-  )`
-    : Prisma.empty;
-
-  return {
-    filterCondition,
-    orderByCondition,
-    observationTimeseriesFilter,
-    searchCondition,
-  };
-}
-
-export function createTracesQuery({
-  select,
-  projectId,
-  observationTimeseriesFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectScoreValues = false,
-}: {
-  select: Prisma.Sql;
-  projectId: string;
-  observationTimeseriesFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  SELECT
-      ${select}
-  FROM
-    "traces" AS t
-  LEFT JOIN LATERAL (
-    SELECT
-      SUM(prompt_tokens) AS "promptTokens",
-      SUM(completion_tokens) AS "completionTokens",
-      SUM(total_tokens) AS "totalTokens",
-      SUM(calculated_total_cost) AS "calculatedTotalCost",
-      SUM(calculated_input_cost) AS "calculatedInputCost",
-      SUM(calculated_output_cost) AS "calculatedOutputCost"
-    FROM
-      "observations_view"
-    WHERE
-      trace_id = t.id
-      AND "type" = 'GENERATION'
-      AND "project_id" = ${projectId}
-      ${observationTimeseriesFilter}
-  ) AS generation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      COUNT(*) AS "observationCount",
-      EXTRACT(EPOCH FROM COALESCE(MAX("end_time"), MAX("start_time"))) - EXTRACT(EPOCH FROM MIN("start_time"))::double precision AS "latency",
-      COALESCE(  
-        MAX(CASE WHEN level = 'ERROR' THEN 'ERROR' END),  
-        MAX(CASE WHEN level = 'WARNING' THEN 'WARNING' END),  
-        MAX(CASE WHEN level = 'DEFAULT' THEN 'DEFAULT' END),  
-        'DEBUG'  
-      ) AS "level"
-    FROM
-        "observations"
-    WHERE
-        trace_id = t.id
-        AND "project_id" = ${projectId}
-         ${observationTimeseriesFilter}
-  ) AS observation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-      jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-    FROM (
-        SELECT
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            AVG(value) avg_value
-        FROM
-            scores
-        WHERE
-            trace_id = t.id
-            AND t."project_id" = ${projectId}     
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-        GROUP BY
-            name
-    ) tmp
-  ) AS s_avg ON true
-  WHERE 
-    t."project_id" = ${projectId}
-    ${searchCondition}
-    ${filterCondition}
-  ${orderByCondition}
-  ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-`;
-}
@@ -1,8 +1,5 @@
 export { createSessionsAllQuery } from "./createSessionsAllQuery";
-export { createTracesQuery, parseTraceAllFilters } from "./createTracesQuery";
 export {
-  createGenerationsQuery,
-  parseGetAllGenerationsInput,
  type FullObservations,
  type FullObservationsWithScores,
  type IOAndMetadataOmittedObservations,
@@ -1,45 +1,16 @@
 import { z } from "zod";
-import { eventTypes, ingestionBatchEvent } from ".";
-
-export const LegacyIngestionEventFull = z.object({
-  useS3EventStore: z.literal(false),
-  data: ingestionBatchEvent,
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEventMeta = z.object({
-  useS3EventStore: z.literal(true),
-  data: z.array(
-    z.object({
-      type: z.nativeEnum(eventTypes),
-      eventBodyId: z.string(),
-      eventId: z.string(),
-    }),
-  ),
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEvent = z.discriminatedUnion("useS3EventStore", [
-  LegacyIngestionEventFull,
-  LegacyIngestionEventMeta,
-]);
+import { eventTypes } from ".";
+import {
+  BatchActionQuerySchema,
+  BatchActionType,
+} from "../features/batchAction/types";
+import { BatchExportTableName } from "../features/batchExport/types";

 export const IngestionEvent = z.object({
  data: z.object({
    type: z.nativeEnum(eventTypes),
    eventBodyId: z.string(),
+    fileKey: z.string().optional(),
  }),
  authCheck: z.object({
    validKey: z.literal(true),
@@ -86,6 +57,19 @@ export const ExperimentCreateEventSchema = z.object({
  runId: z.string(),
  description: z.string().optional(),
 });
+export const DataRetentionProcessingEventSchema = z.object({
+  projectId: z.string(),
+  retention: z.number(),
+});
+export const BatchActionProcessingEventSchema = z.object({
+  projectId: z.string(),
+  actionId: z.string(),
+  query: BatchActionQuerySchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+  cutoffCreatedAt: z.date(),
+  targetId: z.string().optional(),
+  type: z.nativeEnum(BatchActionType),
+});

 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
@@ -95,7 +79,6 @@ export type DatasetRunItemUpsertEventType = z.infer<
  typeof DatasetRunItemUpsertEventSchema
 >;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
-export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
 export type ExperimentCreateEventType = z.infer<
  typeof ExperimentCreateEventSchema
@@ -103,6 +86,12 @@ export type ExperimentCreateEventType = z.infer<
 export type PostHogIntegrationProcessingEventType = z.infer<
  typeof PostHogIntegrationProcessingEventSchema
 >;
+export type DataRetentionProcessingEventType = z.infer<
+  typeof DataRetentionProcessingEventSchema
+>;
+export type BatchActionProcessingEventType = z.infer<
+  typeof BatchActionProcessingEventSchema
+>;

 export enum QueueName {
  TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
@@ -113,12 +102,15 @@ export enum QueueName {
  BatchExport = "batch-export-queue",
  IngestionQueue = "ingestion-queue", // Process single events with S3-merge
  IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
-  LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
  CloudUsageMeteringQueue = "cloud-usage-metering-queue",
  ExperimentCreate = "experiment-create-queue",
  PostHogIntegrationQueue = "posthog-integration-queue",
  PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
  CoreDataS3ExportQueue = "core-data-s3-export-queue",
+  MeteringDataPostgresExportQueue = "metering-data-postgres-export-queue",
+  DataRetentionQueue = "data-retention-queue",
+  DataRetentionProcessingQueue = "data-retention-processing-queue",
+  BatchActionQueue = "batch-action-queue",
 }

 export enum QueueJobs {
@@ -128,7 +120,6 @@ export enum QueueJobs {
  DatasetRunItemUpsert = "dataset-run-item-upsert",
  EvaluationExecution = "evaluation-execution-job",
  BatchExportJob = "batch-export-job",
-  LegacyIngestionJob = "legacy-ingestion-job",
  CloudUsageMeteringJob = "cloud-usage-metering-job",
  IngestionJob = "ingestion-job",
  IngestionSecondaryJob = "secondary-ingestion-job",
@@ -136,6 +127,10 @@ export enum QueueJobs {
  PostHogIntegrationJob = "posthog-integration-job",
  PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
  CoreDataS3ExportJob = "core-data-s3-export-job",
+  MeteringDataPostgresExportJob = "metering-data-postgres-export-job",
+  DataRetentionJob = "data-retention-job",
+  DataRetentionProcessingJob = "data-retention-processing-job",
+  BatchActionProcessingJob = "batch-action-processing-job",
 }

 export type TQueueJobTypes = {
@@ -175,12 +170,6 @@ export type TQueueJobTypes = {
    payload: BatchExportJobType;
    name: QueueJobs.BatchExportJob;
  };
-  [QueueName.LegacyIngestionQueue]: {
-    timestamp: Date;
-    id: string;
-    payload: LegacyIngestionEventType;
-    name: QueueJobs.LegacyIngestionJob;
-  };
  [QueueName.IngestionQueue]: {
    timestamp: Date;
    id: string;
@@ -205,4 +194,16 @@ export type TQueueJobTypes = {
    payload: PostHogIntegrationProcessingEventType;
    name: QueueJobs.PostHogIntegrationProcessingJob;
  };
+  [QueueName.DataRetentionProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: DataRetentionProcessingEventType;
+    name: QueueJobs.DataRetentionProcessingJob;
+  };
+  [QueueName.BatchActionQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: BatchActionProcessingEventType;
+    name: QueueJobs.BatchActionProcessingJob;
+  };
 };
@@ -3,30 +3,30 @@ import { QueueName, TQueueJobTypes } from "../queues";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";

-export class LegacyIngestionQueue {
+export class BatchActionQueue {
  private static instance: Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.BatchActionQueue]
  > | null = null;

  public static getInstance(): Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.BatchActionQueue]
  > | null {
-    if (LegacyIngestionQueue.instance) return LegacyIngestionQueue.instance;
+    if (BatchActionQueue.instance) return BatchActionQueue.instance;

    const newRedis = createNewRedisInstance({
      enableOfflineQueue: false,
      ...redisQueueRetryOptions,
    });

-    LegacyIngestionQueue.instance = newRedis
-      ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
-          QueueName.LegacyIngestionQueue,
+    BatchActionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.BatchActionQueue]>(
+          QueueName.BatchActionQueue,
          {
            connection: newRedis,
            defaultJobOptions: {
              removeOnComplete: true,
-              removeOnFail: 500_000,
-              attempts: 5,
+              removeOnFail: 10_000,
+              attempts: 2,
              backoff: {
                type: "exponential",
                delay: 5000,
@@ -36,10 +36,10 @@ export class LegacyIngestionQueue {
        )
      : null;

-    LegacyIngestionQueue.instance?.on("error", (err) => {
-      logger.error("LegacyIngestionQueue error", err);
+    BatchActionQueue.instance?.on("error", (err) => {
+      logger.error("BatchActionQueue error", err);
    });

-    return LegacyIngestionQueue.instance;
+    return BatchActionQueue.instance;
  }
 }
@@ -45,6 +45,7 @@ export class CloudUsageMeteringQueue {
        QueueJobs.CloudUsageMeteringJob,
        {},
        {
+          // Run at minute 5 of every hour (e.g. 1:05, 2:05, 3:05, etc)
          repeat: { pattern: "5 * * * *" },
        },
      );
@@ -47,7 +47,7 @@ export class CoreDataS3ExportQueue {
          QueueJobs.CoreDataS3ExportJob,
          {},
          {
-            repeat: { pattern: "30 4 * * *" }, // every day at 4:30am
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
          },
        )
        .catch((err) => {
@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionProcessingQueue.instance) {
+      return DataRetentionProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 10000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionProcessingQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionProcessingQueue error", err);
+    });
+
+    return DataRetentionProcessingQueue.instance;
+  }
+}
@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionQueue.instance) {
+      return DataRetentionQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionQueue error", err);
+    });
+
+    if (DataRetentionQueue.instance) {
+      logger.debug("Scheduling jobs for DataRetentionQueue");
+      DataRetentionQueue.instance
+        .add(
+          QueueJobs.DataRetentionJob,
+          {},
+          {
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding DataRetentionQueue schedule", err);
+        });
+    }
+
+    return DataRetentionQueue.instance;
+  }
+}
@@ -6,18 +6,19 @@ import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
 import { IngestionQueue, SecondaryIngestionQueue } from "./ingestionQueue";
-import { LegacyIngestionQueue } from "./legacyIngestion";
 import { TraceUpsertQueue } from "./traceUpsert";
 import { TraceDeleteQueue } from "./traceDelete";
 import { ProjectDeleteQueue } from "./projectDelete";
 import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
 import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
 import { CoreDataS3ExportQueue } from "./coreDataS3ExportQueue";
+import { MeteringDataPostgresExportQueue } from "./meteringDataPostgresExportQueue";
+import { DataRetentionQueue } from "./dataRetentionQueue";
+import { DataRetentionProcessingQueue } from "./dataRetentionProcessingQueue";
+import { BatchActionQueue } from "./batchActionQueue";

 export function getQueue(queueName: QueueName): Queue | null {
  switch (queueName) {
-    case QueueName.LegacyIngestionQueue:
-      return LegacyIngestionQueue.getInstance();
    case QueueName.BatchExport:
      return BatchExportQueue.getInstance();
    case QueueName.CloudUsageMeteringQueue:
@@ -44,6 +45,14 @@ export function getQueue(queueName: QueueName): Queue | null {
      return SecondaryIngestionQueue.getInstance();
    case QueueName.CoreDataS3ExportQueue:
      return CoreDataS3ExportQueue.getInstance();
+    case QueueName.MeteringDataPostgresExportQueue:
+      return MeteringDataPostgresExportQueue.getInstance();
+    case QueueName.DataRetentionQueue:
+      return DataRetentionQueue.getInstance();
+    case QueueName.DataRetentionProcessingQueue:
+      return DataRetentionProcessingQueue.getInstance();
+    case QueueName.BatchActionQueue:
+      return BatchActionQueue.getInstance();
    default:
      const exhaustiveCheckDefault: never = queueName;
      throw new Error(`Queue ${queueName} not found`);
@@ -0,0 +1,63 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class MeteringDataPostgresExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      return MeteringDataPostgresExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    MeteringDataPostgresExportQueue.instance = newRedis
+      ? new Queue(QueueName.MeteringDataPostgresExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    MeteringDataPostgresExportQueue.instance?.on("error", (err) => {
+      logger.error("MeteringDataPostgresExportQueue error", err);
+    });
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      logger.debug("Scheduling jobs for MeteringDataPostgresExportQueue");
+      MeteringDataPostgresExportQueue.instance
+        .add(
+          QueueJobs.MeteringDataPostgresExportJob,
+          {},
+          {
+            repeat: { pattern: "30 2 * * *" }, // every day at 2:30am UTC
+          },
+        )
+        .catch((err) => {
+          logger.error(
+            "Error adding MeteringDataPostgresExportJob schedule",
+            err,
+          );
+        });
+    }
+
+    return MeteringDataPostgresExportQueue.instance;
+  }
+}
@@ -4,30 +4,12 @@ import {
  convertDateToClickhouseDateTime,
 } from "../clickhouse/client";
 import { logger } from "../logger";
-import { instrumentAsync } from "../instrumentation";
-import {
-  StorageService,
-  StorageServiceFactory,
-} from "../services/StorageService";
+import { getTracer, instrumentAsync } from "../instrumentation";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
-
-let s3StorageServiceClient: StorageService;
-
-const getS3StorageServiceClient = (bucketName: string): StorageService => {
-  if (!s3StorageServiceClient) {
-    s3StorageServiceClient = StorageServiceFactory.getInstance({
-      bucketName,
-      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
-      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
-      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
-      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
-      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
-    });
-  }
-  return s3StorageServiceClient;
-};
+import { context, trace } from "@opentelemetry/api";
+import { uploadEventToS3 } from "../utils/eventLog";

 export async function upsertClickhouse<
  T extends Record<string, unknown>,
@@ -40,9 +22,6 @@ export async function upsertClickhouse<
    // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
    span.setAttribute("ch.query.table", opts.table);

-    const s3Client = getS3StorageServiceClient(
-      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-    );
    await Promise.all(
      opts.records.map((record) => {
        // drop trailing s and pretend it's always a create.
@@ -52,11 +31,19 @@ export async function upsertClickhouse<
          // @ts-ignore - If it's an observation we now that `type` is a string
          eventType = `${record["type"].toLowerCase()}-create`;
        }
-        s3Client.uploadJson(
-          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+
+        const eventId = randomUUID();
+        return uploadEventToS3(
+          {
+            projectId: record.project_id as string,
+            entityType: getClickhouseEntityType(eventType),
+            entityId: record.id as string,
+            eventId,
+            traceId: record?.trace_id as string,
+          },
          [
            {
-              id: randomUUID(),
+              id: eventId,
              timestamp: new Date().toISOString(),
              type: eventType,
              body: opts.eventBodyMapper(record),
@@ -101,6 +88,64 @@ export async function upsertClickhouse<
  });
 }

+export async function* queryClickhouseStream<T>(opts: {
+  query: string;
+  params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
+}): AsyncGenerator<T> {
+  const tracer = getTracer("clickhouse-query-stream");
+  const span = tracer.startSpan("clickhouse-query-stream");
+
+  try {
+    const res = await context.with(
+      trace.setSpan(context.active(), span),
+      async () => {
+        // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+        span.setAttribute("ch.query.text", opts.query);
+
+        const res = await clickhouseClient(opts.clickhouseConfigs).query({
+          query: opts.query,
+          format: "JSONEachRow",
+          query_params: opts.params,
+        });
+        // same logic as for prisma. we want to see queries in development
+        if (env.NODE_ENV === "development") {
+          logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+        }
+
+        span.setAttribute("ch.queryId", res.query_id);
+
+        // add summary headers to the span. Helps to tune performance
+        const summaryHeader = res.response_headers["x-clickhouse-summary"];
+        if (summaryHeader) {
+          try {
+            const summary = Array.isArray(summaryHeader)
+              ? JSON.parse(summaryHeader[0])
+              : JSON.parse(summaryHeader);
+            for (const key in summary) {
+              span.setAttribute(`ch.${key}`, summary[key]);
+            }
+          } catch (error) {
+            logger.debug(
+              `Failed to parse clickhouse summary header ${summaryHeader}`,
+              error,
+            );
+          }
+        }
+        return res;
+      },
+    );
+
+    for await (const rows of res.stream<T>()) {
+      for (const row of rows) {
+        yield row.json();
+      }
+    }
+  } finally {
+    span.end();
+  }
+}
+
 export async function queryClickhouse<T>(opts: {
  query: string;
  params?: Record<string, unknown> | undefined;
@@ -59,7 +59,6 @@ export const getObservationsCostGroupedByName = async (
  const appliedFilter = chFilter.apply();

  const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-  // TODO: Validate whether we can filter traces on timestamp here.

  const query = `
    SELECT 
@@ -68,9 +67,10 @@ export const getObservationsCostGroupedByName = async (
      sumMap(usage_details)['total'] as sum_usage_details
    FROM observations o FINAL ${hasTraceFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
    WHERE project_id = {projectId: String}
-    AND ${appliedFilter.query}
+    ${appliedFilter.query ? `AND ${appliedFilter.query}` : ""}
    GROUP BY provided_model_name
    ORDER BY sumMap(cost_details)['total'] DESC
+    LIMIT 50
    `;

  const result = await queryClickhouse<{
@@ -145,23 +145,26 @@ export const getScoreAggregate = async (
 export const groupTracesByTime = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
  ).apply();

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
  const query = `
    SELECT 
-      ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
      count(*) as count
    FROM traces t FINAL
    WHERE project_id = {projectId: String}
    AND ${chFilter.query}
    GROUP BY timestamp
-    ${orderByTimeSeries(groupBy, "timestamp")}
+    ${orderByQuery}
    `;
-
  const result = await queryClickhouse<{
    timestamp: string;
    count: string;
@@ -170,11 +173,12 @@ export const groupTracesByTime = async (
    params: {
      projectId,
      ...chFilter.params,
+      ...orderByParams,
    },
  });

  return result.map((row) => ({
-    timestamp: new Date(row.timestamp),
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
    countTraceId: Number(row.count),
  }));
 };
@@ -182,7 +186,6 @@ export const groupTracesByTime = async (
 export const getObservationUsageByTime = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -200,9 +203,14 @@ export const getObservationUsageByTime = async (
      ) as DateTimeFilter | undefined)
    : undefined;

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time",
+  );
+
  const query = `
    SELECT 
-      ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
      sumMap(usage_details) as units,
      sumMap(cost_details) as cost, 
      provided_model_name
@@ -212,7 +220,7 @@ export const getObservationUsageByTime = async (
    AND ${appliedFilter.query}
    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
    GROUP BY start_time, provided_model_name
-    ${orderByTimeSeries(groupBy, "start_time")}
+    ${orderByQuery}
    `;

  const result = await queryClickhouse<{
@@ -225,6 +233,7 @@ export const getObservationUsageByTime = async (
    params: {
      projectId,
      ...appliedFilter.params,
+      ...orderByParams,
      ...(timeFilter
        ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
        : {}),
@@ -232,7 +241,7 @@ export const getObservationUsageByTime = async (
  });

  return result.map((row) => ({
-    start_time: new Date(row.start_time),
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time),
    units: Object.fromEntries(
      Object.entries(row.units ?? {}).map(([key, value]) => [
        key,
@@ -271,12 +280,15 @@ export const getDistinctModels = async (

  // No need for final as duplicates are caught by distinct anyway.
  const query = `
-    SELECT distinct(provided_model_name) as model
+    SELECT distinct(provided_model_name) as model, count(*) as count
    FROM observations o
    ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
    WHERE project_id = {projectId: String}
    AND ${appliedFilter.query}
    ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    GROUP BY provided_model_name
+    ORDER BY count(*) DESC
+    LIMIT 1000
    `;

  const result = await queryClickhouse<{ model: string }>({
@@ -296,7 +308,6 @@ export const getDistinctModels = async (
 export const getScoresAggregateOverTime = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -307,9 +318,14 @@ export const getScoresAggregateOverTime = async (
  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
  // TODO: Validate whether we can filter traces on timestamp here.

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
  const query = `
  SELECT 
-    ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
    name,
    data_type,
    source,
@@ -324,7 +340,7 @@ export const getScoresAggregateOverTime = async (
    name,
    data_type,
    source
-  ${orderByTimeSeries(groupBy, "timestamp")};
+  ${orderByQuery};
 `;

  const result = await queryClickhouse<{
@@ -338,11 +354,12 @@ export const getScoresAggregateOverTime = async (
    params: {
      projectId,
      ...appliedFilter.params,
+      ...orderByParams,
    },
  });

  return result.map((row) => ({
-    scoreTimestamp: new Date(row.timestamp),
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
    scoreName: row.name,
    scoreDataType: row.data_type,
    scoreSource: row.source,
@@ -418,7 +435,7 @@ export const getObservationLatencies = async (
  // Skipping FINAL here, as the quantiles are approximate to begin with.
  const query = `
    SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles,
      name
    FROM observations o
    ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -465,7 +482,7 @@ export const getTracesLatencies = async (
      select o.trace_id,
             t.name,
             o.project_id,
-             date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
+             date_diff('millisecond', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
      FROM traces t 
      JOIN observations o
      ON o.trace_id = t.id AND o.project_id = t.project_id
@@ -506,7 +523,6 @@ export const getTracesLatencies = async (
 export const getModelLatenciesOverTime = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -516,25 +532,33 @@ export const getModelLatenciesOverTime = async (

  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
  // Skipping FINAL here, as the quantiles are approximate to begin with.
  const query = `
  SELECT 
-    ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket")},
    provided_model_name,
-    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles
  FROM observations o
  ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
  WHERE project_id = {projectId: String}
  AND ${appliedFilter.query}
  GROUP BY provided_model_name, start_time_bucket
-  ${orderByTimeSeries(groupBy, "start_time_bucket")};
+  ${orderByQuery};
 `;

  const result = await queryClickhouse<{
    start_time_bucket: string;
    provided_model_name: string;
    quantiles: string[];
-  }>({ query, params: { projectId, ...appliedFilter.params } });
+  }>({
+    query,
+    params: { projectId, ...appliedFilter.params, ...orderByParams },
+  });

  return result.map((row) => ({
    p50: Number(row.quantiles[0]) / 1000,
@@ -543,14 +567,13 @@ export const getModelLatenciesOverTime = async (
    p95: Number(row.quantiles[3]) / 1000,
    p99: Number(row.quantiles[4]) / 1000,
    model: row.provided_model_name,
-    start_time: new Date(row.start_time_bucket),
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time_bucket),
  }));
 };

 export const getNumericScoreTimeSeries = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -559,9 +582,14 @@ export const getNumericScoreTimeSeries = async (

  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
  const query = `
    SELECT
-    ${selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp")},
    s.name as score_name,
    AVG(s.value) as avg_value
    FROM scores s final
@@ -569,11 +597,11 @@ export const getNumericScoreTimeSeries = async (
    WHERE s.project_id = {projectId: String}
    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
    GROUP BY score_name, score_timestamp
-    ${orderByTimeSeries(groupBy, "score_timestamp")}
+    ${orderByQuery}
  `;

-  return queryClickhouse<{
-    score_timestamp: Date;
+  const result = await queryClickhouse<{
+    score_timestamp: string;
    score_name: string;
    avg_value: number;
  }>({
@@ -581,14 +609,20 @@ export const getNumericScoreTimeSeries = async (
    params: {
      projectId,
      ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
    },
  });
+
+  return result.map((row) => ({
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.score_timestamp),
+    scoreName: row.score_name,
+    avgValue: Number(row.avg_value),
+  }));
 };

 export const getCategoricalScoreTimeSeries = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc | undefined,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -597,9 +631,14 @@ export const getCategoricalScoreTimeSeries = async (

  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
  const query = `
    SELECT
-    ${groupBy ? selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp") + ", " : ""}
+    ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp") + ", " : ""}
    s.name as score_name,
    s.data_type as score_data_type,
    s.source as score_source,
@@ -609,12 +648,12 @@ export const getCategoricalScoreTimeSeries = async (
    ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
    WHERE s.project_id = {projectId: String}
    ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
-    GROUP BY score_name, score_data_type, score_source, score_value ${groupBy ? ", score_timestamp" : ""}
-    ${groupBy ? orderByTimeSeries(groupBy, "score_timestamp") : ""}
+    GROUP BY score_name, score_data_type, score_source, score_value ${bucketSizeInSeconds ? ", score_timestamp" : ""}
+      ${orderByQuery}
  `;

-  return queryClickhouse<{
-    score_timestamp?: Date;
+  const result = await queryClickhouse<{
+    score_timestamp?: string;
    score_name: string;
    score_data_type: string;
    score_source: string;
@@ -625,14 +664,25 @@ export const getCategoricalScoreTimeSeries = async (
    params: {
      projectId,
      ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
    },
  });
+
+  return result.map((row) => ({
+    scoreTimestamp: row.score_timestamp
+      ? parseClickhouseUTCDateTimeFormat(row.score_timestamp)
+      : undefined,
+    scoreName: row.score_name,
+    scoreDataType: row.score_data_type,
+    scoreSource: row.score_source,
+    scoreValue: row.score_value,
+    count: Number(row.count),
+  }));
 };

 export const getObservationsStatusTimeSeries = async (
  projectId: string,
  filter: FilterState,
-  groupBy: DateTrunc | undefined,
 ) => {
  const chFilter = new FilterList(
    createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -641,9 +691,14 @@ export const getObservationsStatusTimeSeries = async (

  const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");

+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
  const query = `
    SELECT 
-      ${groupBy ? selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket") + ", " : ""}
+      ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket") + ", " : ""}
      count(*) as observation_count,
      level as level
    FROM observations o
@@ -651,8 +706,8 @@ export const getObservationsStatusTimeSeries = async (
    WHERE project_id = {projectId: String}
    AND o.level IS NOT NULL
    AND ${chFilterRes?.query}
-    GROUP BY level ${groupBy ? ", start_time_bucket" : ""}
-    ${groupBy ? orderByTimeSeries(groupBy, "start_time_bucket") : ""}
+    GROUP BY level ${bucketSizeInSeconds ? ", start_time_bucket" : ""}
+    ${orderByQuery}
  `;

  const result = await queryClickhouse<{
@@ -661,7 +716,11 @@ export const getObservationsStatusTimeSeries = async (
    level: string;
  }>({
    query,
-    params: { projectId, ...(chFilterRes ? chFilterRes.params : {}) },
+    params: {
+      projectId,
+      ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
+    },
  });

  return result.map((row) => ({
@@ -673,61 +732,70 @@ export const getObservationsStatusTimeSeries = async (
  }));
 };

-const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toIntervalYear(1)";
-      break;
-    case "month":
-      interval = "toIntervalMonth(1)";
-      break;
-    case "week":
-      interval = "toIntervalWeek(1)";
-      break;
-    case "day":
-      interval = "toIntervalDay(1)";
-      break;
-    case "hour":
-      interval = "toIntervalHour(1)";
-      break;
-    case "minute":
-      interval = "toIntervalMinute(1)";
-      break;
-    default:
-      return undefined;
+export const orderByTimeSeries = (
+  filter: FilterState,
+  col: string,
+): [string, { fromTime: number; toTime: number }, number] => {
+  const potentialBucketSizesSeconds = [
+    5, 10, 30, 60, 300, 600, 1800, 3600, 18000, 36000, 86400, 604800, 2592000,
+  ];
+
+  // Calculate time difference in seconds
+  const [from, to] = extractFromAndToTimestampsFromFilter(filter);
+
+  if (!from || !to) {
+    throw new Error("Time Filter is required for time series queries");
  }

-  return `ORDER BY ${col} ASC WITH FILL STEP ${interval}`;
+  const fromDate = new Date(from.value as Date);
+  const toDate = new Date(to.value as Date);
+
+  const diffInSeconds = Math.abs(toDate.getTime() - fromDate.getTime()) / 1000;
+
+  // choose the bucket size that is the closest to the desired number of buckets
+  const bucketSizeInSeconds = potentialBucketSizesSeconds.reduce(
+    (closest, size) => {
+      const diffFromDesiredBuckets = Math.abs(diffInSeconds / size - 50);
+      return diffFromDesiredBuckets < closest.diffFromDesiredBuckets
+        ? { size, diffFromDesiredBuckets }
+        : closest;
+    },
+    { size: 0, diffFromDesiredBuckets: Infinity },
+  ).size;
+
+  // Convert to interval string
+  const interval = `toIntervalSecond(${bucketSizeInSeconds})`;
+
+  return [
+    `ORDER BY ${col} ASC 
+    WITH FILL
+    FROM toStartOfInterval(toDateTime({fromTime: DateTime64(3)}), INTERVAL ${bucketSizeInSeconds} SECOND)
+    TO toDateTime({toTime: DateTime64(3)}) + INTERVAL ${bucketSizeInSeconds} SECOND
+    STEP ${interval}`,
+    { fromTime: fromDate.getTime(), toTime: toDate.getTime() },
+    bucketSizeInSeconds,
+  ];
 };

-const selectTimeseriesColumn = (
-  dateTrunc: DateTrunc,
+export const selectTimeseriesColumn = (
+  bucketSizeInSeconds: number,
  col: string,
  as: String,
 ) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toStartOfYear";
-      break;
-    case "month":
-      interval = "toStartOfMonth";
-      break;
-    case "week":
-      interval = "toStartOfWeek";
-      break;
-    case "day":
-      interval = "toStartOfDay";
-      break;
-    case "hour":
-      interval = "toStartOfHour";
-      break;
-    case "minute":
-      interval = "toStartOfMinute";
-      break;
-    default:
-      return undefined;
-  }
-  return `${interval}(${col}) as ${as}`;
+  return `toStartOfInterval(${col}, INTERVAL ${bucketSizeInSeconds} SECOND) as ${as}`;
+};
+
+export const extractFromAndToTimestampsFromFilter = (filter?: FilterState) => {
+  if (!filter)
+    throw new Error("Time Filter is required for time series queries");
+
+  const fromTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === ">" || f.operator === ">="),
+  );
+
+  const toTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === "<" || f.operator === "<="),
+  );
+
+  return [fromTimestamp[0], toTimestamp[0]];
 };
@@ -2,6 +2,7 @@ import {
  commandClickhouse,
  parseClickhouseUTCDateTimeFormat,
  queryClickhouse,
+  queryClickhouseStream,
  upsertClickhouse,
 } from "./clickhouse";
 import { ObservationLevel } from "@prisma/client";
@@ -414,7 +415,7 @@ export const getObservationsTable = async (
  const observationRecords = await getObservationsTableInternal<
    Omit<
      ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
    >
  >({
    ...opts,
@@ -431,7 +432,7 @@ export const getObservationsTable = async (
  return observationRecords.map((o) => {
    const trace = traces.find((t) => t.id === o.trace_id);
    return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
      latency: o.latency ? Number(o.latency) / 1000 : null,
      timeToFirstToken: o.time_to_first_token
        ? Number(o.time_to_first_token) / 1000
@@ -449,7 +450,7 @@ export const getObservationsTableWithModelData = async (
  const observationRecords = await getObservationsTableInternal<
    Omit<
      ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
    >
  >({
    ...opts,
@@ -490,7 +491,7 @@ export const getObservationsTableWithModelData = async (
    const trace = traces.find((t) => t.id === o.trace_id);
    const model = models.find((m) => m.id === o.internal_model_id);
    return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
      latency: o.latency ? Number(o.latency) / 1000 : null,
      timeToFirstToken: o.time_to_first_token
        ? Number(o.time_to_first_token) / 1000
@@ -517,6 +518,7 @@ const getObservationsTableInternal = async <T>(
      ? "count(*) as count"
      : `
        o.id as id,
+        o.type as type,
        o.project_id as "project_id",
        o.name as name,
        o."model_parameters" as model_parameters,
@@ -540,8 +542,8 @@ const getObservationsTableInternal = async <T>(
        o.prompt_name as "prompt_name",
        o.prompt_version as "prompt_version",
        internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`;
+        if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time)) as "time_to_first_token"`;

  const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
    opts;
@@ -680,7 +682,7 @@ const getObservationsTableInternal = async <T>(
        ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
        ${hasScoresFilter ? `LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id` : ""}
      WHERE ${appliedObservationsFilter.query}
-        AND o.type = 'GENERATION'
+        
        ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
        ${search.query}
      ${chOrderBy}
@@ -906,6 +908,7 @@ export const getObservationsGroupedByPromptName = async (

 export const getCostForTraces = async (
  projectId: string,
+  timestamp: Date,
  traceIds: string[],
 ) => {
  // Wrapping the query in a CTE allows us to skip FINAL which allows Clickhouse to use skip indexes.
@@ -915,6 +918,7 @@ export const getCostForTraces = async (
      FROM observations o
      WHERE o.project_id = {projectId: String}
      AND o.trace_id IN ({traceIds: Array(String)})
+      AND o.start_time >= {timestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}
      ORDER BY o.event_ts DESC
      LIMIT 1 BY o.id, o.project_id
    )
@@ -928,6 +932,7 @@ export const getCostForTraces = async (
    params: {
      projectId,
      traceIds,
+      timestamp: convertDateToClickhouseDateTime(timestamp),
    },
  });
  return res.length > 0 ? Number(res[0].total_cost) : undefined;
@@ -970,6 +975,27 @@ export const deleteObservationsByProjectId = async (projectId: string) => {
  });
 };

+export const deleteObservationsOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String}
+    AND start_time < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const getObservationsWithPromptName = async (
  projectId: string,
  promptNames: string[],
@@ -1010,7 +1036,7 @@ export const getObservationMetricsForPrompts = async (
                  end_time,
                  usage_details,
                  cost_details,
-                  dateDiff('milliseconds', start_time, end_time) AS latency_ms
+                  dateDiff('millisecond', start_time, end_time) AS latency_ms
              FROM observations
              FINAL
              WHERE (type = 'GENERATION') 
@@ -1073,7 +1099,7 @@ export const getLatencyAndTotalCostForObservations = async (
    SELECT
        id,
        cost_details['total'] AS total_cost,
-        dateDiff('milliseconds', start_time, end_time) AS latency_ms
+        dateDiff('millisecond', start_time, end_time) AS latency_ms
    FROM observations FINAL 
    WHERE project_id = {projectId: String} 
    AND id IN ({observationIds: Array(String)})
@@ -1105,7 +1131,7 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
    SELECT
        trace_id,
        sumMap(cost_details)['total'] AS total_cost,
-        dateDiff('milliseconds', min(start_time), max(end_time)) AS latency_ms
+        dateDiff('millisecond', min(start_time), max(end_time)) AS latency_ms
    FROM observations FINAL
    WHERE project_id = {projectId: String} 
    AND trace_id IN ({traceIds: Array(String)})
@@ -1214,23 +1240,23 @@ export const getTraceIdsForObservations = async (
  }));
 };

-export const getGenerationsForPostHog = async (
+export const getGenerationsForPostHog = async function* (
  projectId: string,
  minTimestamp: Date,
  maxTimestamp: Date,
-) => {
+) {
  const query = `
    SELECT
      o.name as name,
      o.start_time as start_time,
      o.id as id,
      o.total_cost as total_cost,
-      if(isNull(completion_start_time), NULL, date_diff('milliseconds', start_time, completion_start_time)) as time_to_first_token,
+      if(isNull(completion_start_time), NULL, date_diff('millisecond', start_time, completion_start_time)) as time_to_first_token,
      o.usage_details['total'] as input_tokens,
      o.usage_details['output'] as output_tokens,
      o.cost_details['total'] as total_tokens,
      o.project_id as project_id,
-      if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time) / 1000) as latency,
+      if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000) as latency,
      o.provided_model_name as model,
      o.level as level,
      o.version as version,
@@ -1252,7 +1278,7 @@ export const getGenerationsForPostHog = async (
    AND o.type = 'GENERATION'
  `;

-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
    query,
    params: {
      projectId,
@@ -1262,32 +1288,34 @@ export const getGenerationsForPostHog = async (
  });

  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.start_time,
-    langfuse_generation_name: record.name,
-    langfuse_trace_name: record.trace_name,
-    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
-    langfuse_id: record.id,
-    langfuse_cost_usd: record.total_cost,
-    langfuse_input_units: record.input_tokens,
-    langfuse_output_units: record.output_tokens,
-    langfuse_total_units: record.total_tokens,
-    langfuse_session_id: record.trace_session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
-    langfuse_latency: record.latency,
-    langfuse_time_to_first_token: record.time_to_first_token,
-    langfuse_release: record.trace_release,
-    langfuse_version: record.version,
-    langfuse_model: record.model,
-    langfuse_level: record.level,
-    langfuse_tags: record.trace_tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.start_time,
+      langfuse_generation_name: record.name,
+      langfuse_trace_name: record.trace_name,
+      langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
+      langfuse_id: record.id,
+      langfuse_cost_usd: record.total_cost,
+      langfuse_input_units: record.input_tokens,
+      langfuse_output_units: record.output_tokens,
+      langfuse_total_units: record.total_tokens,
+      langfuse_session_id: record.trace_session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+      langfuse_latency: record.latency,
+      langfuse_time_to_first_token: record.time_to_first_token,
+      langfuse_release: record.trace_release,
+      langfuse_version: record.version,
+      langfuse_model: record.model,
+      langfuse_level: record.level,
+      langfuse_tags: record.trace_tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };
@@ -9,7 +9,6 @@ import Decimal from "decimal.js";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
 import { parseJsonPrioritised } from "../../utils/json";
-import { jsonSchema } from "../../utils/zod";

 export const convertObservationToView = (
  record: ObservationRecordReadType,
@@ -62,15 +61,22 @@ export const convertObservation = (
      ? parseClickhouseUTCDateTimeFormat(record.end_time)
      : null,
    name: record.name ?? null,
-    metadata: record.metadata,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
    level: record.level as ObservationLevel,
    statusMessage: record.status_message ?? null,
    version: record.version ?? null,
    input: (record.input
-      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      ? parseJsonPrioritised(record.input)
      : null) as Prisma.JsonValue | null,
    output: (record.output
-      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      ? parseJsonPrioritised(record.output)
      : null) as Prisma.JsonValue | null,
    modelParameters: record.model_parameters
      ? JSON.parse(record.model_parameters)
@@ -3,6 +3,7 @@ import {
  commandClickhouse,
  parseClickhouseUTCDateTimeFormat,
  queryClickhouse,
+  queryClickhouseStream,
  upsertClickhouse,
 } from "./clickhouse";
 import { FilterList } from "../queries/clickhouse-sql/clickhouse-filter";
@@ -135,13 +136,16 @@ export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
  });
 };

-export const getScoresForTraces = async (
-  projectId: string,
-  traceIds: string[],
-  timestamp?: Date,
-  limit?: number,
-  offset?: number,
-) => {
+export type GetScoresForTracesProps = {
+  projectId: string;
+  traceIds: string[];
+  timestamp?: Date;
+  limit?: number;
+  offset?: number;
+};
+
+export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
+  const { projectId, traceIds, timestamp, limit, offset } = props;
  const query = `
      select 
        *
@@ -200,20 +204,24 @@ export const getScoresForObservations = async (
  return rows.map(convertToScore);
 };

-export const getScoresGroupedByNameSourceType = async (projectId: string) => {
+export const getScoresGroupedByNameSourceType = async (
+  projectId: string,
+  timestamp: Date | undefined,
+) => {
  // We mainly use queries like this to retrieve filter options.
  // Therefore, we can skip final as some inaccuracy in count is acceptable.
  const query = `
-      select 
-        name,
-        source,
-        data_type
-      from scores s
-      WHERE s.project_id = {projectId: String}
-      GROUP BY name, source, data_type
-      ORDER BY count() desc
-      LIMIT 1000;
-    `;
+    select 
+      name,
+      source,
+      data_type
+    from scores s
+    WHERE s.project_id = {projectId: String}
+    ${timestamp ? `AND s.timestamp >= {timestamp: DateTime64(3)}` : ""}
+    GROUP BY name, source, data_type
+    ORDER BY count() desc
+    LIMIT 1000;
+  `;

  const rows = await queryClickhouse<{
    name: string;
@@ -223,6 +231,9 @@ export const getScoresGroupedByNameSourceType = async (projectId: string) => {
    query: query,
    params: {
      projectId: projectId,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
    },
  });

@@ -525,6 +536,27 @@ export const deleteScoresByProjectId = async (projectId: string) => {
  });
 };

+export const deleteScoresOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String}
+    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const getNumericScoreHistogram = async (
  projectId: string,
  filter: FilterState,
@@ -666,11 +698,11 @@ export const getDistinctScoreNames = async (
  return rows.map((row) => row.name);
 };

-export const getScoresForPostHog = async (
+export const getScoresForPostHog = async function* (
  projectId: string,
  minTimestamp: Date,
  maxTimestamp: Date,
-) => {
+) {
  const query = `
    SELECT
      s.id as id,
@@ -694,7 +726,7 @@ export const getScoresForPostHog = async (
    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
  `;

-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
    query,
    params: {
      projectId,
@@ -704,24 +736,26 @@ export const getScoresForPostHog = async (
  });

  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.timestamp,
-    langfuse_score_name: record.name,
-    langfuse_score_value: record.value,
-    langfuse_score_comment: record.comment,
-    langfuse_trace_name: record.trace_name,
-    langfuse_id: record.id,
-    langfuse_session_id: record.trace_session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
-    langfuse_release: record.trace_release,
-    langfuse_tags: record.trace_tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.timestamp,
+      langfuse_score_name: record.name,
+      langfuse_score_value: record.value,
+      langfuse_score_comment: record.comment,
+      langfuse_trace_name: record.trace_name,
+      langfuse_id: record.id,
+      langfuse_session_id: record.trace_session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+      langfuse_release: record.trace_release,
+      langfuse_tags: record.trace_tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };
@@ -2,6 +2,7 @@ import {
  commandClickhouse,
  parseClickhouseUTCDateTimeFormat,
  queryClickhouse,
+  queryClickhouseStream,
  upsertClickhouse,
 } from "./clickhouse";
 import {
@@ -16,10 +17,7 @@ import {
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
-import { OrderByState } from "../../interfaces/orderBy";
-import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { UiColumnMapping } from "../../tableDefinitions";
-import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -32,7 +30,7 @@ import { env } from "../../env";
 export const checkTraceExists = async (
  projectId: string,
  traceId: string,
-  timestamp: Date | undefined,
+  timestamp: Date,
  filter: FilterState,
 ): Promise<boolean> => {
  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
@@ -63,18 +61,22 @@ export const checkTraceExists = async (
  const query = `
    WITH observations_agg AS (
        SELECT
-          
            multiIf(
              arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
              arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
              arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
              'DEBUG'
-            ) AS level,
+            ) AS aggregated_level,
+            countIf(level = 'ERROR') as error_count,
+            countIf(level = 'WARNING') as warning_count,
+            countIf(level = 'DEFAULT') as default_count,
+            countIf(level = 'DEBUG') as debug_count,
            trace_id,
            project_id
        FROM observations o FINAL 
        WHERE o.project_id = {projectId: String}
        ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        AND o.start_time >= {timestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}
        GROUP BY trace_id, project_id
      )
    SELECT 
@@ -84,7 +86,7 @@ export const checkTraceExists = async (
    ${observationFilterRes ? `INNER JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id` : ""}
    WHERE ${tracesFilterRes.query}
    AND t.project_id = {projectId: String}
-    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+    AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
    GROUP BY t.id, t.project_id
  `;

@@ -383,196 +385,6 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
  return rows;
 };

-export type SessionDataReturnType = {
-  session_id: string;
-  max_timestamp: string;
-  min_timestamp: string;
-  trace_ids: string[];
-  user_ids: string[];
-  trace_count: number;
-  trace_tags: string[];
-  total_observations: number;
-  duration: number;
-  session_usage_details: Record<string, number>;
-  session_cost_details: Record<string, number>;
-  session_input_cost: string;
-  session_output_cost: string;
-  session_total_cost: string;
-  session_input_usage: string;
-  session_output_usage: string;
-  session_total_usage: string;
-};
-
-export const getSessionsTableCount = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<{ count: string }>({
-    select: `
-      count(session_id) as count
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows.length > 0 ? Number(rows[0].count) : 0;
-};
-
-export const getSessionsTable = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
-    select: `
-    session_id, 
-    max_timestamp, 
-    min_timestamp, 
-    trace_ids, 
-    user_ids, 
-    trace_count, 
-    trace_tags,
-    total_observations,
-    duration,
-    session_usage_details,
-    session_cost_details,
-    session_input_cost,
-    session_output_cost,
-    session_total_cost,
-    session_input_usage,
-    session_output_usage,
-    session_total_usage
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows;
-};
-
-export type FetchSessionsTableProps = {
-  select: string;
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-};
-
-const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
-  const { select, projectId, filter, orderBy, limit, page } = props;
-
-  const { tracesFilter, scoresFilter, observationsFilter } =
-    getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
-
-  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
-
-  const tracesFilterRes = tracesFilter.apply();
-  const scoresAvgFilterRes = scoresFilter.apply();
-  const observationsStatsRes = observationsFilter.apply();
-
-  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
-    (f) =>
-      f.field === "min_timestamp" &&
-      (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  const singleTraceFilter = traceTimestampFilter
-    ? new FilterList([
-        new DateTimeFilter({
-          clickhouseTable: "traces",
-          field: "timestamp",
-          operator: traceTimestampFilter.operator,
-          value: traceTimestampFilter.value,
-        }),
-      ]).apply()
-    : undefined;
-
-  const query = `
-      WITH observations_agg AS (
-        SELECT o.trace_id,
-              count(*) as obs_count,
-              min(o.start_time) as min_start_time,
-              max(o.end_time) as max_end_time,
-              sumMap(usage_details) as sum_usage_details,
-              sumMap(cost_details) as sum_cost_details,
-              anyLast(project_id) as project_id
-        FROM observations o FINAL
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            -- Aggregate observations data at session level
-            sum(o.obs_count) as total_observations,
-            date_diff('milliseconds', min(min_start_time), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage
-        FROM traces t FINAL
-        LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id
-        WHERE t.session_id IS NOT NULL
-            AND t.project_id = {projectId: String}
-            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${select}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
-
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...observationsStatsRes.params,
-      ...scoresAvgFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
-    },
-  });
-
-  return res;
-};
-
 export const getTracesIdentifierForSession = async (
  projectId: string,
  sessionId: string,
@@ -630,6 +442,27 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
  });
 };

+export const deleteTracesOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM traces
+    WHERE project_id = {projectId: String}
+    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const deleteTracesByProjectId = async (projectId: string) => {
  const query = `
    DELETE FROM traces
@@ -820,18 +653,18 @@ export const getUserMetrics = async (
  }));
 };

-export const getTracesForPostHog = async (
+export const getTracesForPostHog = async function* (
  projectId: string,
  minTimestamp: Date,
  maxTimestamp: Date,
-) => {
+) {
  const query = `
    WITH observations_agg AS (
      SELECT o.project_id,
             o.trace_id,
             sum(total_cost) as total_cost,
             count(*) as observation_count,
-             date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds
+             date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds
      FROM observations o FINAL
      WHERE o.project_id = {projectId: String}
      AND o.start_time >= {minTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
@@ -858,7 +691,7 @@ export const getTracesForPostHog = async (
    AND t.timestamp <= {maxTimestamp: DateTime64(3)}
  `;

-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
    query,
    params: {
      projectId,
@@ -868,28 +701,30 @@ export const getTracesForPostHog = async (
  });

  const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.timestamp,
-    langfuse_id: record.id,
-    langfuse_trace_name: record.name,
-    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.id as string)}`,
-    langfuse_cost_usd: record.total_cost,
-    langfuse_count_observations: record.observation_count,
-    langfuse_session_id: record.session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.user_id || "langfuse_unknown_user",
-    langfuse_latency: record.latency,
-    langfuse_release: record.release,
-    langfuse_version: record.version,
-    langfuse_tags: record.tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.timestamp,
+      langfuse_id: record.id,
+      langfuse_trace_name: record.name,
+      langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.id as string)}`,
+      langfuse_cost_usd: record.total_cost,
+      langfuse_count_observations: record.observation_count,
+      langfuse_session_id: record.session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.user_id || "langfuse_unknown_user",
+      langfuse_latency: record.latency,
+      langfuse_release: record.release,
+      langfuse_version: record.version,
+      langfuse_tags: record.tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };

 export const getTracesByIdsForAnyProject = async (traceIds: string[]) => {
@@ -3,7 +3,6 @@ import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { parseJsonPrioritised } from "../../utils/json";
-import { jsonSchema } from "../../utils/zod";

 export const convertTraceDomainToClickhouse = (
  trace: Trace,
@@ -46,12 +45,19 @@ export const convertClickhouseToDomain = (
    sessionId: record.session_id ?? null,
    public: record.public,
    input: (record.input
-      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      ? parseJsonPrioritised(record.input)
      : null) as Prisma.JsonValue | null,
    output: (record.output
-      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      ? parseJsonPrioritised(record.output)
      : null) as Prisma.JsonValue | null,
-    metadata: record.metadata,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
    createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
    updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
    externalId: null,
@@ -1,5 +1,6 @@
 import { Readable } from "stream";
 import {
+  DeleteObjectsCommand,
  GetObjectCommand,
  ListObjectsV2Command,
  PutObjectCommand,
@@ -30,7 +31,7 @@ export interface StorageService {

  download(path: string): Promise<string>;

-  listFiles(prefix: string): Promise<string[]>;
+  listFiles(prefix: string): Promise<{ file: string; createdAt: Date }[]>;

  getSignedUrl(
    fileName: string,
@@ -45,6 +46,8 @@ export interface StorageService {
    contentType: string;
    contentLength: number;
  }): Promise<string>;
+
+  deleteFiles(paths: string[]): Promise<void>;
 }

 export class StorageServiceFactory {
@@ -201,7 +204,28 @@ class AzureBlobStorageService implements StorageService {
    }
  }

-  public async listFiles(prefix: string): Promise<string[]> {
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      await this.createContainerIfNotExists();
+
+      await Promise.all(
+        paths.map(async (path) => {
+          const blobClient = this.client.getBlobClient(path);
+          await blobClient.deleteIfExists();
+        }),
+      );
+    } catch (err) {
+      logger.error(
+        `Failed to delete files from Azure Blob Storage ${paths}`,
+        err,
+      );
+      throw Error("Failed to delete files from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
    try {
      await this.createContainerIfNotExists();

@@ -209,7 +233,10 @@ class AzureBlobStorageService implements StorageService {
      const files = [];
      for await (const blob of result) {
        if (blob.name.startsWith(prefix)) {
-          files.push(blob.name);
+          files.push({
+            file: blob.name,
+            createdAt: blob?.properties?.createdOn ?? new Date(),
+          });
        }
      }
      return files;
@@ -364,7 +391,9 @@ class S3StorageService implements StorageService {
    }
  }

-  public async listFiles(prefix: string): Promise<string[]> {
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
    const listCommand = new ListObjectsV2Command({
      Bucket: this.bucketName,
      Prefix: prefix,
@@ -373,7 +402,11 @@ class S3StorageService implements StorageService {
    try {
      const response = await this.client.send(listCommand);
      return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
+        response.Contents?.flatMap((file) =>
+          file.Key
+            ? [{ file: file.Key, createdAt: file.LastModified ?? new Date() }]
+            : [],
+        ) ?? []
      );
    } catch (err) {
      logger.error(`Failed to list files from S3 ${prefix}`, err);
@@ -404,6 +437,37 @@ class S3StorageService implements StorageService {
    }
  }

+  public async deleteFiles(paths: string[]): Promise<void> {
+    const chunkSize = 900;
+    const chunks = [];
+
+    for (let i = 0; i < paths.length; i += chunkSize) {
+      chunks.push(paths.slice(i, i + chunkSize));
+    }
+
+    try {
+      for (const chunk of chunks) {
+        const command = new DeleteObjectsCommand({
+          Bucket: this.bucketName,
+          Delete: {
+            Objects: chunk.map((path) => ({ Key: path })),
+            Quiet: true,
+          },
+        });
+        const result = await this.client.send(command);
+        if (result?.Errors && result?.Errors?.length > 0) {
+          logger.error("Failed to delete files from S3", {
+            errors: result.Errors,
+          });
+          throw new Error("Failed to delete files from S3");
+        }
+      }
+    } catch (err) {
+      logger.error(`Failed to delete files from S3`, err);
+      throw new Error("Failed to delete files from S3");
+    }
+  }
+
  public async getSignedUploadUrl(params: {
    path: string;
    ttlSeconds: number;
@@ -0,0 +1,319 @@
+import { OrderByState } from "../../interfaces/orderBy";
+import { sessionCols } from "../../tableDefinitions/mapSessionTable";
+import { FilterState } from "../../types";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
+import {
+  getProjectIdDefaultFilter,
+  createFilterFromFilterState,
+} from "../queries/clickhouse-sql/factory";
+import {
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+  queryClickhouse,
+} from "../repositories";
+
+export type SessionDataReturnType = {
+  session_id: string;
+  max_timestamp: string;
+  min_timestamp: string;
+  trace_ids: string[];
+  user_ids: string[];
+  trace_count: number;
+  trace_tags: string[];
+};
+
+export type SessionWithMetricsReturnType = SessionDataReturnType & {
+  total_observations: number;
+  duration: number;
+  session_usage_details: Record<string, number>;
+  session_cost_details: Record<string, number>;
+  session_input_cost: string;
+  session_output_cost: string;
+  session_total_cost: string;
+  session_input_usage: string;
+  session_output_usage: string;
+  session_total_usage: string;
+};
+
+export const getSessionsTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<{ count: string }>({
+    select: "count",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.length > 0 ? Number(rows[0].count) : 0;
+};
+
+export const getSessionsTable = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
+    select: "rows",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+  }));
+};
+
+export const getSessionsWithMetrics = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({
+    select: "metrics",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+    total_observations: Number(row.total_observations),
+  }));
+};
+
+export type FetchSessionsTableProps = {
+  select: "count" | "rows" | "metrics";
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
+  const { select, projectId, filter, orderBy, limit, page } = props;
+
+  let sqlSelect: string;
+  switch (select) {
+    case "count":
+      sqlSelect = "count(session_id) as count";
+      break;
+    case "rows":
+      sqlSelect = `
+          session_id, 
+          max_timestamp, 
+          min_timestamp, 
+          trace_ids, 
+          user_ids, 
+          trace_count, 
+          trace_tags`;
+      break;
+    case "metrics":
+      sqlSelect = `
+        session_id, 
+        max_timestamp, 
+        min_timestamp, 
+        trace_ids, 
+        user_ids, 
+        trace_count, 
+        trace_tags,
+        total_observations,
+        duration,
+        session_usage_details,
+        session_cost_details,
+        session_input_cost,
+        session_output_cost,
+        session_total_cost,
+        session_input_usage,
+        session_output_usage,
+        session_total_usage`;
+      break;
+    default:
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "s",
+  });
+
+  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
+
+  const tracesFilterRes = tracesFilter.apply();
+
+  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
+    (f) =>
+      f.field === "min_timestamp" &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const filters = [];
+  if (traceTimestampFilter) {
+    filters.push(
+      new DateTimeFilter({
+        clickhouseTable: "traces",
+        field: "timestamp",
+        operator: traceTimestampFilter.operator,
+        value: traceTimestampFilter.value,
+      }),
+    );
+  }
+
+  const additionalSingleTraceFilter = tracesFilter.find(
+    (f) => f.field === "bookmarked" || f.field === "session_id",
+  );
+
+  if (additionalSingleTraceFilter) {
+    filters.push(additionalSingleTraceFilter);
+  }
+
+  const singleTraceFilter =
+    filters.length > 0 ? new FilterList(filters).apply() : undefined;
+
+  const hasMetricsFilter =
+    tracesFilter.find((f) =>
+      [
+        "session_total_cost",
+        "session_input_cost",
+        "session_output_cost",
+        "duration",
+        "session_total_usage",
+        "session_output_usage",
+        "session_input_usage",
+      ].includes(f.field),
+    ) ||
+    (orderBy &&
+      [
+        "totalCost",
+        "inputCost",
+        "outputCost",
+        "sessionDuration",
+        "totalTokens",
+        "outputTokens",
+        "inputTokens",
+        "usage",
+      ].includes(orderBy?.column));
+
+  const selectMetrics = select === "metrics" || hasMetricsFilter;
+
+  // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
+  const query = `
+    WITH deduplicated_traces AS (
+      SELECT * EXCEPT input, output, metadata
+      FROM traces t
+      WHERE t.session_id IS NOT NULL 
+        AND t.project_id = {projectId: String}
+        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    deduplicated_observations AS (
+        SELECT * 
+        FROM observations o
+        WHERE o.project_id = {projectId: String}
+        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+        AND o.trace_id IN (
+          SELECT id
+          FROM deduplicated_traces
+        )
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    observations_agg AS (
+      SELECT o.trace_id,
+            count(*) as obs_count,
+            min(o.start_time) as min_start_time,
+            max(o.end_time) as max_end_time,
+            sumMap(usage_details) as sum_usage_details,
+            sumMap(cost_details) as sum_cost_details,
+            anyLast(project_id) as project_id
+      FROM deduplicated_observations o
+      WHERE o.project_id = {projectId: String}
+      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+      GROUP BY o.trace_id
+    ),
+    session_data AS (
+        SELECT
+            t.session_id,
+            anyLast(t.project_id) as project_id,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            groupArray(t.id) AS trace_ids,
+            groupUniqArray(t.user_id) AS user_ids,
+            count(*) as trace_count,
+            groupUniqArrayArray(t.tags) as trace_tags
+            -- Aggregate observations data at session level
+            ${
+              selectMetrics
+                ? `
+            ,
+            sum(o.obs_count) as total_observations,
+            date_diff('millisecond', min(min_start_time), max(max_end_time)) as duration,
+            sumMap(o.sum_usage_details) as session_usage_details,
+            sumMap(o.sum_cost_details) as session_cost_details,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+            sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                : ""
+            }
+        FROM deduplicated_traces t
+        ${
+          selectMetrics
+            ? `LEFT JOIN observations_agg o
+        ON t.id = o.trace_id AND t.project_id = o.project_id`
+            : ""
+        }
+        WHERE t.session_id IS NOT NULL
+            AND t.project_id = {projectId: String}
+            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        GROUP BY t.session_id
+    )
+    SELECT ${sqlSelect}
+    FROM session_data s
+    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    `;
+
+  const obsStartTimeValue = traceTimestampFilter
+    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
+    : null;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      projectId,
+      limit: limit,
+      offset: limit && page ? limit * page : 0,
+      ...tracesFilterRes.params,
+      ...singleTraceFilter?.params,
+      ...(obsStartTimeValue
+        ? { observationsStartTime: obsStartTimeValue }
+        : {}),
+    },
+  });
+
+  return res;
+};
@@ -70,6 +70,10 @@ export type TracesMetricsUiReturnType = {
  scores: ScoreAggregate;
  usageDetails: Record<string, number>;
  costDetails: Record<string, number>;
+  errorCount: bigint;
+  warningCount: bigint;
+  defaultCount: bigint;
+  debugCount: bigint;
 };

 export const convertToUiTableRows = (
@@ -125,6 +129,10 @@ export const convertToUITableMetrics = (
      ? new Decimal(row.cost_details.output)
      : null,
    level: row.level,
+    debugCount: BigInt(row.debug_count ?? 0),
+    warningCount: BigInt(row.warning_count ?? 0),
+    errorCount: BigInt(row.error_count ?? 0),
+    defaultCount: BigInt(row.default_count ?? 0),
  };
 };

@@ -138,6 +146,10 @@ export type TracesTableMetricsClickhouseReturnType = {
  usage_details: Record<string, number>;
  cost_details: Record<string, number>;
  scores_avg: Array<{ name: string; avg_value: number }>;
+  error_count: number | null;
+  warning_count: number | null;
+  default_count: number | null;
+  debug_count: number | null;
 };

 export type FetchTracesTableProps = {
@@ -225,7 +237,11 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
        os.latency_milliseconds / 1000 as latency,
        os.cost_details as cost_details,
        os.usage_details as usage_details,
-        os.level as level,
+        os.aggregated_level as level,
+        os.error_count as error_count,
+        os.warning_count as warning_count,
+        os.default_count as default_count,
+        os.debug_count as debug_count,
        os.observation_count as observation_count,
        s.scores_avg as scores_avg,
        t.public as public`;
@@ -245,7 +261,6 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
        t.public as public`;
      break;
    default:
-      const exhaustiveCheckDefault: never = select;
      throw new Error(`Unknown select type: ${select}`);
  }

@@ -322,19 +337,29 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
      clickhouseSelect: "toDate(t.timestamp)",
      uiTableName: "timestamp_to_date",
      uiTableId: "timestamp_to_date",
-      clickhouseTableName: "observations",
+      clickhouseTableName: "traces",
+    },
+    {
+      clickhouseSelect: "t.event_ts",
+      uiTableName: "event_ts",
+      uiTableId: "event_ts",
+      clickhouseTableName: "traces",
    },
  ];
  const chOrderBy = orderByToClickhouseSql(
    [
      defaultOrder
-        ? {
-            column: "timestamp_to_date",
-            order: orderBy.order,
-          }
+        ? [
+            {
+              column: "timestamp_to_date",
+              order: orderBy.order,
+            },
+            { column: "timestamp", order: orderBy.order },
+            { column: "event_ts", order: "DESC" as "DESC" },
+          ]
        : null,
      orderBy ?? null,
-    ],
+    ].flat(),
    orderByCols,
  );

@@ -342,7 +367,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
  // - we only join scores and observations if we really need them to speed up default views
  // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
  // - we filter the observations and scores as much as possible before joining them to traces.
-  // - we order by todate(timestamp), timestamp per default and do not use FINAL.
+  // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
  //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.

  const query = `
@@ -351,13 +376,17 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
        COUNT(*) AS observation_count,
          sumMap(usage_details) as usage_details,
          SUM(total_cost) AS total_cost,
-          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          countIf(level = 'ERROR') as error_count,
+          countIf(level = 'WARNING') as warning_count,
+          countIf(level = 'DEFAULT') as default_count,
+          countIf(level = 'DEBUG') as debug_count,
          multiIf(
            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
            'DEBUG'
-          ) AS level,
+          ) AS aggregated_level,
          sumMap(cost_details) as cost_details,
          trace_id,
          project_id
@@ -0,0 +1,52 @@
+import { EventLog } from "@prisma/client";
+import { prisma } from "../../db";
+import { logger } from "../logger";
+import { env } from "../../env";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
+
+let s3StorageServiceClient: StorageService;
+
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
+  }
+  return s3StorageServiceClient;
+};
+
+export const uploadEventToS3 = async (
+  event: Omit<
+    EventLog,
+    "createdAt" | "updatedAt" | "bucketPath" | "bucketName" | "id"
+  >,
+  data: Record<string, unknown>[],
+) => {
+  const bucketPath = `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${event.projectId}/${event.entityType}/${event.entityId}/${event.eventId}.json`;
+  if (env.LANGFUSE_S3_EVENT_UPLOAD_POSTGRES_LOG_ENABLED === "true") {
+    try {
+      await prisma.eventLog.create({
+        data: {
+          ...event,
+          bucketPath,
+          bucketName: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+        },
+      });
+    } catch (e) {
+      logger.error("Failed to write event log to Postgres", e);
+      // Fallthrough as this shouldn't block further execution right now.
+    }
+  }
+
+  return getS3StorageServiceClient(
+    env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+  ).uploadJson(bucketPath, data);
+};
@@ -26,6 +26,12 @@ export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [

 export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
  ...observationsTableTraceUiColumnDefinitions,
+  {
+    uiTableName: "type",
+    uiTableId: "type",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."type"',
+  },
  {
    uiTableName: "ID",
    uiTableId: "id",
@@ -68,7 +74,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
    uiTableId: "timeToFirstToken",
    clickhouseTableName: "observations",
    clickhouseSelect:
-      "if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time) / 1000)",
+      "if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time) / 1000)",
    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
    clickhouseTypeOverwrite: "Decimal64(3)",
  },
@@ -77,7 +83,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
    uiTableId: "latency",
    clickhouseTableName: "observations",
    clickhouseSelect:
-      "if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time) / 1000)",
+      "if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000)",
    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
    clickhouseTypeOverwrite: "Decimal64(3)",
  },
@@ -86,7 +92,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
    uiTableId: "tokensPerSecond",
    clickhouseTableName: "observations",
    clickhouseSelect:
-      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('milliseconds', start_time, end_time) / 1000))",
+      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
  },
  {
    uiTableName: "Input Cost ($)",
@@ -12,7 +12,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
    uiTableName: "Level",
    uiTableId: "level",
    clickhouseTableName: "observations",
-    clickhouseSelect: "level",
+    clickhouseSelect: "aggregated_level",
  },
  {
    uiTableName: "ID",
@@ -68,6 +68,30 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
    clickhouseTableName: "traces",
    clickhouseSelect: "tags",
  },
+  {
+    uiTableName: "Warning Level Count",
+    uiTableId: "warningCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "warning_count",
+  },
+  {
+    uiTableName: "Error Level Count",
+    uiTableId: "errorCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "error_count",
+  },
+  {
+    uiTableName: "Default Level Count",
+    uiTableId: "defaultCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "default_count",
+  },
+  {
+    uiTableName: "Debug Level Count",
+    uiTableId: "debugCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "debug_count",
+  },
  {
    uiTableName: "Input Tokens",
    uiTableId: "inputTokens",
@@ -88,6 +88,30 @@ export const tracesTableCols: ColumnDefinition[] = [
    internal: 'generation_metrics."completionTokens"',
    nullable: true,
  },
+  {
+    name: "Error Level Count",
+    id: "errorCount",
+    type: "number",
+    internal: 'generation_metrics."errorCount"',
+  },
+  {
+    name: "Warning Level Count",
+    id: "warningCount",
+    type: "number",
+    internal: 'generation_metrics."warningCount"',
+  },
+  {
+    name: "Default Level Count",
+    id: "defaultCount",
+    type: "number",
+    internal: 'generation_metrics."defaultCount"',
+  },
+  {
+    name: "Debug Level Count",
+    id: "debugCount",
+    type: "number",
+    internal: 'generation_metrics."debugCount"',
+  },
  {
    name: "Total Tokens",
    id: "totalTokens",
@@ -102,7 +126,6 @@ export const tracesTableCols: ColumnDefinition[] = [
    internal: 'generation_metrics."totalTokens"',
    nullable: true,
  },
-
  {
    name: "Scores",
    id: "scores_avg",
@@ -1,6 +1,4 @@
-import { z } from "zod";
-import { merge } from "lodash";
-import { JsonNested, jsonSchema, jsonSchemaNullable } from "./zod";
+import { JsonNested } from "./zod";

 /**
 * Deeply parses a JSON string or object for nested stringified JSON
@@ -10,7 +8,6 @@ import { JsonNested, jsonSchema, jsonSchemaNullable } from "./zod";
 export function deepParseJson(json: unknown): unknown {
  if (typeof json === "string") {
    try {
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
      const parsed = JSON.parse(json);
      if (typeof parsed === "number") return json; // numbers that were strings in the input should remain as strings
      return deepParseJson(parsed); // Recursively parse parsed value
@@ -40,61 +37,12 @@ export function deepParseJson(json: unknown): unknown {
  return json;
 }

-export const mergeJson = (
-  json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
-) => {
-  if (json1 === undefined) {
-    return json2;
-  }
-  return merge(json1, json2);
-};
-
 export const parseJsonPrioritised = (
  json: string,
-): z.infer<typeof jsonSchema> | string | undefined => {
+): JsonNested | string | undefined => {
  try {
-    const parsedJson = JSON.parse(json);
-
-    if (Object.keys(parsedJson).length === 0) {
-      return parsedJson;
-    }
-
-    const parsedArray = z.array(jsonSchemaNullable).safeParse(parsedJson);
-    if (parsedArray.success) {
-      return parsedArray.data;
-    }
-
-    const parsedObject = z.record(jsonSchemaNullable).safeParse(parsedJson);
-    if (parsedObject.success) {
-      return parsedObject.data;
-    }
-
-    return jsonSchema.parse(parsedJson);
+    return JSON.parse(json);
  } catch (error) {
-    const parsed = jsonSchema.safeParse(json);
-
-    return parsed.success ? parsed.data : json;
+    return json;
  }
 };
-
-export const convertRecordToJsonSchema = (
-  record: Record<string, string>,
-): JsonNested | undefined => {
-  const jsonSchema: JsonNested = {};
-
-  // if record is empty, return undefined
-  if (Object.keys(record).length === 0) {
-    return undefined;
-  }
-
-  for (const key in record) {
-    try {
-      jsonSchema[key] = JSON.parse(record[key]);
-    } catch (e) {
-      jsonSchema[key] = record[key];
-    }
-  }
-
-  return jsonSchema;
-};
@@ -4,22 +4,27 @@ export function getIsCharOrUnderscore(value: string): boolean {
  return charOrUnderscore.test(value);
 }

+// Regex for valid variable names (letters, underscores, starting with letter)
+export const VARIABLE_REGEX = /^[a-zA-Z][a-zA-Z_]*$/;
+
+// Regex to find variables in mustache syntax
+export const MUSTACHE_REGEX = /{{([^{}]*)}}+/g;
+
+// Regex to find multiline variables
+export const MULTILINE_VARIABLE_REGEX = /{{[^}]*\n[^}]*}}/g;
+
+// Regex to find unclosed variables
+export const UNCLOSED_VARIABLE_REGEX = /{{(?![^{]*}})/g;
+
+export function isValidVariableName(variable: string): boolean {
+  return VARIABLE_REGEX.test(variable);
+}
+
 export function extractVariables(mustacheString: string): string[] {
-  const mustacheRegex = /\{\{(.*?)\}\}/g;
-  const uniqueVariables = new Set<string>();
-
-  for (const match of mustacheString.matchAll(mustacheRegex)) {
-    uniqueVariables.add(match[1]);
-  }
-
-  for (const variable of uniqueVariables) {
-    // if validated fails, remove from set
-    if (!getIsCharOrUnderscore(variable)) {
-      uniqueVariables.delete(variable);
-    }
-  }
-
-  return Array.from(uniqueVariables);
+  const matches = Array.from(mustacheString.matchAll(MUSTACHE_REGEX))
+    .map((match) => match[1])
+    .filter(isValidVariableName);
+  return [...new Set(matches)];
 }

 export function stringifyValue(value: unknown) {
@@ -23,18 +23,18 @@ type Json = Root | { [key: string]: JsonNested } | JsonNested[];
 // Here, you define the schema recursively
 export const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
  z.union([
-    nestedLiteralSchema,
    z.array(jsonSchemaNullable),
    z.record(jsonSchemaNullable),
+    nestedLiteralSchema,
  ]),
 );

 // Root schema that does not allow nulls at the root level
 export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
  z.union([
-    rootLiteralSchema,
    z.array(jsonSchemaNullable),
    z.record(jsonSchemaNullable),
+    rootLiteralSchema,
  ]),
 );

@@ -0,0 +1,26 @@
+diff --git a/core/lib/oauth/client.js b/core/lib/oauth/client.js
+index 52c51eb6ff422dc0899ccec31baf3fa39e42eeae..bc50c35bb617d0e86b68ca42f64d44b475ba4abb 100644
+--- a/core/lib/oauth/client.js
+++ b/core/lib/oauth/client.js
+@@ -1,5 +1,7 @@
+ "use strict";
+ 
+var HttpsProxyAgent = require('https-proxy-agent').HttpsProxyAgent;
+
+ Object.defineProperty(exports, "__esModule", {
+   value: true
+ });
+@@ -7,7 +9,12 @@ exports.openidClient = openidClient;
+ var _openidClient = require("openid-client");
+ async function openidClient(options) {
+   const provider = options.provider;
+-  if (provider.httpOptions) _openidClient.custom.setHttpOptionsDefaults(provider.httpOptions);
+  let httpOptions = {};
+  if (provider.httpOptions) httpOptions = { ...provider.httpOptions };
+  if (process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY) {
+    httpOptions.agent = new HttpsProxyAgent(process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY);
+  }
+  _openidClient.custom.setHttpOptionsDefaults(httpOptions);
+   let issuer;
+   if (provider.wellKnown) {
+     issuer = await _openidClient.Issuer.discover(provider.wellKnown);
@@ -16,9 +16,6 @@
    "db:reset": {},
    "db:push": {},
    "db:seed": {},
-    "models:migrate": {
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
-    },
    "db:seed:examples": {},
    "dev": {
      "cache": false,
@@ -95,6 +95,9 @@ ARG BUILDPLATFORM

 WORKDIR /app

+ARG NEXT_PUBLIC_BUILD_ID
+ENV BUILD_ID=$NEXT_PUBLIC_BUILD_ID
+
 ENV NODE_ENV production
 # Uncomment the following line in case you want to disable telemetry during runtime.
 ENV NEXT_TELEMETRY_DISABLED 1
@@ -105,13 +108,16 @@ ENV NEXT_MANUAL_SIG_HANDLE true

 RUN apk add --no-cache dumb-init tzdata

-RUN addgroup --system --gid 1001 nodejs
-RUN adduser --system --uid 1001 nextjs
+# Don't run production as root
+ARG UID=1001
+ARG GID=1001
+RUN addgroup --system --gid ${GID} nodejs
+RUN adduser --system --uid ${UID} nextjs

-RUN npm install -g --no-package-lock --no-save prisma@5.22.0
+RUN npm install -g --no-package-lock --no-save prisma@6.3.0

 RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
-    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.0/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
+    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
    mv migrate /usr/bin/migrate

 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .
@@ -12,6 +12,11 @@ if [ -z "$DATABASE_URL" ]; then
        echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
        exit 1
    fi
+    if [ -n "$DATABASE_ARGS" ]; then
+        # Append ARGS to DATABASE_URL
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
+        export DATABASE_URL
+    fi
 fi

 # Check if CLICKHOUSE_URL is not set
@@ -22,7 +27,7 @@ fi

 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
-    export DIRECT_URL=$DATABASE_URL
+    export DIRECT_URL="${DATABASE_URL}"
 fi

 # Always execute the postgres migration, except when disabled.
@@ -12,8 +12,8 @@ import { env } from "./src/env.mjs";
 */
 const cspHeader = `
  default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://settings.crisp.chat https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com;
-  style-src 'self' 'unsafe-inline' https://client.crisp.chat;
+  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://settings.crisp.chat https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com;
+  style-src 'self' 'unsafe-inline' https://client.crisp.chat https://uptime.betterstack.com;
  img-src 'self' https: blob: data: http://localhost:* https://client.crisp.chat https://image.crisp.chat https://storage.crisp.chat;
  font-src 'self' https://client.crisp.chat;
  frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com https://game.crisp.chat;
@@ -22,7 +22,7 @@ const cspHeader = `
  base-uri 'self';
  form-action 'self';
  frame-ancestors 'none';
-  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://storage.crisp.chat wss://client.relay.crisp.chat wss://stream.relay.crisp.chat https://*.ingest.us.sentry.io;
+  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://storage.crisp.chat wss://client.relay.crisp.chat wss://stream.relay.crisp.chat https://*.ingest.us.sentry.io https://uptime.betterstack.com;
  media-src 'self' https: http://localhost:* https://client.crisp.chat;
  ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
  ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
@@ -128,22 +128,22 @@ const nextConfig = {
      // Required to check authentication status from langfuse.com
      ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
        ? [
-          {
-            source: "/api/auth/session",
-            headers: [
-              {
-                key: "Access-Control-Allow-Origin",
-                value: "https://langfuse.com",
-              },
-              { key: "Access-Control-Allow-Credentials", value: "true" },
-              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-              {
-                key: "Access-Control-Allow-Headers",
-                value: "Content-Type, Authorization",
-              },
-            ],
-          },
-        ]
+            {
+              source: "/api/auth/session",
+              headers: [
+                {
+                  key: "Access-Control-Allow-Origin",
+                  value: "https://langfuse.com",
+                },
+                { key: "Access-Control-Allow-Credentials", value: "true" },
+                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+                {
+                  key: "Access-Control-Allow-Headers",
+                  value: "Content-Type, Authorization",
+                },
+              ],
+            },
+          ]
        : []),
      // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
      {
@@ -1,6 +1,6 @@
 {
  "name": "web",
-  "version": "3.5.2",
+  "version": "3.26.0",
  "private": true,
  "license": "MIT",
  "engines": {
@@ -19,15 +19,15 @@
    "test-client": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
    "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
    "test:e2e": "dotenv -e ../.env -- playwright test",
-    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server",
-    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
+    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server"
  },
  "dependencies": {
    "@anthropic-ai/tokenizer": "^0.0.4",
    "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.3",
    "@baselime/trpc-opentelemetry-middleware": "^0.1.2",
    "@codemirror/lang-json": "^6.0.1",
-    "@codemirror/lint": "^6.8.0",
+    "@codemirror/language": "^6.10.1",
+    "@codemirror/lint": "^6.4.2",
    "@dnd-kit/core": "^6.1.0",
    "@dnd-kit/modifiers": "^7.0.0",
    "@dnd-kit/sortable": "^8.0.0",
@@ -41,6 +41,7 @@
    "@langchain/openai": "^0.3.14",
    "@langfuse/ee": "workspace:*",
    "@langfuse/shared": "workspace:*",
+    "@lezer/highlight": "^1.2.1",
    "@marsidev/react-turnstile": "^0.5.4",
    "@mui/x-tree-view": "^7.19.0",
    "@next-auth/prisma-adapter": "^1.0.7",
@@ -58,7 +59,7 @@
    "@opentelemetry/sdk-node": "^0.53.0",
    "@opentelemetry/sdk-trace-base": "^1.26.0",
    "@opentelemetry/sdk-trace-node": "^1.26.0",
-    "@prisma/instrumentation": "^5.22.0",
+    "@prisma/instrumentation": "^6.3.0",
    "@radix-ui/react-accordion": "^1.2.1",
    "@radix-ui/react-alert-dialog": "^1.1.2",
    "@radix-ui/react-avatar": "^1.1.1",
@@ -70,6 +71,7 @@
    "@radix-ui/react-icons": "^1.3.0",
    "@radix-ui/react-label": "^2.1.0",
    "@radix-ui/react-popover": "^1.1.2",
+    "@radix-ui/react-progress": "^1.1.1",
    "@radix-ui/react-radio-group": "^1.2.1",
    "@radix-ui/react-scroll-area": "^1.2.0",
    "@radix-ui/react-select": "^2.1.2",
@@ -84,7 +86,7 @@
    "@remixicon/react": "^4.2.0",
    "@repo/eslint-config": "workspace:*",
    "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.39.0",
+    "@sentry/nextjs": "^8.52.0",
    "@t3-oss/env-nextjs": "^0.11.1",
    "@tailwindcss/container-queries": "^0.1.1",
    "@tanstack/react-query": "^4.36.1",
@@ -94,25 +96,27 @@
    "@trpc/next": "^10.45.0",
    "@trpc/react-query": "^10.45.0",
    "@trpc/server": "^10.45.0",
-    "@uiw/codemirror-theme-github": "^4.23.0",
-    "@uiw/codemirror-theme-tokyo-night": "^4.23.5",
-    "@uiw/react-codemirror": "^4.21.25",
+    "@uiw/codemirror-themes": "^4.23.7",
+    "@uiw/react-codemirror": "^4.21.24",
    "ai": "^3.4.9",
    "bcryptjs": "^2.4.3",
-    "bullmq": "^5.12.10",
+    "bullmq": "^5.34.10",
    "class-variance-authority": "^0.7.0",
    "clsx": "^2.1.1",
-    "cmdk": "^1.0.0",
+    "cmdk": "^1.0.4",
    "core-js": "^3.38.1",
    "cors": "^2.8.5",
+    "csv-parse": "^5.6.0",
    "date-fns": "^3.3.1",
    "dd-trace": "^5.23.1",
    "decimal.js": "^10.4.3",
+    "diff": "^7.0.0",
    "dompurify": "^3.1.5",
+    "fastest-levenshtein": "^1.0.16",
    "graphql": "^16.9.0",
+    "https-proxy-agent": "^7.0.6",
    "ioredis": "^5.4.1",
    "ip-address": "^9.0.5",
-    "js-tiktoken": "^1.0.15",
    "kysely": "^0.27.4",
    "langchain": "^0.3.6",
    "lodash": "^4.17.21",
@@ -124,7 +128,8 @@
    "posthog-js": "^1.176.0",
    "posthog-node": "^4.3.1",
    "prexit": "^2.2.0",
-    "prisma": "^5.22.0",
+    "prisma": "^6.3.0",
+    "protobufjs": "^7.4.0",
    "rate-limiter-flexible": "^5.0.3",
    "react": "18.2.0",
    "react-day-picker": "^8.10.1",
@@ -138,11 +143,15 @@
    "react18-json-view": "^0.2.8-canary.6",
    "remark-gfm": "^4.0.0",
    "remark-math": "^6.0.0",
+    "remark-parse": "^11.0.0",
+    "remark-rehype": "^11.1.1",
+    "remark-stringify": "^11.0.0",
    "sonner": "^1.4.41",
    "stripe": "^17.4.0",
    "superjson": "2.2.1",
    "tailwind-merge": "^2.5.2",
    "tailwindcss-animate": "^1.0.7",
+    "unified": "^11.0.5",
    "use-query-params": "^2.2.1",
    "uuid": "^9.0.1",
    "vaul": "^0.9.1",
@@ -157,6 +166,7 @@
    "@testing-library/react": "^15.0.7",
    "@types/bcryptjs": "^2.4.6",
    "@types/cors": "^2.8.17",
+    "@types/diff": "^7.0.0",
    "@types/dompurify": "^3.0.5",
    "@types/eslint": "^8.56.7",
    "@types/jest": "^29.5.12",
@@ -175,7 +185,6 @@
    "jest": "^29.7.0",
    "jest-environment-jsdom": "^29.7.0",
    "node-mocks-http": "^1.14.1",
-    "postcss": "^8.4.47",
    "prettier": "^3.3.3",
    "prettier-plugin-tailwindcss": "^0.6.6",
    "tailwindcss": "^3.4.17",
@@ -193,7 +202,9 @@
  },
  "pnpm": {
    "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "0.16.21"
    }
  }
 }
@@ -53,6 +53,7 @@ components:
      properties:
        id:
          type: string
+          nullable: true
        traceId:
          type: string
          example: cdef-1234-5678-90ab
@@ -67,15 +68,19 @@ components:
            values must equal either 1 or 0 (true or false)
        observationId:
          type: string
+          nullable: true
        comment:
          type: string
+          nullable: true
        dataType:
          $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
          description: >-
            When set, must match the score value's type. If not set, will be
            inferred from the score value or config
        configId:
          type: string
+          nullable: true
          description: >-
            Reference a score config on a score. When set, the score name must
            equal the config name and scores must comply with the config's range
@@ -100,6 +105,7 @@ components:
          $ref: '#/components/schemas/ScoreSource'
        observationId:
          type: string
+          nullable: true
        timestamp:
          type: string
          format: date-time
@@ -111,10 +117,13 @@ components:
          format: date-time
        authorUserId:
          type: string
+          nullable: true
        comment:
          type: string
+          nullable: true
        configId:
          type: string
+          nullable: true
          description: >-
            Reference a score config on a score. When set, config and score name
            must be equal and value must comply to optionally defined numerical
@@ -166,6 +175,7 @@ components:
        value:
          type: number
          format: double
+          nullable: true
          description: >-
            Only defined if a config is linked. Represents the numeric category
            mapping of the stringValue
@@ -737,7 +737,7 @@ paths:
        - Ingestion
      parameters: []
      responses:
-        '200':
+        '207':
          description: ''
          content:
            application/json:
@@ -1426,6 +1426,76 @@ paths:
              schema: {}
      security:
        - BasicAuth: []
+  /api/public/v2/prompts/{name}/versions/{version}:
+    patch:
+      description: Update labels for a specific prompt version
+      operationId: promptVersion_update
+      tags:
+        - PromptVersion
+      parameters:
+        - name: name
+          in: path
+          description: The name of the prompt
+          required: true
+          schema:
+            type: string
+        - name: version
+          in: path
+          description: Version of the prompt to update
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Prompt'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                newLabels:
+                  type: array
+                  items:
+                    type: string
+                  description: >-
+                    New labels for the prompt version. Labels are unique across
+                    versions. The "latest" label is reserved and managed by
+                    Langfuse.
+              required:
+                - newLabels
  /api/public/v2/prompts/{promptName}:
    get:
      description: Get a prompt
@@ -3186,11 +3256,12 @@ components:
            to exact match, use `(?i)^modelname$`
        startDate:
          type: string
-          format: date
+          format: date-time
          nullable: true
          description: Apply only to generations which are newer than this ISO date.
        unit:
          $ref: '#/components/schemas/ModelUsageUnit'
+          nullable: true
          description: Unit used by this model.
        inputPrice:
          type: number
@@ -3226,7 +3297,6 @@ components:
        - id
        - modelName
        - matchPattern
-        - unit
        - isLangfuseManaged
    ModelUsageUnit:
      title: ModelUsageUnit
@@ -4279,6 +4349,7 @@ components:
          description: Apply only to generations which are newer than this ISO date.
        unit:
          $ref: '#/components/schemas/ModelUsageUnit'
+          nullable: true
          description: Unit used by this model.
        inputPrice:
          type: number
@@ -4311,7 +4382,6 @@ components:
      required:
        - modelName
        - matchPattern
-        - unit
    Observations:
      title: Observations
      type: object
@@ -4453,6 +4523,10 @@ components:
            type: string
          nullable: true
          description: List of tags to apply to all versions of this prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
      required:
        - name
        - prompt
@@ -4478,6 +4552,10 @@ components:
            type: string
          nullable: true
          description: List of tags to apply to all versions of this prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
      required:
        - name
        - prompt
@@ -4527,6 +4605,10 @@ components:
          description: >-
            List of tags. Used to filter via UI and API. The same across
            versions of a prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
      required:
        - name
        - version
@@ -1057,6 +1057,61 @@
        }
      ]
    },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Prompt Version",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Update",
+          "request": {
+            "description": "Update labels for a specific prompt version",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/v2/prompts/:name/versions/:version",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "v2",
+                "prompts",
+                ":name",
+                "versions",
+                ":version"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "name",
+                  "value": "",
+                  "description": "The name of the prompt"
+                },
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": "Version of the prompt to update"
+                }
+              ]
+            },
+            "header": [],
+            "method": "PATCH",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"newLabels\": [\n        \"example\"\n    ]\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
    {
      "_type": "container",
      "description": null,
@@ -1192,7 +1247,7 @@
            "auth": null,
            "body": {
              "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ],\n    \"commitMessage\": \"example\"\n}",
              "options": {
                "raw": {
                  "language": "json"
@@ -1,260 +0,0 @@
-// Description: New model definitions in Langfuse are automatically applied to new observations.
-// You can optionally run this script to apply new model definitions to existing observations.
-// See docs: https://langfuse.com/docs/deployment/self-host#migrate-models
-// Execute: `npm run models:migrate`
-
-import "dotenv/config";
-
-import { findModel } from "@langfuse/shared/src/server";
-import { prisma } from "@langfuse/shared/src/db";
-import lodash from "lodash";
-import { type Prisma } from "@langfuse/shared/src/db";
-import { tokenCount } from "@/src/features/ingest/usage";
-
-async function main() {
-  return await modelMatch();
-}
-
-// Call the function
-main().catch((err) => {
-  console.error("An error occurred:", err);
-});
-
-export async function modelMatch() {
-  console.log("Starting model match");
-  const start = Date.now();
-
-  const BATCH_SIZE = 10_000;
-  let continueLoop = true;
-  let index = 0;
-  let totalObservations = 0;
-
-  while (continueLoop) {
-    type ObservationSelect = {
-      model: string | null;
-      id: string;
-      projectId: string;
-      startTime: Date;
-      unit: string | null;
-      promptTokens: number;
-      completionTokens: number;
-      totalTokens: number;
-      input: Prisma.JsonValue;
-      output: Prisma.JsonValue;
-    };
-
-    const observations = await prisma.observation.findMany({
-      select: {
-        id: true,
-        startTime: true,
-        model: true,
-        unit: true,
-        projectId: true,
-        promptTokens: true,
-        completionTokens: true,
-        totalTokens: true,
-        input: true,
-        output: true,
-      },
-      orderBy: {
-        startTime: "desc",
-      },
-      where: {
-        internalModel: null,
-        type: "GENERATION",
-        model: { not: null },
-      },
-      take: BATCH_SIZE,
-      skip: index * BATCH_SIZE,
-    });
-
-    console.log(`Found ${observations.length} observations to migrate`);
-
-    type Config = {
-      startTime: Date;
-      model: string;
-      unit: string;
-      projectId: string | null;
-    };
-
-    interface GroupedObservations {
-      [key: string]: ObservationSelect[];
-    }
-
-    const groupedObservations = observations.reduce<GroupedObservations>(
-      (acc, observation) => {
-        const config = {
-          startTime: observation.startTime.toISOString().slice(0, 10),
-          model: observation.model,
-          unit: observation.unit,
-          projectId: observation.projectId,
-        };
-
-        const key = JSON.stringify(config);
-
-        // Ensure the array is initialized before using it
-        acc[key] = acc[key] ?? [];
-        acc[key]?.push(observation);
-
-        return acc;
-      },
-      {},
-    );
-
-    let updatedObservations = 0;
-    const dbPromises = [];
-
-    for (const [key, observationsGroup] of Object.entries(
-      groupedObservations,
-    )) {
-      const { startTime, model, unit, projectId } = JSON.parse(key) as Config;
-
-      console.log("Execute key: ", startTime, model, unit, projectId);
-
-      if (!projectId) {
-        throw new Error("No project id");
-      }
-
-      const foundModel = await findModel({
-        event: { projectId, model, unit, startTime: startTime },
-      });
-
-      console.log(
-        "Found model: ",
-        foundModel?.id,
-        " for key: ",
-        key,
-        " with observations: ",
-        observationsGroup.length,
-      );
-
-      if (foundModel) {
-        // find all the observations with all tokens 0 and tokenize them individually
-        const observationsWithAllTokensZero = observationsGroup.filter(
-          (observation) =>
-            observation.promptTokens === 0 &&
-            observation.completionTokens === 0 &&
-            observation.totalTokens === 0,
-        );
-
-        for (const observation of observationsWithAllTokensZero) {
-          console.log("Tokenizing observation: ", observation.id);
-          const newInputCount = tokenCount({
-            model: foundModel,
-            text: observation.input,
-          });
-          const newOutputCount = tokenCount({
-            model: foundModel,
-            text: observation.output,
-          });
-
-          dbPromises.push(
-            prisma.observation.update({
-              where: {
-                id: observation.id,
-              },
-              data: {
-                promptTokens: newInputCount,
-                completionTokens: newOutputCount,
-                totalTokens: (newInputCount ?? 0) + (newOutputCount ?? 0),
-                internalModel: foundModel.modelName,
-              },
-            }),
-          );
-        }
-
-        // for all remaining observations, batch update them with the model id
-        const observationsWithTokens = observationsGroup.filter(
-          (observation) =>
-            observation.promptTokens !== 0 ||
-            observation.completionTokens !== 0 ||
-            observation.totalTokens !== 0,
-        );
-
-        // Push the promise for updating observations into the array
-        lodash.chunk(observationsWithTokens, 32000).map((chunk) => {
-          dbPromises.push(
-            prisma.observation.updateMany({
-              where: {
-                id: {
-                  in: chunk.map((observation) => observation.id),
-                },
-              },
-              data: {
-                internalModel: foundModel.modelName,
-              },
-            }),
-          );
-        });
-
-        updatedObservations += observationsGroup.length;
-      } else {
-        lodash.chunk(observationsGroup, 32000).map((chunk) => {
-          dbPromises.push(
-            prisma.observation.updateMany({
-              where: {
-                id: {
-                  in: chunk.map((observation) => observation.id),
-                },
-              },
-              data: {
-                internalModel: "LANGFUSETMPNOMODEL",
-              },
-            }),
-          );
-        });
-
-        updatedObservations += observationsGroup.length;
-      }
-    }
-
-    totalObservations += updatedObservations;
-    // Wait for all update operations to complete
-    const promiseChunk = lodash.chunk(dbPromises, 10);
-
-    for (const promises of promiseChunk) {
-      console.log("Waiting for promises to complete", promises.length);
-      await Promise.all(promises);
-    }
-
-    console.log(
-      "Updated observations count: ",
-      updatedObservations,
-      " in total: ",
-      totalObservations,
-    );
-
-    console.log(updatedObservations, observations.length);
-
-    if (updatedObservations === 0) {
-      index++;
-    }
-
-    if (observations.length === 0) {
-      console.log("No more observations to migrate");
-      continueLoop = false;
-    }
-  }
-
-  let updatedCount;
-  do {
-    console.log(`Updating LANGFUSETMPNOMODEL ${updatedCount}`);
-    const result = await prisma.$queryRaw<[{ id: string }]>`
-    WITH to_update AS (
-      SELECT id 
-      FROM observations 
-      WHERE internal_model = 'LANGFUSETMPNOMODEL'
-      AND "type" = 'GENERATION'
-      LIMIT 50000
-    )
-    UPDATE observations
-    set internal_model = NULL
-    WHERE id IN (SELECT id FROM to_update)
-    RETURNING id;`;
-    updatedCount = result.length;
-  } while (updatedCount > 0);
-
-  const end = Date.now();
-
-  console.log(`Model match took ${end - start} ms`);
-}
@@ -123,7 +123,7 @@ test("Unauthenticated user should not be redirected to non-relative URLs after l
 test("Unauthenticated user should be redirected to relative URL after login", async ({
  page,
 }) => {
-  const relativeUrl = "/support";
+  const relativeUrl = "/setup";
  await page.goto(
    `/auth/sign-in?targetPath=${encodeURIComponent(relativeUrl)}`,
  );
@@ -23,11 +23,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
      createObservation({
        trace_id: createdTrace.id,
        project_id: createdTrace.project_id,
-        name: "observation-name",
+        name: "observation-name-1",
        end_time: new Date().getTime(),
        start_time: new Date().getTime() - 1000,
-        input: "input",
-        output: "output",
        provided_model_name: "model-1",
      }),
      createObservation({
@@ -35,9 +33,23 @@ describe("/api/public/metrics/daily API Endpoint", () => {
        project_id: createdTrace.project_id,
        name: "observation-name-2",
        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        provided_model_name: "model-1",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-3",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 100000,
+        provided_model_name: "model-2",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-4",
+        end_time: new Date().getTime(),
        start_time: new Date().getTime() - 100000,
-        input: "input-2",
-        output: "output-2",
        provided_model_name: "model-2",
      }),
    ];
@@ -56,17 +68,17 @@ describe("/api/public/metrics/daily API Endpoint", () => {

    const metric = metrics.body.data[0];
    expect(metric.countTraces).toBe(1);
-    expect(metric.countObservations).toBe(2);
+    expect(metric.countObservations).toBe(4);
    expect(metric.usage).toHaveLength(2);
-    expect(metric.totalCost).toBe(600);
+    expect(metric.totalCost).toBe(1200);
    for (const usage of metric.usage) {
      expect(usage.model).toMatch(/model-\d/g);
-      expect(usage.inputUsage).toBe(1234);
-      expect(usage.outputUsage).toBe(5678);
-      expect(usage.totalUsage).toBe(6912);
-      expect(usage.countObservations).toBe(1);
+      expect(usage.inputUsage).toBe(1234 * 2);
+      expect(usage.outputUsage).toBe(5678 * 2);
+      expect(usage.totalUsage).toBe(6912 * 2);
+      expect(usage.countObservations).toBe(2);
      expect(usage.countTraces).toBe(1);
-      expect(usage.totalCost).toBe(300);
+      expect(usage.totalCost).toBe(600);
    }
  });
 });
@@ -192,7 +192,6 @@ describe("Fetch datasets for UI presentation", () => {
    const runs = await createDatasetRunsTable({
      projectId,
      datasetId,
-      queryClickhouse: false,
      page: 0,
      limit: 10,
    });
@@ -238,7 +237,8 @@ describe("Fetch datasets for UI presentation", () => {
    expect(secondRun.run_id).toEqual(datasetRun2Id);
    expect(secondRun.run_description).toBeNull();
    expect(secondRun.run_metadata).toEqual({});
-    expect(secondRun.avgLatency).toEqual(1);
+    expect(secondRun.avgLatency).toBeGreaterThanOrEqual(1);
+    expect(secondRun.avgLatency).toBeLessThanOrEqual(1.002);
    expect(secondRun.avgTotalCost.toString()).toStrictEqual("300");

    expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
@@ -328,7 +328,6 @@ describe("Fetch datasets for UI presentation", () => {
    const runs = await createDatasetRunsTable({
      projectId,
      datasetId,
-      queryClickhouse: false,
      page: 0,
      limit: 10,
    });
@@ -0,0 +1,298 @@
+import { randomUUID } from "crypto";
+import { makeAPICall } from "@/src/__tests__/test-utils";
+import waitForExpect from "wait-for-expect";
+import {
+  getObservationById,
+  getScoreById,
+  getTraceById,
+} from "@langfuse/shared/src/server";
+import { v4 } from "uuid";
+import { prisma } from "@langfuse/shared/src/db";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("/api/public/ingestion API Endpoint", () => {
+  it.each([
+    [
+      "plain",
+      {
+        id: randomUUID(),
+        type: "trace-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          timestamp: new Date().toISOString(),
+        },
+      },
+    ],
+    [
+      "metadata+io",
+      {
+        id: randomUUID(),
+        type: "trace-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          timestamp: new Date().toISOString(),
+          metadata: { hello: "world" },
+          input: "input",
+          output: "output",
+        },
+      },
+    ],
+    [
+      "complex-io",
+      {
+        id: randomUUID(),
+        type: "trace-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          timestamp: new Date().toISOString(),
+          metadata: { hello: "world" },
+          input: ["hello", { world: "world" }, [1, 2, 3]],
+        },
+      },
+    ],
+  ])(
+    "should create traces via the ingestion API (%s)",
+    async (_name: string, entity: any) => {
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [entity],
+      });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const trace = await getTraceById(entity.body.id, projectId);
+        expect(trace).toBeDefined();
+        expect(trace!.id).toBe(entity.body.id);
+        expect(trace!.projectId).toBe(projectId);
+        expect(trace!.metadata).toEqual(entity.body?.metadata ?? {});
+        expect(trace!.input).toEqual(entity.body?.input ?? null);
+        expect(trace!.output).toEqual(entity.body?.output ?? null);
+      });
+    },
+  );
+
+  it.each([
+    [
+      "generation",
+      "GENERATION",
+      {
+        id: randomUUID(),
+        type: "generation-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          traceId: randomUUID(),
+          parentObservationId: randomUUID(),
+          startTime: new Date().toISOString(),
+          model: "gpt-4",
+          input: { text: "input" },
+          output: { text: "output" },
+        },
+      },
+    ],
+    [
+      "span",
+      "SPAN",
+      {
+        id: randomUUID(),
+        type: "span-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          traceId: randomUUID(),
+          startTime: new Date().toISOString(),
+        },
+      },
+    ],
+    [
+      "span-complex-io",
+      "SPAN",
+      {
+        id: randomUUID(),
+        type: "span-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          traceId: randomUUID(),
+          startTime: new Date().toISOString(),
+          input: ["hello", { world: "world" }, [1, 2, 3]],
+          output: ["hello", { world: [2, 3, "test"] }, [1, 2, 3]],
+        },
+      },
+    ],
+  ])(
+    "should create observations via the ingestion API (%s)",
+    async (_name: string, type: string, entity: any) => {
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [entity],
+      });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const observation = await getObservationById(
+          entity.body.id,
+          projectId,
+          true,
+        );
+        expect(observation).toBeDefined();
+        expect(observation!.id).toBe(entity.body.id);
+        expect(observation!.projectId).toBe(projectId);
+        expect(observation!.metadata).toEqual(entity.body?.metadata ?? {});
+        expect(observation!.input).toEqual(entity.body?.input ?? null);
+        expect(observation!.output).toEqual(entity.body?.output ?? null);
+        expect(observation!.type).toBe(type);
+      });
+    },
+  );
+
+  it.each([
+    [
+      "plain",
+      {
+        id: randomUUID(),
+        type: "score-create",
+        timestamp: new Date().toISOString(),
+        body: {
+          id: randomUUID(),
+          name: "score-name",
+          traceId: randomUUID(),
+          value: 100.5,
+          observationId: randomUUID(),
+        },
+      },
+    ],
+  ])(
+    "should create scores via the ingestion API (%s)",
+    async (_name: string, entity: any) => {
+      const response = await makeAPICall("POST", "/api/public/ingestion", {
+        batch: [entity],
+      });
+
+      expect(response.status).toBe(207);
+
+      await waitForExpect(async () => {
+        const score = await getScoreById(projectId, entity.body.id);
+        expect(score).toBeDefined();
+        expect(score!.id).toBe(entity.body.id);
+        expect(score!.projectId).toBe(projectId);
+        expect(score!.value).toEqual(100.5);
+      });
+    },
+  );
+
+  it("should fail for long trace name", async () => {
+    const traceId = v4();
+
+    const baseString =
+      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ";
+    const repeatCount = Math.ceil(1500 / baseString.length);
+    const name = baseString.repeat(repeatCount);
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name,
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+
+    expect(response.status).toBe(207);
+    expect("errors" in response.body).toBe(true);
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+    expect(response.body.errors.length).toBe(1);
+    expect(response.body.errors[0].message).toBe("Invalid request data");
+  });
+
+  // Disabled until eventLog becomes the default behaviour.
+  it.skip("should create a log entry for the S3 file", async () => {
+    const traceId = v4();
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [
+        {
+          id: v4(),
+          type: "trace-create",
+          timestamp: new Date().toISOString(),
+          body: {
+            id: traceId,
+            name: "Foo Bar",
+            userId: "user-1",
+            metadata: { key: "value" },
+            release: "1.0.0",
+            version: "2.0.0",
+          },
+        },
+      ],
+    });
+    expect(response.status).toBe(207);
+
+    const logs = await prisma.eventLog.findMany({
+      where: {
+        entityType: "trace",
+        entityId: traceId,
+      },
+    });
+    expect(logs.length).toBe(1);
+  });
+
+  it("#4900: should clear score comment on update with `null`", async () => {
+    const scoreId = randomUUID();
+    const score1 = {
+      id: randomUUID(),
+      type: "score-create",
+      timestamp: new Date().toISOString(),
+      body: {
+        id: scoreId,
+        name: "score-name",
+        traceId: randomUUID(),
+        value: 100.5,
+        observationId: randomUUID(),
+        comment: "Foo Bar",
+      },
+    };
+
+    const score2 = {
+      id: randomUUID(),
+      type: "score-create",
+      timestamp: new Date(Date.now() + 1000).toISOString(),
+      body: {
+        id: scoreId,
+        name: "score-name",
+        traceId: randomUUID(),
+        value: 100.5,
+        observationId: randomUUID(),
+        comment: null, // Explicitly set to null to clear the comment
+      },
+    };
+
+    const response = await makeAPICall("POST", "/api/public/ingestion", {
+      batch: [score1, score2],
+    });
+
+    expect(response.status).toBe(207);
+
+    await waitForExpect(async () => {
+      const score = await getScoreById(projectId, scoreId);
+      expect(score).toBeDefined();
+      expect(score!.id).toBe(scoreId);
+      expect(score!.projectId).toBe(projectId);
+      expect(score!.value).toEqual(100.5);
+      expect(score!.comment).toBe(null);
+    });
+  });
+});
@@ -204,6 +204,7 @@ describe("/api/public/observations API Endpoint", () => {
      trace_id: traceId,
      name: "generation-name",
      start_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
+      event_ts: new Date("2021-01-01T00:00:00.000Z").getTime(),
      end_time: new Date("2021-01-01T00:00:00.000Z").getTime(),
      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      type: "GENERATION",
@@ -214,6 +215,7 @@ describe("/api/public/observations API Endpoint", () => {
      trace_id: traceId,
      name: "generation-name",
      start_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
+      event_ts: new Date("2021-02-01T00:00:00.000Z").getTime(),
      end_time: new Date("2021-02-01T00:00:00.000Z").getTime(),
      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      type: "SPAN",
@@ -223,6 +225,7 @@ describe("/api/public/observations API Endpoint", () => {
      trace_id: traceId,
      name: "generation-name",
      start_time: new Date("2021-03-01T00:00:00.000Z").getTime(),
+      event_ts: new Date("2021-03-01T00:00:00.000Z").getTime(),
      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      type: "EVENT",
    });
@@ -231,6 +234,7 @@ describe("/api/public/observations API Endpoint", () => {
      trace_id: traceId,
      name: "generation-name",
      start_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
+      event_ts: new Date("2021-04-01T00:00:00.000Z").getTime(),
      end_time: new Date("2021-04-01T00:00:00.000Z").getTime(),
      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
      type: "GENERATION",
@@ -0,0 +1,116 @@
+import { prepareUsageDataForTimeseriesChart } from "@/src/features/dashboard/components/ModelUsageChart";
+import { orderByTimeSeries } from "@langfuse/shared/src/server";
+
+describe("orderByTimeSeries", () => {
+  test("should return correct bucket size and query for 1 hour time range", () => {
+    const filter = [
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: ">=" as const,
+        value: new Date("2024-01-01T00:00:00Z"),
+      },
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: "<=" as const,
+        value: new Date("2024-01-01T01:00:00Z"),
+      },
+    ];
+
+    const [query, params, bucketSize] = orderByTimeSeries(filter, "timestamp");
+
+    // For 1 hour difference, should pick 60 second buckets to get ~60 data points
+    expect(bucketSize).toBe(60);
+    expect(query).toBe(
+      "ORDER BY timestamp ASC \n    WITH FILL\n    FROM toStartOfInterval(toDateTime({fromTime: DateTime64(3)}), INTERVAL 60 SECOND)\n    TO toDateTime({toTime: DateTime64(3)}) + INTERVAL 60 SECOND\n    STEP toIntervalSecond(60)",
+    );
+    expect(params.fromTime).toBe(new Date("2024-01-01T00:00:00Z").getTime());
+    expect(params.toTime).toBe(new Date("2024-01-01T01:00:00Z").getTime());
+  });
+
+  test("should return correct bucket size and query for 1 day time range", () => {
+    const filter = [
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: ">=" as const,
+        value: new Date("2024-01-01T00:00:00Z"),
+      },
+      {
+        type: "datetime" as const,
+        column: "timestamp",
+        operator: "<=" as const,
+        value: new Date("2024-01-02T00:00:00Z"),
+      },
+    ];
+
+    const [query, params, bucketSize] = orderByTimeSeries(filter, "timestamp");
+
+    // For 24 hour difference, should pick 1800 second (30 min) buckets
+    expect(bucketSize).toBe(1800);
+    expect(query).toBe(
+      "ORDER BY timestamp ASC \n    WITH FILL\n    FROM toStartOfInterval(toDateTime({fromTime: DateTime64(3)}), INTERVAL 1800 SECOND)\n    TO toDateTime({toTime: DateTime64(3)}) + INTERVAL 1800 SECOND\n    STEP toIntervalSecond(1800)",
+    );
+    expect(params.fromTime).toBe(new Date("2024-01-01T00:00:00Z").getTime());
+    expect(params.toTime).toBe(new Date("2024-01-02T00:00:00Z").getTime());
+  });
+
+  test("should handle empty filter by using default time range", () => {
+    expect(() => orderByTimeSeries([], "timestamp")).toThrow(
+      "Time Filter is required for time series queries",
+    );
+  });
+
+  describe("aggregate time series for model cost and usage", () => {
+    it("should aggregate time series for model cost and usage", async () => {
+      const metricHistory = prepareUsageDataForTimeseriesChart(
+        ["gpt-4o-mini", "text-embedding-ada-002"],
+        [
+          {
+            startTime: "2025-02-10T13:30:00.000Z",
+            units: {
+              input: 422,
+              output: 61,
+              total: 483,
+            },
+            cost: {
+              input: 0.0000633,
+              output: 0.0000366,
+              total: 0.0000999,
+            },
+            model: "gpt-4o-mini",
+          },
+          {
+            startTime: "2025-02-10T13:30:00.000Z",
+            units: {
+              input: 6,
+              total: 6,
+            },
+            cost: {
+              total: 6e-7,
+            },
+            model: "text-embedding-ada-002",
+          },
+        ],
+      );
+
+      expect(metricHistory.get("total")).toEqual([
+        {
+          startTime: "2025-02-10T13:30:00.000Z",
+          units: 483,
+          cost: 0.0000999,
+          model: "gpt-4o-mini",
+          usageType: "total",
+        },
+        {
+          startTime: "2025-02-10T13:30:00.000Z",
+          units: 6,
+          cost: 6e-7,
+          model: "text-embedding-ada-002",
+          usageType: "total",
+        },
+      ]);
+    });
+  });
+});
@@ -312,4 +312,52 @@ describe("Clickhouse Traces Repository Test", () => {
    const exists = await checkTraceExists(projectId, traceId, new Date(), []);
    expect(exists).toBe(true);
  });
+  it("should check if trace exists with error level count > 0", async () => {
+    const traceId = v4();
+    const trace = createTrace({
+      id: traceId,
+      user_id: "user-1",
+      project_id: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
+      metadata: { key: "value" },
+      release: "1.0.0",
+      version: "2.0.0",
+    });
+
+    const observations = [
+      createObservation({
+        trace_id: trace.id,
+        project_id: trace.project_id,
+        name: "observation-name",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        input: "input",
+        output: "output",
+        provided_model_name: "model-1",
+        level: "ERROR",
+      }),
+      createObservation({
+        trace_id: trace.id,
+        project_id: trace.project_id,
+        name: "observation-name-2",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 100000,
+        input: "input-2",
+        output: "output-2",
+        provided_model_name: "model-2",
+      }),
+    ];
+
+    await createTracesCh([trace]);
+    await createObservationsCh(observations);
+
+    const exists = await checkTraceExists(projectId, traceId, new Date(), [
+      {
+        type: "number",
+        column: "errorCount",
+        operator: ">",
+        value: 0,
+      },
+    ]);
+    expect(exists).toBe(true);
+  });
 });
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`UPDATE "llm_api_keys" SET adapter = 'google-vertex-ai' WHERE adapter = 'vertex-ai';`