chore: release v3.26.0

* fix: fix charts

* security: upgrade jsonpath-plus

* security: upgrade jsonpath-plus

* fix

* fix

* fix

* fix

* push

* push

.env.dev.legacy.example

@@ -1,86 +0,0 @@
-# When adding additional environment variables, the schema in "/src/env.mjs"
-# should be updated accordingly.
-
-# Prisma
-# https://www.prisma.io/docs/reference/database-reference/connection-urls#env
-DIRECT_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-
-# Clickhouse
-CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
-CLICKHOUSE_URL="http://localhost:8123"
-CLICKHOUSE_USER="clickhouse"
-CLICKHOUSE_PASSWORD="clickhouse"
-CLICKHOUSE_CLUSTER_ENABLED="false"
-
-# Next Auth
-# You can generate a new secret on the command line with:
-# openssl rand -base64 32
-# https://next-auth.js.org/configuration/options#secret
-# NEXTAUTH_SECRET=""
-NEXTAUTH_URL="http://localhost:3000"
-NEXTAUTH_SECRET="secret"
-
-# Langfuse Cloud Environment
-NEXT_PUBLIC_LANGFUSE_CLOUD_REGION="DEV"
-
-# Langfuse experimental features
-LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES="true"
-
-# Salt for API key hashing
-SALT="salt"
-
-# Email
-EMAIL_FROM_ADDRESS=""  # Defines the email address to use as the from address.
-SMTP_CONNECTION_URL=""  # Defines the connection url for smtp server.
-
-# S3 Batch Exports
-LANGFUSE_S3_BATCH_EXPORT_ENABLED=true
-LANGFUSE_S3_BATCH_EXPORT_BUCKET=langfuse
-LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID=minio
-LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_BATCH_EXPORT_REGION=us-east-1
-LANGFUSE_S3_BATCH_EXPORT_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
-LANGFUSE_S3_BATCH_EXPORT_PREFIX=exports/
-
-# # S3 Media Upload LOCAL
-LANGFUSE_S3_MEDIA_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_MEDIA_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_MEDIA_UPLOAD_PREFIX=media/
-
-# S3 Event Bucket Upload
-## Set to true to test uploading all events to S3
-LANGFUSE_S3_EVENT_UPLOAD_BUCKET=langfuse
-LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID=minio
-LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=miniosecret
-LANGFUSE_S3_EVENT_UPLOAD_REGION=us-east-1
-LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT=http://localhost:9090
-## Necessary for minio compatibility
-LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
-LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
-
-# Set during docker build of application
-# Used to disable environment verification at build time
-# DOCKER_BUILD=1
-
-REDIS_HOST="127.0.0.1"
-REDIS_PORT=6379
-REDIS_AUTH="myredissecret"
-
-# openssl rand -hex 32 used only here
-ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
-
-# speeds up local development by not executing init scripts on server startup
-NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
-
-LANGFUSE_READ_FROM_POSTGRES_ONLY=true
-LANGFUSE_RETURN_FROM_CLICKHOUSE=false
-LANGFUSE_READ_DASHBOARDS_FROM_CLICKHOUSE=false
-LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false

.env.prod.example

@@ -55,6 +55,7 @@ OTEL_SERVICE_NAME="langfuse"
 
 # Auth, optional configuration
 # AUTH_DOMAINS_WITH_SSO_ENFORCEMENT=domain1.com,domain2.com
+# AUTH_IGNORE_ACCOUNT_FIELDS=foo,bar
 # AUTH_DISABLE_USERNAME_PASSWORD=true
 # AUTH_DISABLE_SIGNUP=true
 # AUTH_SESSION_MAX_AGE=43200 # 30 days in minutes (default)
@@ -100,7 +101,9 @@ OTEL_SERVICE_NAME="langfuse"
 # AUTH_CUSTOM_ISSUER=
 # AUTH_CUSTOM_NAME=
 # AUTH_CUSTOM_SCOPE="openid email profile" # optional
+# AUTH_CUSTOM_CLIENT_AUTH_METHOD="client_secret_basic" # optional
 # AUTH_CUSTOM_ALLOW_ACCOUNT_LINKING=false
+# AUTH_CUSTOM_ID_TOKEN=false # optional, default is true
 
 # Transactional email, optional
 # Defines the email address to use as the from address.
@@ -243,13 +246,12 @@ OTEL_SERVICE_NAME="langfuse"
 # CLICKHOUSE_URL=
 # CLICKHOUSE_USER=
 # CLICKHOUSE_PASSWORD=
+# CLICKHOUSE_DB=
 
 # Ingestion
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_BATCH_SIZE=
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS= 
 # LANGFUSE_INGESTION_CLICKHOUSE_MAX_ATTEMPTS=
-# LANGFUSE_LEGACY_INGESTION_WORKER_CONCURRENCY=
-# QUEUE_CONSUMER_LEGACY_INGESTION_QUEUE_IS_ENABLED="true"
 
-## END Langfuse V3 Ingestion
+## END Langfuse V3 Ingestion

.github/workflows/codespell.yml

@@ -3,9 +3,13 @@ name: Codespell
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - "v*"
   pull_request:
-    branches: [main]
+    branches:
+      - "**"
   merge_group:
 
 permissions:

.github/workflows/pipeline.yml

@@ -17,8 +17,22 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
+  pre-job:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    timeout-minutes: 15
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          do_not_skip: '["workflow_dispatch"]'
+
   lint:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
@@ -41,6 +55,9 @@ jobs:
   test-docker-build:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -74,6 +91,9 @@ jobs:
   tests-web-sync:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-web-sync (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }})
     strategy:
       matrix:
@@ -87,7 +107,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -111,8 +131,9 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.legacy.example > .env
-          echo "LANGFUSE_POSTGRES_INGESTION_ENABLED=true" >> .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -146,6 +167,9 @@ jobs:
   tests-web-async:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-web-async (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
@@ -160,7 +184,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -184,7 +208,9 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev${{ matrix.blob-provider }}.example .env
-          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^REDIS_HOST=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev${{ matrix.blob-provider }}.example > .env
+          echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev${{ matrix.blob-provider }}.yml up -d
@@ -216,6 +242,9 @@ jobs:
   tests-worker:
     timeout-minutes: 20
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     name: tests-worker (node${{ matrix.node-version }}, pg${{ matrix.postgres-version }}, mode${{ matrix.blob-provider }})
     strategy:
       matrix:
@@ -248,7 +277,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -281,6 +310,9 @@ jobs:
 
   e2e-tests:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
@@ -322,6 +354,9 @@ jobs:
 
   e2e-server-tests:
     runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
     steps:
       - uses: actions/checkout@v4
       - name: Login to Docker Hub
@@ -343,7 +378,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.16.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -459,9 +494,9 @@ jobs:
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (web)
         uses: docker/build-push-action@v4
         with:
@@ -487,9 +522,9 @@ jobs:
             type=ref,event=pr
             type=sha
             type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') }}
+            type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref, '-rc') }}
+            type=semver,pattern={{major}},enable=${{ !contains(github.ref, '-rc') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v3') && !contains(github.ref, '-rc') }}
       - name: Build and push Docker image (worker)
         uses: docker/build-push-action@v4
         with:

.github/workflows/snyk.yml

@@ -6,7 +6,7 @@ on:
   push:
     branches:
       - main
-  merge_group:
+  # Snyk cannot upload results in merge group. Hence, we only run on PRs and when pushingon the main branch https://github.com/github/codeql-action/issues/1572
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.gitignore

@@ -39,7 +39,6 @@ yarn-error.log*
 !.env.dev.example
 !.env.dev-azure.example
 !.env.prod.example
-!.env.dev.legacy.example
 
 # vercel
 .vercel

CONTRIBUTING.md

@@ -121,36 +121,40 @@ flowchart TB
         end
     end
 
-    DB[Postgres Database]
-    Redis[Redis Cache/Queue]
-    Clickhouse[Clickhouse Database]
+    subgraph s9 ["VPC (US and EU separated)"]
+        DB[Postgres Database]
+        Redis[Redis Cache/Queue]
+        Clickhouse[Clickhouse Database]
+
+        subgraph s1["Application (langfuse/langfuse/web)"]
+            API[Public HTTP API]
+            G[TRPC API]
+            I[NextAuth]
+            H[React Frontend]
+            ORM
+            H --> G
+            H --> I
+            G --> I
+            G --- ORM
+            API --- ORM
+            I --- ORM
+        end
+
+        subgraph s5["Application (langfuse/langfuse/worker)"]
+            Worker
+        end
+
+        Worker --- DB
+        Worker --- Redis
+        Worker --- Clickhouse
+
+        ORM --- DB
+        ORM --- Redis
+        ORM --- Clickhouse
+
 
-    subgraph s1["Application (langfuse/langfuse/web)"]
-        API[Public HTTP API]
-        G[TRPC API]
-        I[NextAuth]
-        H[React Frontend]
-        ORM
-        H --> G
-        H --> I
-        G --> I
-        G --- ORM
-        API --- ORM
-        I --- ORM
     end
 
-    subgraph s5["Application (langfuse/langfuse/worker)"]
-        Worker
-    end
-
-    Worker --- DB
-    Worker --- Redis
-    Worker --- Clickhouse
-
-    ORM --- DB
-    ORM --- Redis
-    ORM --- Clickhouse
-
     JS --- API
     Python --- API
 ```
@@ -397,7 +401,7 @@ The background color of the following component will be `hsl(var(--primary))` an
 | --primary-accent         | Primary accent color used for branding                             | Layout                           |
 | --hover-primary-accent   | Primary accent color used for hover effects for links              | SignIn and AuthCloudRegionSwitch |
 | --muted-green            | Muted green for Event label                                        | ObservationTree                  |
-| --muted-orange           | Muted orange for Generation label                                  | ObservationTree                  |
+| --muted-magenta          | Muted magenta for Generation label                                 | ObservationTree                  |
 | --muted-blue             | Muted blue for Span label                                          | ObservationTree                  |
 | --muted-gray             | Muted gray for disabled status badges                              | StatusBadge                      |
 | --accent-light-green     | Light green accent for background of output and assistant messages | IOPreview, Generations, Traces   |
@@ -439,7 +443,7 @@ You can update the default AI models and prices by adding or updating an entry i
 Please note that
 
 - prices are in USD
-- the list is ordered by ID, so make sure to keep this order
+- the list is ordered by ID, so make sure to keep this order and insert new models at the end of the list
 - the `updated_at` field must be updated with the current date in ISO 8601 format. Otherwise, the change will be ignored.
 
 ### Transition period until V3 release

README.md

@@ -42,9 +42,7 @@
 
 ## Langfuse Overview
 
-_Unmute video for voice-over_
-
-https://github.com/langfuse/langfuse/assets/2834609/a94062e9-c782-4ee9-af59-dee6370149a8
+[![Langfuse Overview Video](https://github.com/user-attachments/assets/3926b288-ff61-4b95-8aa1-45d041c70866)](https://langfuse.com/watch-demo)
 
 ### Develop
 
@@ -194,37 +192,51 @@ You can opt-out by setting `TELEMETRY_ENABLED=false`.
 
 ### Open Source Projects Using Langfuse
 
-Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents/python.md)):
+Top open-source Python projects that use Langfuse, ranked by stars ([Source](https://github.com/langfuse/langfuse-docs/blob/main/components-mdx/dependents)):
 
 | Repository                                                                                                                                                                                                                                                          | Stars |
 | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----: |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 53464 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 49387 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37087 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 36690 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26871 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22348 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 14508 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7319 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5352 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2084 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  1936 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1140 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1044 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   945 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   770 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   675 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   583 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   505 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   383 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   379 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   357 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/127165244?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langgenius](https://github.com/langgenius) / [dify](https://github.com/langgenius/dify)                                            | 54865 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [open-webui](https://github.com/open-webui/open-webui)                                | 51531 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/131470832?s=40&v=4" width="20" height="20" alt=""> &nbsp; [lobehub](https://github.com/lobehub) / [lobe-chat](https://github.com/lobehub/lobe-chat)                                           | 49003 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85702467?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langflow-ai](https://github.com/langflow-ai) / [langflow](https://github.com/langflow-ai/langflow)                                  | 39093 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/130722866?s=40&v=4" width="20" height="20" alt=""> &nbsp; [run-llama](https://github.com/run-llama) / [llama_index](https://github.com/run-llama/llama_index)                                 | 37368 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/139558948?s=40&v=4" width="20" height="20" alt=""> &nbsp; [chatchat-space](https://github.com/chatchat-space) / [Langchain-Chatchat](https://github.com/chatchat-space/Langchain-Chatchat)    | 32486 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/128289781?s=40&v=4" width="20" height="20" alt=""> &nbsp; [FlowiseAI](https://github.com/FlowiseAI) / [Flowise](https://github.com/FlowiseAI/Flowise)                                         | 32448 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/31035808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mindsdb](https://github.com/mindsdb) / [mindsdb](https://github.com/mindsdb/mindsdb)                                                | 26931 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/119600397?s=40&v=4" width="20" height="20" alt=""> &nbsp; [twentyhq](https://github.com/twentyhq) / [twenty](https://github.com/twentyhq/twenty)                                              | 24195 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog](https://github.com/PostHog/posthog)                                                | 22618 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/121462774?s=40&v=4" width="20" height="20" alt=""> &nbsp; [BerriAI](https://github.com/BerriAI) / [litellm](https://github.com/BerriAI/litellm)                                               | 15151 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/179202840?s=40&v=4" width="20" height="20" alt=""> &nbsp; [mediar-ai](https://github.com/mediar-ai) / [screenpipe](https://github.com/mediar-ai/screenpipe)                                   | 11037 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/105877416?s=40&v=4" width="20" height="20" alt=""> &nbsp; [formbricks](https://github.com/formbricks) / [formbricks](https://github.com/formbricks/formbricks)                                |  9386 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/76263028?s=40&v=4" width="20" height="20" alt=""> &nbsp; [anthropics](https://github.com/anthropics) / [courses](https://github.com/anthropics/courses)                                       |  8385 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/78410652?s=40&v=4" width="20" height="20" alt=""> &nbsp; [GreyDGL](https://github.com/GreyDGL) / [PentestGPT](https://github.com/GreyDGL/PentestGPT)                                          |  7374 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/152537519?s=40&v=4" width="20" height="20" alt=""> &nbsp; [superagent-ai](https://github.com/superagent-ai) / [superagent](https://github.com/superagent-ai/superagent)                       |  5391 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/137907881?s=40&v=4" width="20" height="20" alt=""> &nbsp; [promptfoo](https://github.com/promptfoo) / [promptfoo](https://github.com/promptfoo/promptfoo)                                     |  4976 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/157326433?s=40&v=4" width="20" height="20" alt=""> &nbsp; [onlook-dev](https://github.com/onlook-dev) / [onlook](https://github.com/onlook-dev/onlook)                                        |  4141 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/7250217?s=40&v=4" width="20" height="20" alt=""> &nbsp; [Canner](https://github.com/Canner) / [WrenAI](https://github.com/Canner/WrenAI)                                                      |  2526 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/11855343?s=40&v=4" width="20" height="20" alt=""> &nbsp; [pingcap](https://github.com/pingcap) / [autoflow](https://github.com/pingcap/autoflow)                                              |  2061 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/85268109?s=40&v=4" width="20" height="20" alt=""> &nbsp; [MLSysOps](https://github.com/MLSysOps) / [MLE-agent](https://github.com/MLSysOps/MLE-agent)                                         |  1161 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/158137808?s=40&v=4" width="20" height="20" alt=""> &nbsp; [open-webui](https://github.com/open-webui) / [pipelines](https://github.com/open-webui/pipelines)                                  |  1100 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/18422723?s=40&v=4" width="20" height="20" alt=""> &nbsp; [alishobeiri](https://github.com/alishobeiri) / [thread](https://github.com/alishobeiri/thread)                                      |  1074 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/125468716?s=40&v=4" width="20" height="20" alt=""> &nbsp; [topoteretes](https://github.com/topoteretes) / [cognee](https://github.com/topoteretes/cognee)                                     |   971 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/188657705?s=40&v=4" width="20" height="20" alt=""> &nbsp; [bRAGAI](https://github.com/bRAGAI) / [bRAG-langchain](https://github.com/bRAGAI/bRAG-langchain)                                    |   823 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169500408?s=40&v=4" width="20" height="20" alt=""> &nbsp; [opslane](https://github.com/opslane) / [opslane](https://github.com/opslane/opslane)                                               |   677 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/151867818?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dynamiq-ai](https://github.com/dynamiq-ai) / [dynamiq](https://github.com/dynamiq-ai/dynamiq)                                      |   639 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/48585267?s=40&v=4" width="20" height="20" alt=""> &nbsp; [theopenconversationkit](https://github.com/theopenconversationkit) / [tock](https://github.com/theopenconversationkit/tock)         |   514 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/20493493?s=40&v=4" width="20" height="20" alt=""> &nbsp; [andysingal](https://github.com/andysingal) / [llm-course](https://github.com/andysingal/llm-course)                                 |   394 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/132396805?s=40&v=4" width="20" height="20" alt=""> &nbsp; [phospho-app](https://github.com/phospho-app) / [phospho](https://github.com/phospho-app/phospho)                                   |   384 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/178644984?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sentient-engineering](https://github.com/sentient-engineering) / [agent-q](https://github.com/sentient-engineering/agent-q)        |   370 |
 | <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/168552753?s=40&v=4" width="20" height="20" alt=""> &nbsp; [sql-agi](https://github.com/sql-agi) / [DB-GPT](https://github.com/sql-agi/DB-GPT)                                                 |   324 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   300 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   278 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   237 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   164 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   133 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   122 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/134601687?s=40&v=4" width="20" height="20" alt=""> &nbsp; [langfuse](https://github.com/langfuse) / [langfuse-python](https://github.com/langfuse/langfuse-python)                            |   108 |
-| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/42655961?s=40&v=4" width="20" height="20" alt=""> &nbsp; [zozoheir](https://github.com/zozoheir) / [tinyllm](https://github.com/zozoheir/tinyllm)                                             |    95 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/60330232?s=40&v=4" width="20" height="20" alt=""> &nbsp; [PostHog](https://github.com/PostHog) / [posthog-foss](https://github.com/PostHog/posthog-foss)                                      |   305 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/154247157?s=40&v=4" width="20" height="20" alt=""> &nbsp; [vespperhq](https://github.com/vespperhq) / [vespper](https://github.com/vespperhq/vespper)                                         |   304 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/185116535?s=40&v=4" width="20" height="20" alt=""> &nbsp; [block](https://github.com/block) / [goose](https://github.com/block/goose)                                                         |   295 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/609489?s=40&v=4" width="20" height="20" alt=""> &nbsp; [aorwall](https://github.com/aorwall) / [moatless-tools](https://github.com/aorwall/moatless-tools)                                    |   291 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/2357342?s=40&v=4" width="20" height="20" alt=""> &nbsp; [dmayboroda](https://github.com/dmayboroda) / [minima](https://github.com/dmayboroda/minima)                                          |   221 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/66303003?s=40&v=4" width="20" height="20" alt=""> &nbsp; [RobotecAI](https://github.com/RobotecAI) / [rai](https://github.com/RobotecAI/rai)                                                  |   172 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/148684274?s=40&v=4" width="20" height="20" alt=""> &nbsp; [i-am-alice](https://github.com/i-am-alice) / [3rd-devs](https://github.com/i-am-alice/3rd-devs)                                    |   148 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/171735272?s=40&v=4" width="20" height="20" alt=""> &nbsp; [8090-inc](https://github.com/8090-inc) / [xrx-sample-apps](https://github.com/8090-inc/xrx-sample-apps)                            |   138 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/104478511?s=40&v=4" width="20" height="20" alt=""> &nbsp; [babelcloud](https://github.com/babelcloud) / [LLM-RGB](https://github.com/babelcloud/LLM-RGB)                                      |   135 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/15125613?s=40&v=4" width="20" height="20" alt=""> &nbsp; [souzatharsis](https://github.com/souzatharsis) / [tamingLLMs](https://github.com/souzatharsis/tamingLLMs)                           |   129 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/169401942?s=40&v=4" width="20" height="20" alt=""> &nbsp; [LibreChat-AI](https://github.com/LibreChat-AI) / [librechat.ai](https://github.com/LibreChat-AI/librechat.ai)                      |   128 |
+| <img class="avatar mr-2" src="https://avatars.githubusercontent.com/u/51827949?s=40&v=4" width="20" height="20" alt=""> &nbsp; [deepset-ai](https://github.com/deepset-ai) / [haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) |   126 |

careers

@@ -1,13 +1,19 @@
 _We are Hiring_
-Join us in scaling Langfuse in Berlin, Germany. We are an open source company, we hire in person, we are only hiring technical talent.
+Join us in building out Langfuse in Berlin, Germany. Langfuse is the open source LLM engineering platform: we build tooling to help developers [build & improve LLM applications](https://langfuse.com/docs). 
+
+We are an open source company, we hire in person (4+ days a week), we only hire excellent technical talent. Find more information on our [careers page](https://langfuse.com/careers)
 
 _Open Roles_
 
-- Product Engineer, 70-130k EUR, 0.25-0.75% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
-- Developer Advocate, 60-110k EUR, 0.25-0.5% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+- Product Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/aAvmoFB-product-engineer
+- Backend Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/1bO16H6-backend-engineer
+- Design Engineer, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/mDquP95-design-engineer
+- Developer Advocate, 70-130k EUR, 0.1-0.35% Equity, https://www.ycombinator.com/companies/langfuse/jobs/uHysbKH-developer-advocate-devrel
+
 
 _More Info_
 
 - https://langfuse.com/careers
 - https://langfuse.com/docs
+- https://langfuse.com/why
 - https://langfuse.com/changelog

docker-compose.build.yml

@@ -155,4 +155,3 @@ volumes:
     driver: local
   langfuse_minio_data:
     driver: local
-

docker-compose.yml

@@ -38,6 +38,8 @@ services:
       LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: ${LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT:-http://minio:9000}
       LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: ${LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE:-true}
       LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: ${LANGFUSE_S3_MEDIA_UPLOAD_PREFIX:-media/}
+      LANGFUSE_INGESTION_QUEUE_DELAY_MS: ${LANGFUSE_INGESTION_QUEUE_DELAY_MS:-}
+      LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS: ${LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS:-}
       REDIS_HOST: ${REDIS_HOST:-redis}
       REDIS_PORT: ${REDIS_PORT:-6379}
       REDIS_AUTH: ${REDIS_AUTH:-myredissecret}
@@ -61,10 +63,6 @@ services:
       LANGFUSE_INIT_USER_EMAIL: ${LANGFUSE_INIT_USER_EMAIL:-}
       LANGFUSE_INIT_USER_NAME: ${LANGFUSE_INIT_USER_NAME:-}
       LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
-      LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: ${LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED:-false}
-      LANGFUSE_READ_FROM_POSTGRES_ONLY: ${LANGFUSE_READ_FROM_POSTGRES_ONLY:-false}
-      LANGFUSE_READ_FROM_CLICKHOUSE_ONLY: ${LANGFUSE_READ_FROM_CLICKHOUSE_ONLY:-true}
-      LANGFUSE_RETURN_FROM_CLICKHOUSE: ${LANGFUSE_RETURN_FROM_CLICKHOUSE:-true}
 
   clickhouse:
     image: clickhouse/clickhouse-server

ee/package.json

@@ -27,7 +27,8 @@
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
     "axios": "^1.7.7",
-    "next": "^14.2.15",
+    "https-proxy-agent": "^7.0.6",
+    "next": "^14.2.21",
     "next-auth": "^4.24.11",
     "zod": "^3.23.8"
   },
@@ -47,7 +48,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8"
     }
   }
 }

fern/apis/client/generators.yml

@@ -12,7 +12,7 @@ groups:
       #     namespaceExport: Langfuse
       #     allowCustomFetcher: true
       - name: fernapi/fern-openapi
-        version: 0.0.26
+        version: 0.1.7
         output:
           location: local-file-system
           path: ../../../web/public/generated/api-client

fern/apis/server/definition/commons.yml

@@ -378,10 +378,10 @@ types:
         type: string
       startDate:
         docs: Apply only to generations which are newer than this ISO date.
-        type: optional<date>
+        type: optional<datetime>
       unit:
         docs: Unit used by this model.
-        type: ModelUsageUnit
+        type: optional<ModelUsageUnit>
       inputPrice:
         docs: Price (USD) per input unit
         type: optional<double>

fern/apis/server/definition/ingestion.yml

@@ -11,6 +11,7 @@ service:
         Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.
 
         Notes:
+        - Introduction to data model: https://langfuse.com/docs/tracing-data-model
         - Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.
         - The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.
       method: POST
@@ -127,7 +128,7 @@ types:
       modelParameters: optional<map<string, commons.MapValue>>
       usage: optional<IngestionUsage>
       usageDetails: optional<UsageDetails>
-      costDetails: optional<map<string, float>>
+      costDetails: optional<map<string, double>>
       promptName: optional<string>
       promptVersion: optional<integer>
 
@@ -140,7 +141,7 @@ types:
       usage: optional<IngestionUsage>
       promptName: optional<string>
       usageDetails: optional<UsageDetails>
-      costDetails: optional<map<string, float>>
+      costDetails: optional<map<string, double>>
       promptVersion: optional<integer>
 
   ObservationBody:

fern/apis/server/definition/models.yml

@@ -58,7 +58,7 @@ types:
         type: optional<datetime>
       unit:
         docs: Unit used by this model.
-        type: commons.ModelUsageUnit
+        type: optional<commons.ModelUsageUnit>
       inputPrice:
         docs: Price (USD) per input unit
         type: optional<double>

fern/apis/server/definition/prompt-version.yml

@@ -0,0 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  prompts: ./prompts.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public/v2
+  endpoints:
+    update:
+      docs: Update labels for a specific prompt version
+      method: PATCH
+      path: /prompts/{name}/versions/{version}
+      path-parameters:
+        name:
+          type: string
+          docs: The name of the prompt
+        version:
+          type: integer
+          docs: Version of the prompt to update
+      request:
+        name: UpdatePromptRequest
+        body:
+          properties:
+            newLabels:
+              type: list<string>
+              docs: New labels for the prompt version. Labels are unique across versions. The "latest" label is reserved and managed by Langfuse.
+      response: prompts.Prompt

fern/apis/server/definition/prompts.yml

@@ -90,6 +90,9 @@ types:
       tags:
         type: optional<list<string>>
         docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   CreateTextPromptRequest:
     properties:
@@ -102,6 +105,9 @@ types:
       tags:
         type: optional<list<string>>
         docs: List of tags to apply to all versions of this prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   Prompt:
     union:
@@ -119,6 +125,9 @@ types:
       tags:
         type: list<string>
         docs: List of tags. Used to filter via UI and API. The same across versions of a prompt.
+      commitMessage:
+        type: optional<string>
+        docs: Commit message for this prompt version.
 
   ChatMessage:
     properties:

fern/apis/server/definition/score.yml

@@ -59,7 +59,7 @@ service:
             type: optional<commons.ScoreDataType>
             docs: Retrieve only scores with a specific dataType.
           traceTags:
-            type: optional<list<string>>
+            type: optional<string>
             allow-multiple: true
             docs: Only scores linked to traces that include all of these tags will be returned.
       response: GetScoresResponse

fern/apis/server/generators.yml

@@ -3,7 +3,7 @@ groups:
   local:
     generators:
       - name: fernapi/fern-openapi
-        version: 0.0.31
+        version: 0.1.7
         output:
           location: local-file-system
           path: ../../../web/public/generated/api

fern/fern.config.json

@@ -1,4 +1,5 @@
 {
-  "organization": "finto",
+  "organization": "langfuse",
   "version": "0.43.7"
-}
+}
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.2.0",
+  "version": "3.26.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -25,7 +25,6 @@
     "dev": "turbo run dev",
     "lint": "turbo run lint",
     "test": "turbo run test",
-    "models:migrate": "turbo run models:migrate",
     "release": "dotenv -e ../.env -- release-it",
     "prepare": "husky"
   },
@@ -84,7 +83,12 @@
   "packageManager": "pnpm@9.5.0",
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "^0.16.21"
+    },
+    "patchedDependencies": {
+      "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"
     }
   }
 }

packages/shared/clickhouse/scripts/down.sh

@@ -18,21 +18,31 @@ then
     exit 1
 fi
 
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" down
 else
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
   fi
 
   # Execute the up command

packages/shared/clickhouse/scripts/drop.sh

@@ -12,11 +12,16 @@ then
     exit 1
 fi
 
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
 else
-    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+    DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
 fi
 # Execute the drop command
 migrate -source file://clickhouse/migrations -database "$DATABASE_URL" drop

packages/shared/clickhouse/scripts/up.sh

@@ -18,21 +18,31 @@ then
     exit 1
 fi
 
+# Ensure CLICKHOUSE_DB is set
+if [ -z "${CLICKHOUSE_DB}" ]; then
+    export CLICKHOUSE_DB="default"
+fi
+
+# Ensure CLICKHOUSE_CLUSTER_NAME is set
+if [ -z "${CLICKHOUSE_CLUSTER_NAME}" ]; then
+    export CLICKHOUSE_CLUSTER_NAME="default"
+fi
+
 # Construct the database URL
 if [ "$CLICKHOUSE_CLUSTER_ENABLED" == "false" ] ; then
   if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-migrations-table-engine=MergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-migrations-table-engine=MergeTree"
   fi
 
   # Execute the up command
   migrate -source file://clickhouse/migrations/unclustered -database "$DATABASE_URL" up
 else
 if [ "$CLICKHOUSE_MIGRATION_SSL" = true ] ; then
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&secure=true&skip_verify=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
   else
-      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=default&x-multi-statement=true&x-cluster-name=default&x-migrations-table-engine=ReplicatedMergeTree"
+      DATABASE_URL="${CLICKHOUSE_MIGRATION_URL}?username=${CLICKHOUSE_USER}&password=${CLICKHOUSE_PASSWORD}&database=${CLICKHOUSE_DB}&x-multi-statement=true&x-cluster-name=${CLICKHOUSE_CLUSTER_NAME}&x-migrations-table-engine=ReplicatedMergeTree"
   fi
 
   # Execute the up command

packages/shared/package.json

@@ -33,7 +33,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "lint": "eslint . --ext .js,.jsx,.ts,.tsx --max-warnings 110",
     "lint:fix": "eslint . --ext .js,.jsx,.ts,.tsx --fix",
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
@@ -61,25 +61,26 @@
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
     "@clickhouse/client": "^1.4.0",
-    "@langchain/anthropic": "^0.3.8",
-    "@langchain/aws": "^0.1.2",
-    "@langchain/core": "^0.3.18",
-    "@langchain/google-vertexai": "^0.1.3",
-    "@langchain/openai": "^0.3.14",
+    "@langchain/anthropic": "^0.3.12",
+    "@langchain/aws": "^0.1.3",
+    "@langchain/core": "^0.3.37",
+    "@langchain/google-vertexai": "^0.1.8",
+    "@langchain/openai": "^0.3.17",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "@prisma/client": "^5.22.0",
+    "@prisma/client": "^6.3.0",
     "@react-email/components": "^0.0.19",
     "@react-email/render": "^0.0.15",
     "@types/bcryptjs": "^2.4.6",
     "axios": "^1.7.7",
     "bcryptjs": "^2.4.3",
-    "bullmq": "^5.12.10",
+    "bullmq": "^5.34.10",
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
     "exponential-backoff": "^3.1.1",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "kysely": "^0.27.4",
-    "langchain": "^0.3.6",
+    "langchain": "^0.3.15",
     "langfuse-langchain": "3.30.3",
     "lodash": "^4.17.21",
     "next-auth": "^4.24.11",
@@ -106,7 +107,7 @@
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
     "prettier": "^3.3.3",
-    "prisma": "^5.22.0",
+    "prisma": "^6.3.0",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
@@ -121,7 +122,8 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8"
     }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -170,6 +170,17 @@ export type BatchExport = {
     url: string | null;
     log: string | null;
 };
+export type BillingMeterBackup = {
+    stripe_customer_id: string;
+    meter_id: string;
+    start_time: Timestamp;
+    end_time: Timestamp;
+    aggregated_value: number;
+    event_name: string;
+    org_id: string;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Comment = {
     id: string;
     project_id: string;
@@ -242,6 +253,18 @@ export type EvalTemplate = {
     vars: Generated<string[]>;
     output_schema: unknown;
 };
+export type EventLog = {
+    id: string;
+    bucket_name: string;
+    bucket_path: string;
+    project_id: string;
+    entity_type: string;
+    entity_id: string;
+    event_id: string;
+    trace_id: string | null;
+    created_at: Generated<Timestamp>;
+    updated_at: Generated<Timestamp>;
+};
 export type Events = {
     id: string;
     created_at: Generated<Timestamp>;
@@ -293,6 +316,8 @@ export type LlmApiKeys = {
     base_url: string | null;
     custom_models: Generated<string[]>;
     with_default_models: Generated<boolean>;
+    extra_headers: string | null;
+    extra_header_keys: Generated<string[]>;
     config: unknown | null;
     project_id: string;
 };
@@ -455,6 +480,7 @@ export type Project = {
     updated_at: Generated<Timestamp>;
     deleted_at: Timestamp | null;
     name: string;
+    retention_days: number | null;
 };
 export type ProjectMembership = {
     org_membership_id: string;
@@ -478,6 +504,14 @@ export type Prompt = {
     config: Generated<unknown>;
     tags: Generated<string[]>;
     labels: Generated<string[]>;
+    commit_message: string | null;
+};
+export type QueueBackUp = {
+    id: string;
+    project_id: string | null;
+    queue_name: string;
+    content: unknown;
+    created_at: Generated<Timestamp>;
 };
 export type Score = {
     id: string;
@@ -604,6 +638,7 @@ export type DB = {
     audit_logs: AuditLog;
     background_migrations: BackgroundMigration;
     batch_exports: BatchExport;
+    billing_meter_backups: BillingMeterBackup;
     comments: Comment;
     cron_jobs: CronJobs;
     dataset_items: DatasetItem;
@@ -611,6 +646,7 @@ export type DB = {
     dataset_runs: DatasetRuns;
     datasets: Dataset;
     eval_templates: EvalTemplate;
+    event_log: EventLog;
     events: Events;
     job_configurations: JobConfiguration;
     job_executions: JobExecution;
@@ -628,6 +664,7 @@ export type DB = {
     project_memberships: ProjectMembership;
     projects: Project;
     prompts: Prompt;
+    queue_backups: QueueBackUp;
     score_configs: ScoreConfig;
     scores: Score;
     Session: Session;

packages/shared/prisma/migrations/20250108220721_add_queue_backup_table/migration.sql

@@ -0,0 +1,13 @@
+-- CreateTable
+CREATE TABLE "queue_backups" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT,
+    "queue_name" TEXT NOT NULL,
+    "content" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "queue_backups_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "traces" ADD CONSTRAINT "traces_session_id_project_id_fkey" FOREIGN KEY ("session_id", "project_id") REFERENCES "trace_sessions"("id", "project_id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250109083346_drop_trace_tracesession_fk/migration.sql

@@ -0,0 +1,2 @@
+-- DropForeignKey
+ALTER TABLE "traces" DROP CONSTRAINT "traces_session_id_project_id_fkey";

packages/shared/prisma/migrations/20250116154613_add_billing_meter_backups/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "billing_meter_backups" (
+    "stripe_customer_id" TEXT NOT NULL,
+    "meter_id" TEXT NOT NULL,
+    "start_time" TIMESTAMP(3) NOT NULL,
+    "end_time" TIMESTAMP(3) NOT NULL,
+    
+    "aggregated_value" INTEGER NOT NULL,
+    
+    "event_name" TEXT NOT NULL,
+    "org_id" TEXT NOT NULL,
+    
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- CreateIndex
+CREATE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_idx" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "billing_meter_backups_stripe_customer_id_meter_id_start_tim_key" ON "billing_meter_backups"("stripe_customer_id", "meter_id", "start_time", "end_time");

packages/shared/prisma/migrations/20250122152102_add_llm_api_keys_extra_headers/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "llm_api_keys"
+    ADD COLUMN "extra_headers" TEXT,
+    ADD COLUMN "extra_header_keys" TEXT[] NOT NULL DEFAULT '{}'::TEXT[];

packages/shared/prisma/migrations/20250123103200_add_retention_days_to_projects/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "projects"
+    ADD COLUMN "retention_days" INTEGER;

packages/shared/prisma/migrations/20250128144418_llm_adapter_rename_google_vertex_ai/migration.sql

@@ -0,0 +1 @@
+UPDATE "llm_api_keys" SET adapter = 'google-vertex-ai' WHERE adapter = 'vertex-ai';

packages/shared/prisma/migrations/20250128163035_add_nullable_commit_message_prompts/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "prompts" ADD COLUMN     "commit_message" TEXT;

packages/shared/prisma/migrations/20250204180200_add_event_log_table/migration.sql

@@ -0,0 +1,20 @@
+-- CreateTable
+CREATE TABLE "event_log" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "bucket_name" TEXT NOT NULL,
+    "bucket_path" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "entity_type" TEXT NOT NULL,
+    "entity_id" TEXT NOT NULL,
+    "event_id" TEXT NOT NULL,
+
+    "trace_id" TEXT,
+
+    CONSTRAINT "event_log_pkey" PRIMARY KEY ("id"),
+    CONSTRAINT "event_log_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE INDEX "event_log_project_id_entity_type_entity_id_idx" ON "event_log"("project_id", "entity_type", "entity_id");

packages/shared/prisma/schema.prisma

@@ -115,6 +115,7 @@ model Project {
   updatedAt           DateTime               @default(now()) @updatedAt @map("updated_at")
   deletedAt           DateTime?              @map("deleted_at")
   name                String
+  retentionDays       Int?                   @map("retention_days")
   projectMembers      ProjectMembership[]
   organization        Organization           @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
   traces              Trace[]
@@ -140,6 +141,7 @@ model Project {
   TraceMedia          TraceMedia[]
   Media               Media[]
   ObservationMedia    ObservationMedia[]
+  EventLog            EventLog[]
 
   @@index([orgId])
   @@map("projects")
@@ -192,6 +194,8 @@ model LlmApiKeys {
   baseURL           String?  @map("base_url")
   customModels      String[] @default([]) @map("custom_models")
   withDefaultModels Boolean  @default(true) @map("with_default_models")
+  extraHeaders      String?  @map("extra_headers")
+  extraHeaderKeys   String[] @default([]) @map("extra_header_keys")
   config            Json?
 
   projectId String  @map("project_id")
@@ -272,7 +276,6 @@ model TraceSession {
   project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
   bookmarked Boolean  @default(false)
   public     Boolean  @default(false)
-  traces     Trace[]
 
   @@id([id, projectId])
   @@index([projectId])
@@ -284,25 +287,24 @@ model TraceSession {
 // Update TraceView below when making changes to this model!
 
 model Trace {
-  id         String        @id @default(cuid())
-  externalId String?       @map("external_id")
-  timestamp  DateTime      @default(now())
+  id         String   @id @default(cuid())
+  externalId String?  @map("external_id")
+  timestamp  DateTime @default(now())
   name       String?
-  userId     String?       @map("user_id")
+  userId     String?  @map("user_id")
   metadata   Json?
   release    String?
   version    String?
-  projectId  String        @map("project_id")
-  project    Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  public     Boolean       @default(false)
-  bookmarked Boolean       @default(false)
-  tags       String[]      @default([])
+  projectId  String   @map("project_id")
+  project    Project  @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  public     Boolean  @default(false)
+  bookmarked Boolean  @default(false)
+  tags       String[] @default([])
   input      Json?
   output     Json?
-  sessionId  String?       @map("session_id")
-  session    TraceSession? @relation(fields: [sessionId, projectId], references: [id, projectId])
-  createdAt  DateTime      @default(now()) @map("created_at")
-  updatedAt  DateTime      @default(now()) @updatedAt @map("updated_at")
+  sessionId  String?  @map("session_id")
+  createdAt  DateTime @default(now()) @map("created_at")
+  updatedAt  DateTime @default(now()) @updatedAt @map("updated_at")
 
   @@index([projectId, timestamp])
   @@index([sessionId])
@@ -705,6 +707,29 @@ model Events {
   @@map("events")
 }
 
+model EventLog {
+  id String @id @default(cuid())
+
+  bucketName String  @map("bucket_name")
+  bucketPath String  @map("bucket_path")
+  projectId  String  @map("project_id")
+  project    Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  entityType String  @map("entity_type")
+  entityId   String  @map("entity_id")
+  eventId    String  @map("event_id")
+
+  // traceId is only optional, because we cannot guarantee its presence for observations.
+  // We use it to support deletions on traces as we cannot cascade it down to scores and observations otherwise.
+  // We expect a traceId to be present in most cases.
+  traceId String? @map("trace_id")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, entityType, entityId])
+  @@map("event_log")
+}
+
 model Comment {
   id           String            @id @default(cuid())
   projectId    String            @map("project_id")
@@ -737,14 +762,15 @@ model Prompt {
 
   createdBy String @map("created_by")
 
-  prompt   Json
-  name     String
-  version  Int
-  type     String   @default("text")
-  isActive Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
-  config   Json     @default("{}") @db.Json
-  tags     String[] @default([])
-  labels   String[] @default([])
+  prompt        Json
+  name          String
+  version       Int
+  type          String   @default("text")
+  isActive      Boolean? @map("is_active") // Deprecated. To be removed once 'production' labels work as expected.
+  config        Json     @default("{}") @db.Json
+  tags          String[] @default([])
+  labels        String[] @default([])
+  commitMessage String?  @map("commit_message")
 
   @@unique([projectId, name, version])
   @@index([projectId, id])
@@ -1017,3 +1043,37 @@ model ObservationMedia {
   @@index([projectId, observationId])
   @@map("observation_media")
 }
+
+model QueueBackUp {
+  id        String  @id @default(cuid())
+  projectId String? @map("project_id")
+  queueName String  @map("queue_name")
+  content   Json
+
+  createdAt DateTime @default(now()) @map("created_at")
+
+  @@map("queue_backups")
+}
+
+model BillingMeterBackup {
+  // unique
+  stripeCustomerId String   @map("stripe_customer_id")
+  meterId          String   @map("meter_id")
+  startTime        DateTime @map("start_time")
+  endTime          DateTime @map("end_time")
+
+  // value
+  aggregatedValue Int @map("aggregated_value")
+
+  // labels
+  eventName String @map("event_name")
+  orgId     String @map("org_id")
+
+  // ts
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([stripeCustomerId, meterId, startTime, endTime])
+  @@index([stripeCustomerId, meterId, startTime, endTime])
+  @@map("billing_meter_backups")
+}

packages/shared/scripts/prepareClickhouse.ts

@@ -97,19 +97,31 @@ export const prepareClickhouse = async (
       repeat('input', toInt64(randExponential(1 / 100))) AS input,
       repeat('output', toInt64(randExponential(1 / 100))) AS output,
       case
-        when number % 2 = 0 then 'claude-3-haiku-20240307'
+        when number % 2 = 0 then 'clause-3-haiku-20230407'
         else 'gpt-4'
       end as provided_model_name,
       case
-        when number % 2 = 0 then 'cltr0w45b000008k1407o9qv1'
-        else 'clrntkjgy000f08jx79v9g1xj'
+        when number % 2 = 0 then 'cltra4wbs0000k1407g0ya3'
+        else '1cmtk9y0000y3y79x9jgxj'
       end as internal_model_id,
-      '{"temperature": 0.7, "max_tokens": 150}' AS model_parameters,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS provided_usage_details,
-      map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))) AS usage_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS provided_cost_details,
-      map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)) AS cost_details,
-      toDecimal64(randUniform(0, 2000), 12) AS total_cost,
+      if("type" = 'GENERATION',
+        '{"temperature": 0.7, "max_tokens": 150}',
+        '{}') AS model_parameters,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS provided_usage_details,
+      if("type" = 'GENERATION',
+        map('input', toUInt64(randUniform(0, 1000)), 'output', toUInt64(randUniform(0, 1000)), 'total', toUInt64(randUniform(0, 2000))),
+        map()) AS usage_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS provided_cost_details,
+      if("type" = 'GENERATION',
+        map('input', toDecimal64(randUniform(0, 1000), 12), 'output', toDecimal64(randUniform(0, 1000), 12), 'total', toDecimal64(randUniform(0, 2000), 12)),
+        map()) AS cost_details,
+      if("type" = 'GENERATION',
+        toDecimal64(randUniform(0, 2000), 12),
+        NULL) AS total_cost,
       addMilliseconds(start_time, if(rand() < 0.6, floor(randUniform(0, 500)), floor(randUniform(0, 600)))) AS completion_start_time,
       array(${SEED_PROMPTS.map((p) => `concat('${p.id}',project_id)`).join(
         ",",

packages/shared/src/env.ts

@@ -29,17 +29,10 @@ const EnvSchema = z.object({
   LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
   LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
   CLICKHOUSE_URL: z.string().url(),
+  CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
+  CLICKHOUSE_DB: z.string().default("default"),
   CLICKHOUSE_USER: z.string(),
   CLICKHOUSE_PASSWORD: z.string(),
-  LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED: z
-    .enum(["true", "false"])
-    .default("false"),
-  LANGFUSE_CLICKHOUSE_INGESTION_ENABLED: z
-    .enum(["true", "false"])
-    .default("true"),
-  LANGFUSE_POSTGRES_INGESTION_ENABLED: z
-    .enum(["true", "false"])
-    .default("false"),
 
   LANGFUSE_INGESTION_QUEUE_DELAY_MS: z.coerce
     .number()
@@ -53,6 +46,9 @@ const EnvSchema = z.object({
   ENABLE_AWS_CLOUDWATCH_METRIC_PUBLISHING: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_S3_EVENT_UPLOAD_POSTGRES_LOG_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
   LANGFUSE_S3_EVENT_UPLOAD_BUCKET: z.string({
     required_error: "Langfuse requires a bucket name for S3 Event Uploads.",
   }),
@@ -66,6 +62,13 @@ const EnvSchema = z.object({
     .default("false"),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
   STRIPE_SECRET_KEY: z.string().optional(),
+
+  LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED: z
+    .enum(["true", "false"])
+    .default("false"),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/errors/ApiError.ts

@@ -1,7 +1,7 @@
 import { BaseError } from "./BaseError";
 
 export class ApiError extends BaseError {
-  constructor(description = "Api call failed") {
-    super("ApiError", 500, description, true);
+  constructor(description = "Api call failed", status = 500) {
+    super("ApiError", status, description, true);
   }
 }

packages/shared/src/features/batchAction/types.ts

@@ -0,0 +1,38 @@
+import z from "zod";
+import { singleFilter } from "../../interfaces/filters";
+import { orderBy } from "../../interfaces/orderBy";
+import { BatchExportTableName } from "../batchExport/types";
+
+/* eslint-disable no-unused-vars */
+export enum BatchActionType {
+  Create = "create",
+  Delete = "delete",
+}
+
+const ActionIdSchema = z.enum([
+  "trace-delete",
+  "trace-add-to-annotation-queue",
+]);
+
+export type ActionId = z.infer<typeof ActionIdSchema>;
+
+export const BatchActionQuerySchema = z.object({
+  filter: z.array(singleFilter).nullable(),
+  orderBy,
+});
+
+export type BatchActionQuery = z.infer<typeof BatchActionQuerySchema>;
+
+export const CreateBatchActionSchema = z.object({
+  projectId: z.string(),
+  actionId: ActionIdSchema,
+  targetId: z.string().optional(),
+  query: BatchActionQuerySchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+});
+
+export const GetIsBatchActionInProgressSchema = z.object({
+  projectId: z.string(),
+  actionId: ActionIdSchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+});

packages/shared/src/features/evals/types.ts

@@ -17,6 +17,7 @@ export const variableMapping = z
     objectName: z.string().nullish(),
     langfuseObject: z.enum(langfuseObjects),
     selectedColumnId: z.string(),
+    jsonSelector: z.string().nullish(),
   })
   .refine(
     (value) => value.langfuseObject === "trace" || value.objectName !== null,
@@ -32,6 +33,7 @@ export const wipVariableMapping = z.object({
   objectName: z.string().nullish(),
   langfuseObject: z.enum(langfuseObjects),
   selectedColumnId: z.string().nullish(),
+  jsonSelector: z.string().nullish(),
 });
 
 const observationCols = [

packages/shared/src/index.ts

@@ -21,7 +21,10 @@ export * from "./server/llm/types";
 
 // evals
 export * from "./features/evals/types";
+
+// table actions
 export * from "./features/batchExport/types";
+export * from "./features/batchAction/types";
 
 // annotation
 export * from "./features/annotation/types";

packages/shared/src/observationsTable.ts

@@ -20,6 +20,13 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "type",
+    id: "type",
+    type: "stringOptions",
+    options: [],
+    internal: 'o."type"',
+  },
   { name: "Trace ID", id: "traceId", type: "string", internal: 't."id"' },
   {
     name: "Trace Name",
@@ -112,6 +119,14 @@ export const observationsTableCols: ColumnDefinition[] = [
     options: [], // to be added at runtime
     nullable: true,
   },
+  {
+    name: "Model ID",
+    id: "modelId",
+    type: "stringOptions",
+    internal: 'o."internal_model_id"',
+    options: [], // to be added at runtime
+    nullable: true,
+  },
   {
     name: "Input Tokens",
     id: "inputTokens",
@@ -186,20 +201,25 @@ export const observationsTableCols: ColumnDefinition[] = [
 // allows for undefined options, to offer filters while options are still loading
 export type ObservationOptions = {
   model: Array<OptionsDefinition>;
+  modelId: Array<OptionsDefinition>;
   name: Array<OptionsDefinition>;
   traceName: Array<OptionsDefinition>;
   scores_avg: Array<string>;
   promptName: Array<OptionsDefinition>;
   tags: Array<OptionsDefinition>;
+  type: Array<OptionsDefinition>;
 };
 
 export function observationsTableColsWithOptions(
-  options?: ObservationOptions
+  options?: ObservationOptions,
 ): ColumnDefinition[] {
   return observationsTableCols.map((col) => {
     if (col.id === "model") {
       return { ...col, options: options?.model ?? [] };
     }
+    if (col.id === "modelId") {
+      return { ...col, options: options?.modelId ?? [] };
+    }
     if (col.id === "name") {
       return { ...col, options: options?.name ?? [] };
     }
@@ -215,6 +235,9 @@ export function observationsTableColsWithOptions(
     if (col.id === "tags") {
       return { ...col, options: options?.tags ?? [] };
     }
+    if (col.id === "type") {
+      return { ...col, options: options?.type ?? [] };
+    }
     return col;
   });
 }

packages/shared/src/server/auth/customSsoProvider.ts

@@ -17,7 +17,6 @@ export function CustomSSOProvider<P extends CustomSSOUser>(
     wellKnown: `${options.issuer}/.well-known/openid-configuration`,
     authorization: { params: { scope: "openid email profile" } }, // overridden by options.authorization to be able to set custom scopes, deep merged with this default
     checks: ["pkce", "state"],
-    idToken: true,
     profile(profile) {
       return {
         id: profile.sub,

packages/shared/src/server/auth/gitHubEnterpriseProvider.ts

@@ -2,64 +2,64 @@ import type { OAuthConfig, OAuthUserConfig } from "next-auth/providers/oauth";
 import type { GithubProfile, GithubEmail } from "next-auth/providers/github";
 
 export function GitHubEnterpriseProvider<P extends GithubProfile>(
-    options: OAuthUserConfig<P> & {
-        enterprise?: {
-            baseUrl?: string;
-        };
-    }
+  options: OAuthUserConfig<P> & {
+    enterprise?: {
+      baseUrl?: string;
+    };
+  },
 ): OAuthConfig<P> {
-    const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com"
-    const apiBaseUrl = options?.enterprise?.baseUrl
-        ? `${options?.enterprise?.baseUrl}/api/v3`
-        : "https://api.github.com"
+  const baseUrl = options?.enterprise?.baseUrl ?? "https://github.com";
+  const apiBaseUrl = options?.enterprise?.baseUrl
+    ? `${options?.enterprise?.baseUrl}/api/v3`
+    : "https://api.github.com";
 
-    return {
-        id: "github-enterprise",
-        name: "GitHub Enterprise",
-        type: "oauth",
-        authorization: {
-            url: `${baseUrl}/login/oauth/authorize`,
-            params: { scope: "read:user user:email" },
-        },
-        token: `${baseUrl}/login/oauth/access_token`,
-        userinfo: {
-            url: `${apiBaseUrl}/user`,
-            async request({ client, tokens }) {
-                const profile = await client.userinfo(tokens.access_token!)
+  return {
+    id: "github-enterprise",
+    name: "GitHub Enterprise",
+    type: "oauth",
+    authorization: {
+      url: `${baseUrl}/login/oauth/authorize`,
+      params: { scope: "read:user user:email" },
+    },
+    token: `${baseUrl}/login/oauth/access_token`,
+    userinfo: {
+      url: `${apiBaseUrl}/user`,
+      async request({ client, tokens }) {
+        const profile = await client.userinfo(tokens.access_token!);
 
-                if (!profile.email) {
-                    // If the user does not have a public email, get another via the GitHub API
-                    // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
-                    const res = await fetch(`${apiBaseUrl}/user/emails`, {
-                        headers: { Authorization: `token ${tokens.access_token}` },
-                    })
+        if (!profile.email) {
+          // If the user does not have a public email, get another via the GitHub API
+          // See https://docs.github.com/en/rest/users/emails#list-email-addresses-for-the-authenticated-user
+          const res = await fetch(`${apiBaseUrl}/user/emails`, {
+            headers: { Authorization: `token ${tokens.access_token}` },
+          });
 
-                    if (res.ok) {
-                        const emails: GithubEmail[] = await res.json()
-                        profile.email = (emails.find((e) => e.primary) ?? emails[0]).email
-                    }
-                }
+          if (res.ok) {
+            const emails = (await res.json()) as GithubEmail[];
+            profile.email = (emails.find((e) => e.primary) ?? emails[0]).email;
+          }
+        }
 
-                return profile
-            },
-        },
-        profile(profile) {
-            return {
-                id: profile.id.toString(),
-                name: profile.name ?? profile.login,
-                email: profile.email,
-                image: profile.avatar_url,
-            }
-        },
-        style: {
-            logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
-            logoDark:
-                "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
-            bg: "#fff",
-            bgDark: "#000",
-            text: "#000",
-            textDark: "#fff",
-        },
-        options,
-    }
-}
+        return profile;
+      },
+    },
+    profile(profile) {
+      return {
+        id: profile.id.toString(),
+        name: profile.name ?? profile.login,
+        email: profile.email,
+        image: profile.avatar_url,
+      };
+    },
+    style: {
+      logo: "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github.svg",
+      logoDark:
+        "https://raw.githubusercontent.com/nextauthjs/next-auth/main/packages/next-auth/provider-logos/github-dark.svg",
+      bg: "#fff",
+      bgDark: "#000",
+      text: "#000",
+      textDark: "#fff",
+    },
+    options,
+  };
+}

packages/shared/src/server/clickhouse/client.ts

@@ -10,7 +10,7 @@ export const clickhouseClient = (opts?: NodeClickHouseClientConfigOptions) =>
     url: env.CLICKHOUSE_URL,
     username: env.CLICKHOUSE_USER,
     password: env.CLICKHOUSE_PASSWORD,
-    database: "default",
+    database: env.CLICKHOUSE_DB,
     clickhouse_settings: {
       async_insert: 1,
       wait_for_async_insert: 1, // if disabled, we won't get errors from clickhouse

packages/shared/src/server/index.ts

@@ -8,9 +8,11 @@ export * from "./auth/apiKeys";
 export * from "./auth/customSsoProvider";
 export * from "./auth/gitHubEnterpriseProvider";
 export * from "./llm/fetchLLMCompletion";
+export * from "./llm/utils";
 export * from "./llm/types";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
+export * from "./utils/eventLog";
 export * from "./clickhouse/client";
 export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
@@ -28,15 +30,17 @@ export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
-export * from "./redis/legacyIngestion";
+export * from "./redis/batchActionQueue";
 export * from "./redis/ingestionQueue";
 export * from "./redis/postHogIntegrationQueue";
 export * from "./redis/postHogIntegrationProcessingQueue";
+export * from "./redis/dataRetentionQueue";
+export * from "./redis/dataRetentionProcessingQueue";
+export * from "./redis/coreDataS3ExportQueue";
+export * from "./redis/meteringDataPostgresExportQueue";
 export * from "./redis/experimentCreateQueue";
 export * from "./auth/types";
-export * from "./ingestion/legacy/index";
 export * from "./queues";
-export * from "./ingestion/legacy/EventProcessor";
 export * from "./orderByToPrisma";
 export * from "./filterToPrisma";
 export * from "./instrumentation";
@@ -44,6 +48,7 @@ export * from "./logger";
 export * from "./queries";
 export * from "./repositories";
 export * from "./redis/evalExecutionQueue";
+export * from "./services/sessions-ui-table-service";
 
 // test utils
 export * from "./test-utils";

packages/shared/src/server/ingestion/legacy/EventProcessor.ts

@@ -1,796 +0,0 @@
-import { v4 } from "uuid";
-import { type z } from "zod";
-import Decimal from "decimal.js";
-import { findModel } from "../modelMatch";
-import {
-  ObservationEvent,
-  eventTypes,
-  legacyObservationCreateEvent,
-  generationCreateEvent,
-  traceEvent,
-  scoreEvent,
-  sdkLogEvent,
-  ingestionEvent,
-} from "../types";
-import { validateAndInflateScore } from "../validateAndInflateScore";
-import { Trace, Observation, Score, Prisma, Model } from "@prisma/client";
-import { ForbiddenError, LangfuseNotFoundError } from "../../../errors";
-import { mergeJson } from "../../../utils/json";
-import { jsonSchema } from "../../../utils/zod";
-import { prisma } from "../../../db";
-import { LegacyIngestionAccessScope } from ".";
-import { logger } from "../../logger";
-import { env } from "../../../env";
-import { upsertTrace } from "../../repositories";
-import { convertDateToClickhouseDateTime } from "../../clickhouse/client";
-
-export interface EventProcessor {
-  auth(apiScope: LegacyIngestionAccessScope): void;
-
-  process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> | undefined;
-}
-
-export const getProcessorForEvent = (
-  event: z.infer<typeof ingestionEvent>,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-): EventProcessor => {
-  switch (event.type) {
-    case eventTypes.TRACE_CREATE:
-      return new TraceProcessor(event);
-    case eventTypes.OBSERVATION_CREATE:
-    case eventTypes.OBSERVATION_UPDATE:
-    case eventTypes.EVENT_CREATE:
-    case eventTypes.SPAN_CREATE:
-    case eventTypes.SPAN_UPDATE:
-    case eventTypes.GENERATION_CREATE:
-    case eventTypes.GENERATION_UPDATE:
-      return new ObservationProcessor(event, calculateTokenDelegate);
-    case eventTypes.SCORE_CREATE: {
-      return new ScoreProcessor(event);
-    }
-    case eventTypes.SDK_LOG:
-      return new SdkLogProcessor(event);
-  }
-};
-
-export class ObservationProcessor implements EventProcessor {
-  event: ObservationEvent;
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined;
-
-  constructor(
-    event: ObservationEvent,
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-  ) {
-    this.event = event;
-    this.calculateTokenDelegate = calculateTokenDelegate;
-  }
-
-  async convertToObservation(
-    apiScope: LegacyIngestionAccessScope,
-    existingObservation: Omit<Observation, "input" | "output"> | null,
-  ): Promise<{
-    id: string;
-    create: Prisma.ObservationUncheckedCreateInput;
-    update: Prisma.ObservationUncheckedUpdateInput;
-  }> {
-    let type: "EVENT" | "SPAN" | "GENERATION";
-    switch (this.event.type) {
-      case eventTypes.OBSERVATION_CREATE:
-      case eventTypes.OBSERVATION_UPDATE:
-        type = this.event.body.type;
-        break;
-      case eventTypes.EVENT_CREATE:
-        type = "EVENT" as const;
-        break;
-      case eventTypes.SPAN_CREATE:
-      case eventTypes.SPAN_UPDATE:
-        type = "SPAN" as const;
-        break;
-      case eventTypes.GENERATION_CREATE:
-      case eventTypes.GENERATION_UPDATE:
-        type = "GENERATION" as const;
-        break;
-    }
-
-    if (
-      this.event.type === eventTypes.OBSERVATION_UPDATE &&
-      !existingObservation
-    ) {
-      throw new LangfuseNotFoundError(
-        `Observation with id ${this.event.id} not found`,
-      );
-    }
-
-    // find matching model definition based on event and existing observation in db
-    const internalModel: Model | undefined | null =
-      type === "GENERATION"
-        ? await findModel({
-            event: {
-              projectId: apiScope.projectId,
-              model:
-                "model" in this.event.body
-                  ? (this.event.body.model ?? undefined)
-                  : undefined,
-              unit:
-                "usage" in this.event.body
-                  ? (this.event.body.usage?.unit ?? undefined)
-                  : undefined,
-              startTime: this.event.body.startTime
-                ? new Date(this.event.body.startTime)
-                : undefined,
-            },
-            existingDbObservation: existingObservation ?? undefined,
-          })
-        : undefined;
-
-    // Token counts
-    const [newInputCount, newOutputCount] =
-      "usage" in this.event.body || "usageDetails" in this.event.body
-        ? await this.calculateTokenCounts(
-            apiScope.projectId,
-            this.event.body,
-            this.calculateTokenDelegate,
-            internalModel ?? undefined,
-            existingObservation ?? undefined,
-          )
-        : [undefined, undefined];
-
-    const newTotalCount =
-      "usage" in this.event.body || "usageDetails" in this.event.body
-        ? (this.event.body.usage?.total ??
-          this.event.body.usageDetails?.total ??
-          (newInputCount != null || newOutputCount != null
-            ? (newInputCount ?? 0) + (newOutputCount ?? 0)
-            : undefined))
-        : undefined;
-
-    const userProvidedTokenCosts = {
-      inputCost:
-        "usage" in this.event.body && this.event.body.usage?.inputCost != null // inputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.inputCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.input != null
-            ? new Decimal(this.event.body.costDetails?.input)
-            : existingObservation?.inputCost,
-      outputCost:
-        "usage" in this.event.body && this.event.body.usage?.outputCost != null // outputCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.outputCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.output != null
-            ? new Decimal(this.event.body.costDetails?.output)
-            : existingObservation?.outputCost,
-      totalCost:
-        "usage" in this.event.body && this.event.body.usage?.totalCost != null // totalCost can be explicitly 0. Note only one equal sign to capture null AND undefined
-          ? new Decimal(this.event.body.usage?.totalCost)
-          : "costDetails" in this.event.body &&
-              this.event.body.costDetails?.total != null
-            ? new Decimal(this.event.body.costDetails?.total)
-            : existingObservation?.totalCost,
-    };
-
-    const tokenCounts = {
-      input: newInputCount ?? existingObservation?.promptTokens,
-      output: newOutputCount ?? existingObservation?.completionTokens,
-      total: newTotalCount || existingObservation?.totalTokens,
-    };
-
-    const calculatedCosts = ObservationProcessor.calculateTokenCosts(
-      internalModel,
-      userProvidedTokenCosts,
-      tokenCounts,
-    );
-
-    // merge metadata from existingObservation.metadata and metadata
-    const mergedMetadata = mergeJson(
-      existingObservation?.metadata
-        ? jsonSchema.parse(existingObservation.metadata)
-        : undefined,
-      this.event.body.metadata ?? undefined,
-    );
-
-    const prompt =
-      "promptName" in this.event.body &&
-      typeof this.event.body.promptName === "string" &&
-      "promptVersion" in this.event.body &&
-      typeof this.event.body.promptVersion === "number"
-        ? await prisma.prompt.findUnique({
-            where: {
-              projectId_name_version: {
-                projectId: apiScope.projectId,
-                name: this.event.body.promptName,
-                version: this.event.body.promptVersion,
-              },
-            },
-          })
-        : undefined;
-
-    // Only null if promptName and promptVersion are set but prompt is not found
-    if (prompt === null) {
-      logger.warn("Prompt not found for observation", this.event.body);
-    }
-
-    const observationId =
-      this.event.body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `observation.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    let traceId = this.event.body?.traceId;
-    if (!this.event.body.traceId && !existingObservation) {
-      // Create trace if no traceId
-      traceId = observationId;
-
-      // Insert trace into postgres
-      await prisma.trace.upsert({
-        where: {
-          id: observationId,
-        },
-        create: {
-          projectId: apiScope.projectId,
-          name: this.event.body.name,
-          id: observationId,
-          timestamp: this.event.body.startTime || new Date(),
-        },
-        update: {},
-      });
-
-      if (env.CLICKHOUSE_URL) {
-        // Insert trace into clickhouse if enabled
-        await upsertTrace({
-          id: observationId,
-          project_id: apiScope.projectId,
-          timestamp: convertDateToClickhouseDateTime(
-            this.event.body.startTime
-              ? new Date(this.event.body.startTime)
-              : new Date(),
-          ),
-          created_at: convertDateToClickhouseDateTime(new Date()),
-          updated_at: convertDateToClickhouseDateTime(new Date()),
-        });
-      }
-    }
-
-    return {
-      id: observationId,
-      create: {
-        id: observationId,
-        traceId,
-        type: type,
-        name: this.event.body.name,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        projectId: apiScope.projectId,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-      update: {
-        name: this.event.body.name ?? undefined,
-        startTime: this.event.body.startTime
-          ? new Date(this.event.body.startTime)
-          : undefined,
-        endTime:
-          "endTime" in this.event.body && this.event.body.endTime
-            ? new Date(this.event.body.endTime)
-            : undefined,
-        completionStartTime:
-          "completionStartTime" in this.event.body &&
-          this.event.body.completionStartTime
-            ? new Date(this.event.body.completionStartTime)
-            : undefined,
-        metadata: mergedMetadata ?? this.event.body.metadata ?? undefined,
-        model: "model" in this.event.body ? this.event.body.model : undefined,
-        modelParameters:
-          "modelParameters" in this.event.body
-            ? (this.event.body.modelParameters ?? undefined)
-            : undefined,
-        input: this.event.body.input ?? undefined,
-        output: this.event.body.output ?? undefined,
-        promptTokens: newInputCount,
-        completionTokens: newOutputCount,
-        totalTokens: newTotalCount,
-        unit:
-          "usage" in this.event.body
-            ? (this.event.body.usage?.unit ?? internalModel?.unit)
-            : internalModel?.unit,
-        level: this.event.body.level ?? undefined,
-        statusMessage: this.event.body.statusMessage ?? undefined,
-        parentObservationId: this.event.body.parentObservationId ?? undefined,
-        version: this.event.body.version ?? undefined,
-        promptId: prompt ? prompt.id : undefined,
-        ...(internalModel
-          ? { internalModel: internalModel.modelName }
-          : undefined),
-        inputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.inputCost
-            : undefined,
-        outputCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.outputCost
-            : undefined,
-        totalCost:
-          "usage" in this.event.body
-            ? this.event.body.usage?.totalCost
-            : undefined,
-        calculatedInputCost: calculatedCosts?.inputCost,
-        calculatedOutputCost: calculatedCosts?.outputCost,
-        calculatedTotalCost: calculatedCosts?.totalCost,
-        internalModelId: internalModel?.id,
-      },
-    };
-  }
-
-  async calculateTokenCounts(
-    projectId: string,
-    body:
-      | z.infer<typeof legacyObservationCreateEvent>["body"]
-      | z.infer<typeof generationCreateEvent>["body"],
-    calculateTokenDelegate: (p: {
-      model: Model;
-      text: unknown;
-    }) => number | undefined,
-    model?: Model,
-    existingObservation?: Omit<Observation, "input" | "output">,
-  ) {
-    let newPromptTokens = body.usage?.input || body.usageDetails?.input;
-    if (newPromptTokens === undefined && model && model.tokenizerId) {
-      if (body.input) {
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: body.input,
-        });
-      } else {
-        logger.debug(
-          `No input provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationInput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            input: true,
-          },
-        });
-
-        newPromptTokens = calculateTokenDelegate({
-          model: model,
-          text: observationInput?.input,
-        });
-      }
-    }
-
-    let newCompletionTokens = body.usage?.output || body.usageDetails?.output;
-
-    if (newCompletionTokens === undefined && model && model.tokenizerId) {
-      if (body.output) {
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: body.output,
-        });
-      } else {
-        logger.debug(
-          `No output provided, trying to calculate for id: ${existingObservation?.id}`,
-        );
-        const observationOutput = await prisma.observation.findFirst({
-          where: { id: existingObservation?.id, projectId: projectId },
-          select: {
-            output: true,
-          },
-        });
-        newCompletionTokens = calculateTokenDelegate({
-          model: model,
-          text: observationOutput?.output,
-        });
-      }
-    }
-
-    return [newPromptTokens ?? undefined, newCompletionTokens ?? undefined];
-  }
-
-  static calculateTokenCosts(
-    model: Model | null | undefined,
-    userProvidedCosts: {
-      inputCost?: Decimal | null;
-      outputCost?: Decimal | null;
-      totalCost?: Decimal | null;
-    },
-    tokenCounts: { input?: number; output?: number; total?: number },
-  ): {
-    inputCost?: Decimal | null;
-    outputCost?: Decimal | null;
-    totalCost?: Decimal | null;
-  } {
-    // If user has provided any cost point, do not calculate anything else
-    if (
-      userProvidedCosts.inputCost ||
-      userProvidedCosts.outputCost ||
-      userProvidedCosts.totalCost
-    ) {
-      return {
-        ...userProvidedCosts,
-        totalCost:
-          userProvidedCosts.totalCost ??
-          (userProvidedCosts.inputCost ?? new Decimal(0)).add(
-            userProvidedCosts.outputCost ?? new Decimal(0),
-          ),
-      };
-    }
-
-    const finalInputCost =
-      tokenCounts.input !== undefined && model?.inputPrice
-        ? model.inputPrice.mul(tokenCounts.input)
-        : undefined;
-
-    const finalOutputCost =
-      tokenCounts.output !== undefined && model?.outputPrice
-        ? model.outputPrice.mul(tokenCounts.output)
-        : finalInputCost
-          ? new Decimal(0)
-          : undefined;
-
-    const finalTotalCost =
-      tokenCounts.total !== undefined && model?.totalPrice
-        ? model.totalPrice.mul(tokenCounts.total)
-        : (finalInputCost ?? finalOutputCost)
-          ? new Decimal(finalInputCost ?? 0).add(finalOutputCost ?? 0)
-          : undefined;
-
-    return {
-      inputCost: finalInputCost,
-      outputCost: finalOutputCost,
-      totalCost: finalTotalCost,
-    };
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for observation creation");
-  }
-
-  async process(apiScope: LegacyIngestionAccessScope): Promise<Observation> {
-    this.auth(apiScope);
-
-    const existingObservation = this.event.body.id
-      ? await prisma.observation.findFirst({
-          select: {
-            // do not select I/O to spare our db
-            input: false,
-            output: false,
-
-            id: true,
-            traceId: true,
-            projectId: true,
-            type: true,
-            startTime: true,
-            endTime: true,
-            name: true,
-            metadata: true,
-            parentObservationId: true,
-            level: true,
-            statusMessage: true,
-            version: true,
-            createdAt: true,
-            updatedAt: true,
-            model: true,
-            internalModelId: true,
-            modelParameters: true,
-            promptTokens: true,
-            completionTokens: true,
-            totalTokens: true,
-            unit: true,
-            inputCost: true,
-            outputCost: true,
-            totalCost: true,
-            calculatedInputCost: true,
-            calculatedOutputCost: true,
-            calculatedTotalCost: true,
-            completionStartTime: true,
-            promptId: true,
-            internalModel: true,
-          },
-          where: { id: this.event.body.id, projectId: apiScope.projectId },
-        })
-      : null;
-
-    if (
-      existingObservation &&
-      existingObservation.projectId !== apiScope.projectId
-    ) {
-      throw new ForbiddenError(
-        `Access denied for observation creation ${existingObservation.projectId} `,
-      );
-    }
-
-    const obs = await this.convertToObservation(apiScope, existingObservation);
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    return await prisma.observation.upsert({
-      where: {
-        id: obs.id,
-      },
-      create: obs.create,
-      update: obs.update,
-    });
-  }
-}
-
-export class TraceProcessor implements EventProcessor {
-  event: z.infer<typeof traceEvent>;
-
-  constructor(event: z.infer<typeof traceEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope): void {
-    if (apiScope.accessLevel !== "all")
-      throw new ForbiddenError("Access denied for trace creation");
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const internalId =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `trace.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    logger.debug(
-      `Trying to create trace, project ${apiScope.projectId}, id: ${internalId}`,
-    );
-
-    const existingTrace = await prisma.trace.findFirst({
-      where: {
-        id: internalId,
-      },
-    });
-
-    if (existingTrace && existingTrace.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for trace creation ${existingTrace.projectId}`,
-      );
-    }
-
-    const mergedMetadata = mergeJson(
-      existingTrace?.metadata
-        ? jsonSchema.parse(existingTrace.metadata)
-        : undefined,
-      body.metadata ?? undefined,
-    );
-
-    const mergedTags =
-      existingTrace?.tags && body.tags
-        ? Array.from(new Set(existingTrace.tags.concat(body.tags ?? []))).sort()
-        : body.tags
-          ? Array.from(new Set(body.tags)).sort()
-          : undefined;
-
-    if (body.sessionId) {
-      try {
-        await prisma.traceSession.upsert({
-          where: {
-            id_projectId: {
-              id: body.sessionId,
-              projectId: apiScope.projectId,
-            },
-          },
-          create: {
-            id: body.sessionId,
-            projectId: apiScope.projectId,
-          },
-          update: {},
-        });
-      } catch (e) {
-        if (
-          e instanceof Prisma.PrismaClientKnownRequestError &&
-          e.code === "P2002"
-        ) {
-          logger.warn(
-            `Failed to upsert session. Session ${body.sessionId} in project ${apiScope.projectId} already exists`,
-          );
-        } else {
-          throw e;
-        }
-      }
-    }
-
-    // Do not use nested upserts or multiple where conditions as this should be a single native database upsert
-    // https://www.prisma.io/docs/orm/reference/prisma-client-reference#database-upserts
-    const upsertedTrace = await prisma.trace.upsert({
-      where: {
-        id: internalId,
-      },
-      create: {
-        id: internalId,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        name: body.name ?? undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        projectId: apiScope.projectId,
-        tags: mergedTags ?? undefined,
-      },
-      update: {
-        name: body.name ?? undefined,
-        timestamp: this.event.body.timestamp
-          ? new Date(this.event.body.timestamp)
-          : undefined,
-        userId: body.userId ?? undefined,
-        input: body.input ?? undefined,
-        output: body.output ?? undefined,
-        metadata: mergedMetadata ?? body.metadata ?? undefined,
-        release: body.release ?? undefined,
-        version: body.version ?? undefined,
-        sessionId: body.sessionId ?? undefined,
-        public: body.public ?? undefined,
-        tags: mergedTags ?? undefined,
-      },
-    });
-    return upsertedTrace;
-  }
-}
-
-export class ScoreProcessor implements EventProcessor {
-  event: z.infer<typeof scoreEvent>;
-
-  constructor(event: z.infer<typeof scoreEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    if (apiScope.accessLevel !== "scores" && apiScope.accessLevel !== "all")
-      throw new ForbiddenError(
-        `Access denied for score creation, ${apiScope.accessLevel}`,
-      );
-  }
-
-  async process(
-    apiScope: LegacyIngestionAccessScope,
-  ): Promise<Trace | Observation | Score> {
-    const { body } = this.event;
-
-    this.auth(apiScope);
-
-    const id =
-      body.id ??
-      (() => {
-        const newId = v4();
-        logger.info(
-          `score.id is null. Generating for projectId: ${apiScope.projectId}, id: ${newId}`,
-        );
-        return newId;
-      })();
-
-    const existingScore = await prisma.score.findFirst({
-      where: {
-        id: id,
-      },
-      select: {
-        projectId: true,
-      },
-    });
-    if (existingScore && existingScore.projectId !== apiScope.projectId) {
-      throw new ForbiddenError(
-        `Access denied for score creation ${existingScore.projectId}`,
-      );
-    }
-
-    const validatedScore = await validateAndInflateScore({
-      body,
-      scoreId: id,
-      projectId: apiScope.projectId,
-    });
-
-    return await prisma.score.upsert({
-      where: {
-        id_projectId: {
-          id,
-          projectId: apiScope.projectId,
-        },
-      },
-      create: {
-        ...validatedScore,
-      },
-      update: {
-        ...validatedScore,
-      },
-    });
-  }
-}
-
-export class SdkLogProcessor implements EventProcessor {
-  event: z.infer<typeof sdkLogEvent>;
-
-  constructor(event: z.infer<typeof sdkLogEvent>) {
-    this.event = event;
-  }
-
-  auth(apiScope: LegacyIngestionAccessScope) {
-    return;
-  }
-
-  process() {
-    try {
-      logger.info("SDK Log", this.event);
-      return undefined;
-    } catch (error) {
-      return undefined;
-    }
-  }
-}

packages/shared/src/server/ingestion/legacy/index.ts

@@ -1,161 +0,0 @@
-import { env } from "node:process";
-import z from "zod";
-import { ForbiddenError, UnauthorizedError } from "../../../errors";
-import { eventTypes, ingestionApiSchema, IngestionEventType } from "../types";
-import { getProcessorForEvent } from "./EventProcessor";
-import { ApiAccessScope } from "../../auth/types";
-import { backOff } from "exponential-backoff";
-import { Model } from "../../..";
-import { logger } from "../../logger";
-
-export type BatchResult = {
-  result: unknown;
-  id: string;
-  type: string;
-};
-
-type TokenCountInput = {
-  model: Model;
-  text: unknown;
-};
-
-export type LegacyIngestionAccessScope = Omit<
-  ApiAccessScope,
-  "orgId" | "plan" | "rateLimitOverrides"
->;
-
-type LegacyIngestionAuthHeaderVerificationResult =
-  | {
-      validKey: true;
-      scope: LegacyIngestionAccessScope;
-    }
-  | {
-      validKey: false;
-      error: string;
-    };
-
-export const handleBatch = async (
-  events: z.infer<typeof ingestionApiSchema>["batch"],
-  authCheck: LegacyIngestionAuthHeaderVerificationResult,
-  calculateTokenDelegate: (p: TokenCountInput) => number | undefined,
-) => {
-  logger.debug(`handling ingestion ${events.length} events`);
-
-  if (!authCheck.validKey) throw new UnauthorizedError(authCheck.error);
-
-  const results: BatchResult[] = []; // Array to store the results
-
-  const errors: {
-    error: unknown;
-    id: string;
-    type: string;
-  }[] = []; // Array to store the errors
-
-  for (const singleEvent of events) {
-    try {
-      const result = await retry(async () => {
-        return await handleSingleEvent(
-          singleEvent,
-          authCheck.scope,
-          calculateTokenDelegate,
-        );
-      });
-      results.push({
-        result: result,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      }); // Push each result into the array
-    } catch (error) {
-      // Handle or log the error if `handleSingleEvent` fails
-      logger.error("Error handling event:", error);
-      // Decide how to handle the error: rethrow, continue, or push an error object to results
-      // For example, push an error object:
-      errors.push({
-        error,
-        id: singleEvent.id,
-        type: singleEvent.type,
-      });
-    }
-  }
-
-  return { results, errors };
-};
-
-async function retry<T>(request: () => Promise<T>): Promise<T> {
-  return await backOff(request, {
-    numOfAttempts: 5,
-    retry: (e: Error, attemptNumber: number) => {
-      if (e instanceof UnauthorizedError || e instanceof ForbiddenError) {
-        logger.info("not retrying auth error");
-        return false;
-      }
-      logger.info(`retrying processing events ${attemptNumber}`);
-      return true;
-    },
-  });
-}
-
-const handleSingleEvent = async (
-  event: IngestionEventType,
-  apiScope: LegacyIngestionAccessScope,
-  calculateTokenDelegate: (p: {
-    model: Model;
-    text: unknown;
-  }) => number | undefined,
-) => {
-  const { body } = event;
-  let restEvent = body;
-  if ("input" in body) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { input, ...rest } = body;
-    restEvent = rest;
-  }
-  if ("output" in restEvent) {
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    const { output, ...rest } = restEvent;
-    restEvent = rest;
-  }
-
-  logger.debug(
-    `handling single event ${event.id} of type ${event.type}:  ${JSON.stringify({ body: restEvent })}`,
-  );
-
-  const cleanedEvent = cleanEvent(event) as IngestionEventType;
-
-  // Deny access to non-score events if the access level is not "all"
-  // This is an additional safeguard to auth checks in EventProcessor
-  if (
-    apiScope.accessLevel !== "all" &&
-    cleanedEvent.type !== eventTypes.SCORE_CREATE
-  ) {
-    throw new ForbiddenError("Access denied. Event type not allowed.");
-  }
-
-  return getProcessorForEvent(cleanedEvent, calculateTokenDelegate).process(
-    apiScope,
-  );
-};
-
-// cleans NULL characters from the event
-export function cleanEvent(obj: unknown): unknown {
-  if (typeof obj === "string") {
-    return obj.replace(/\u0000/g, "");
-  } else if (typeof obj === "object" && obj !== null) {
-    if (Array.isArray(obj)) {
-      return obj.map(cleanEvent);
-    } else {
-      // Here we assert that obj is a Record<string, unknown>
-      const objAsRecord = obj as Record<string, unknown>;
-      const newObj: Record<string, unknown> = {};
-      for (const key in objAsRecord) {
-        newObj[key] = cleanEvent(objAsRecord[key]);
-      }
-      return newObj;
-    }
-  } else {
-    return obj;
-  }
-}
-
-export const isUndefinedOrNull = <T>(val?: T | null): val is undefined | null =>
-  val === undefined || val === null;

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -18,43 +18,51 @@ import {
   traceException,
 } from "../instrumentation";
 import { logger } from "../logger";
-import { LegacyIngestionEventType, QueueJobs } from "../queues";
+import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
-import { LegacyIngestionQueue } from "../redis/legacyIngestion";
 import { redis } from "../redis/redis";
-import { handleBatch } from "./legacy";
-import {
-  StorageService,
-  StorageServiceFactory,
-} from "../services/StorageService";
-import { getProcessorForEvent } from "./legacy/EventProcessor";
 import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+import { uploadEventToS3 } from "../utils/eventLog";
 
 export type TokenCountDelegate = (p: {
   model: Model;
   text: unknown;
 }) => number | undefined;
 
-let s3StorageServiceClient: StorageService;
-
-const getS3StorageServiceClient = (bucketName: string): StorageService => {
-  if (!s3StorageServiceClient) {
-    s3StorageServiceClient = StorageServiceFactory.getInstance({
-      bucketName,
-      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
-      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
-      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
-      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
-      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
-    });
+/**
+ * Get the delay for the event based on the event type. Uses delay if set, 0 if current UTC timestamp is not between
+ * 23:45 and 00:15, and env.LANGFUSE_INGESTION_QUEUE_DELAY_MS otherwise.
+ * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
+ * @param delay - Delay overwrite. Used if non-null.
+ */
+const getDelay = (delay: number | null) => {
+  if (delay !== null) {
+    return delay;
   }
-  return s3StorageServiceClient;
+  const now = new Date();
+  const hours = now.getUTCHours();
+  const minutes = now.getUTCMinutes();
+
+  if ((hours === 23 && minutes >= 45) || (hours === 0 && minutes <= 15)) {
+    return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
+  }
+
+  // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
+  // we use this instead.
+  // Values should be revisited based on a cost/performance trade-off.
+  return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
 };
 
+/**
+ * Processes a batch of events.
+ * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
+ * @param authCheck - AuthHeaderValidVerificationResult
+ * @param delay - (Optional) Delay in ms to wait before processing events in the batch.
+ */
 export const processEventBatch = async (
   input: unknown[],
   authCheck: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
+  delay: number | null = null,
 ): Promise<{
   successes: { id: string; status: number }[];
   errors: {
@@ -99,7 +107,7 @@ export const processEventBatch = async (
         });
         return [];
       }
-      if (!isAuthorized(parsed.data, authCheck, tokenCountDelegate)) {
+      if (!isAuthorized(parsed.data, authCheck)) {
         authenticationErrors.push({
           id: parsed.data.id,
           error: new UnauthorizedError("Access Scope Denied"),
@@ -157,9 +165,6 @@ export const processEventBatch = async (
    ********************/
   let s3UploadErrored = false;
   await instrumentAsync({ name: "s3-upload-events" }, async () => {
-    const s3Client = getS3StorageServiceClient(
-      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-    );
     // S3 Event Upload is blocking, but non-failing.
     // If a promise rejects, we log it below, but do not throw an error.
     // In this case, we upload the full batch into the Redis queue.
@@ -169,8 +174,21 @@ export const processEventBatch = async (
         // That way we batch updates from the same invocation into a single file and reduce
         // write operations on S3.
         const { data, key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-        return s3Client.uploadJson(
-          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${authCheck.scope.projectId}/${getClickhouseEntityType(type)}/${eventBodyId}/${key}.json`,
+        return uploadEventToS3(
+          {
+            projectId: authCheck.scope.projectId,
+            entityType: getClickhouseEntityType(type),
+            entityId: eventBodyId,
+            eventId: key,
+            traceId:
+              data // Use the first truthy traceId for the event log.
+                .flatMap((event) =>
+                  "traceId" in event.body && event.body.traceId
+                    ? [event.body.traceId]
+                    : [],
+                )
+                .shift() ?? null,
+          },
           data,
         );
       }),
@@ -185,154 +203,65 @@ export const processEventBatch = async (
     });
   });
 
-  // This is a workaround to allow us to disable async ingestion processing for SDK CI testing
-  // TODO: remove this block after SDKs are ready for V3 async ingestion processing
-  if (env.LANGFUSE_SDK_CI_SYNC_PROCESSING_ENABLED === "true") {
-    const result = await handleBatch(
-      sortedBatch,
-      authCheck,
-      tokenCountDelegate,
-    );
-
-    //  in case we did not return early, we return the result here
-    return aggregateBatchResult(
-      [...validationErrors, ...authenticationErrors, ...result.errors],
-      result.results,
-      authCheck.scope.projectId,
-    );
-  }
-
   // Send each event individually to IngestionQueue for ClickHouse processing
-  if (env.LANGFUSE_CLICKHOUSE_INGESTION_ENABLED === "true") {
-    if (s3UploadErrored) {
-      throw new Error(
-        "Failed to upload events to blob storage, aborting event processing",
-      );
-    }
-
-    if (redis) {
-      const queue = IngestionQueue.getInstance();
-      await Promise.all(
-        Object.keys(sortedBatchByEventBodyId).map(async (id) =>
-          queue
-            ? queue.add(
-                QueueJobs.IngestionJob,
-                {
-                  id: randomUUID(),
-                  timestamp: new Date(),
-                  name: QueueJobs.IngestionJob as const,
-                  payload: {
-                    data: {
-                      type: sortedBatchByEventBodyId[id].type,
-                      eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
-                    },
-                    authCheck,
-                  },
-                },
-                {
-                  delay: env.LANGFUSE_INGESTION_QUEUE_DELAY_MS,
-                },
-              )
-            : Promise.reject("Failed to instantiate queue"),
-        ),
-      );
-      if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED !== "true") {
-        // If postgres ingestion is disabled, we return early
-        return aggregateBatchResult(
-          [...validationErrors, ...authenticationErrors],
-          sortedBatch.map((event) => ({ id: event.id, result: event })),
-          authCheck.scope.projectId,
-        );
-      }
-    }
-  }
-
-  if (env.LANGFUSE_POSTGRES_INGESTION_ENABLED === "true") {
-    // As part of the legacy processing we sent the entire batch to the worker.
-    if (redis) {
-      const queue = LegacyIngestionQueue.getInstance();
-
-      if (queue) {
-        let addToQueueFailed = false;
-
-        const queuePayload: LegacyIngestionEventType = !s3UploadErrored
-          ? {
-              data: Object.keys(sortedBatchByEventBodyId).map((id) => {
-                const { key, type, eventBodyId } = sortedBatchByEventBodyId[id];
-                return {
-                  type,
-                  eventBodyId,
-                  eventId: key,
-                };
-              }),
-              authCheck,
-              useS3EventStore: true,
-            }
-          : { data: sortedBatch, authCheck, useS3EventStore: false };
-
-        try {
-          await queue.add(QueueJobs.LegacyIngestionJob, {
-            payload: queuePayload,
-            id: randomUUID(),
-            timestamp: new Date(),
-            name: QueueJobs.LegacyIngestionJob as const,
-          });
-        } catch (e: unknown) {
-          logger.warn(
-            "Failed to add batch to queue, falling back to sync processing",
-            e,
-          );
-          addToQueueFailed = true;
-        }
-
-        if (!addToQueueFailed) {
-          return aggregateBatchResult(
-            // we are not sending additional server errors to the client in case of early return
-            [...validationErrors, ...authenticationErrors],
-            sortedBatch.map((event) => ({ id: event.id, result: event })),
-            authCheck.scope.projectId,
-          );
-        }
-      } else {
-        logger.warn(
-          "Ingestion queue not initialized, falling back to sync processing",
-        );
-      }
-    }
-
-    /*******************
-     * SYNC PROCESSING *
-     *******************/
-    const result = await handleBatch(
-      sortedBatch,
-      authCheck,
-      tokenCountDelegate,
-    );
-
-    //  in case we did not return early, we return the result here
-    return aggregateBatchResult(
-      [...validationErrors, ...authenticationErrors, ...result.errors],
-      result.results,
-      authCheck.scope.projectId,
+  if (s3UploadErrored) {
+    throw new Error(
+      "Failed to upload events to blob storage, aborting event processing",
     );
   }
 
-  throw new Error(
-    "Either Clickhouse or Postgres ingestion (or both) must be enabled",
+  if (!redis) {
+    throw new Error("Redis not initialized, aborting event processing");
+  }
+
+  const queue = IngestionQueue.getInstance();
+  await Promise.all(
+    Object.keys(sortedBatchByEventBodyId).map(async (id) =>
+      queue
+        ? queue.add(
+            QueueJobs.IngestionJob,
+            {
+              id: randomUUID(),
+              timestamp: new Date(),
+              name: QueueJobs.IngestionJob as const,
+              payload: {
+                data: {
+                  type: sortedBatchByEventBodyId[id].type,
+                  eventBodyId: sortedBatchByEventBodyId[id].eventBodyId,
+                  fileKey: sortedBatchByEventBodyId[id].key,
+                },
+                authCheck,
+              },
+            },
+            { delay: getDelay(delay) },
+          )
+        : Promise.reject("Failed to instantiate queue"),
+    ),
+  );
+
+  return aggregateBatchResult(
+    [...validationErrors, ...authenticationErrors],
+    sortedBatch.map((event) => ({ id: event.id, result: event })),
+    authCheck.scope.projectId,
   );
 };
 
 const isAuthorized = (
   event: IngestionEventType,
   authScope: AuthHeaderValidVerificationResult,
-  tokenCountDelegate: TokenCountDelegate,
 ): boolean => {
-  try {
-    getProcessorForEvent(event, tokenCountDelegate).auth(authScope.scope);
+  if (event.type === eventTypes.SDK_LOG) {
     return true;
-  } catch (error) {
-    return false;
   }
+
+  if (event.type === eventTypes.SCORE_CREATE) {
+    return (
+      authScope.scope.accessLevel === "scores" ||
+      authScope.scope.accessLevel === "all"
+    );
+  }
+
+  return authScope.scope.accessLevel === "all";
 };
 
 /**

packages/shared/src/server/ingestion/types.ts

@@ -66,8 +66,12 @@ const OpenAIUsageSchema = z
     prompt_tokens: z.number().nonnegative(),
     completion_tokens: z.number().nonnegative(),
     total_tokens: z.number().nonnegative(),
-    prompt_tokens_details: z.record(z.string(), z.number().nonnegative()),
-    completion_tokens_details: z.record(z.string(), z.number().nonnegative()),
+    prompt_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
+    completion_tokens_details: z
+      .record(z.string(), z.number().nonnegative())
+      .nullish(),
   })
   .strict()
   .transform((v) => {
@@ -112,13 +116,16 @@ export const UsageOrCostDetails = z
   .union([OpenAIUsageSchema, RawUsageOrCostDetails])
   .nullish();
 
+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 export const TraceBody = z.object({
   id: z.string().nullish(),
   timestamp: stringDateTime,
   name: z.string().max(1000).nullish(),
   externalId: z.string().nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
   sessionId: z.string().nullish(),
   userId: z.string().nullish(),
   metadata: jsonSchema.nullish(),
@@ -133,8 +140,8 @@ export const OptionalObservationBody = z.object({
   name: z.string().nullish(),
   startTime: stringDateTime,
   metadata: jsonSchema.nullish(),
-  input: jsonSchema.nullish(),
-  output: jsonSchema.nullish(),
+  input: z.any().nullish(),
+  output: z.any().nullish(),
   level: z.nativeEnum(ObservationLevel).nullish(),
   statusMessage: z.string().nullish(),
   parentObservationId: z.string().nullish(),

packages/shared/src/server/instrumentation/index.ts

@@ -226,6 +226,14 @@ export const recordHistogram = (
   dd.dogstatsd.histogram(stat, value, tags);
 };
 
+export const recordDistribution = (
+  stat: string,
+  value?: number | undefined,
+  tags?: { [tag: string]: string | number } | undefined,
+) => {
+  dd.dogstatsd.distribution(stat, value, tags);
+};
+
 /**
  * Converts a queue name to the matching datadog metric name.
  * Consumer only needs to append the relevant suffix.

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -1,7 +1,5 @@
 import type { ZodSchema } from "zod";
 
-import { CallbackHandler } from "langfuse-langchain";
-
 import { ChatAnthropic } from "@langchain/anthropic";
 import { ChatVertexAI } from "@langchain/google-vertexai";
 import { ChatBedrockConverse } from "@langchain/aws";
@@ -21,27 +19,20 @@ import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
 } from "../../interfaces/customLLMProviderConfigSchemas";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
-import {
-  processEventBatch,
-  type TokenCountDelegate,
-} from "../ingestion/processEventBatch";
+import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
-import { ChatMessage, ChatMessageRole, LLMAdapter, ModelParams } from "./types";
-
+import {
+  ChatMessage,
+  ChatMessageRole,
+  LLMAdapter,
+  ModelParams,
+  TraceParams,
+} from "./types";
+import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
 type ProcessTracedEvents = () => Promise<void>;
 
-export type TraceParams = {
-  traceName: string;
-  traceId: string;
-  projectId: string;
-  tags: string[];
-  tokenCountDelegate: TokenCountDelegate;
-  authCheck: AuthHeaderValidVerificationResult;
-};
-
 type LLMCompletionParams = {
   messages: ChatMessage[];
   modelParams: ModelParams;
@@ -49,9 +40,11 @@ type LLMCompletionParams = {
   callbacks?: BaseCallbackHandler[];
   baseURL?: string;
   apiKey: string;
+  extraHeaders?: Record<string, string>;
   maxRetries?: number;
   config?: Record<string, string> | null;
   traceParams?: TraceParams;
+  throwOnError?: boolean; // default is true
 };
 
 type FetchLLMCompletionParams = LLMCompletionParams & {
@@ -100,6 +93,8 @@ export async function fetchLLMCompletion(
     maxRetries,
     config,
     traceParams,
+    extraHeaders,
+    throwOnError = true,
   } = params;
 
   let finalCallbacks: BaseCallbackHandler[] | undefined = callbacks ?? [];
@@ -111,7 +106,6 @@ export async function fetchLLMCompletion(
       _isLocalEventExportEnabled: true,
       tags: traceParams.tags,
     });
-
     finalCallbacks.push(handler);
 
     processTracedEvents = async () => {
@@ -122,7 +116,6 @@ export async function fetchLLMCompletion(
         await processEventBatch(
           JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
           traceParams.authCheck,
-          traceParams.tokenCountDelegate,
         );
       } catch (e) {
         logger.error("Failed to process traced events", { error: e });
@@ -177,6 +170,7 @@ export async function fetchLLMCompletion(
       maxRetries,
       configuration: {
         baseURL,
+        defaultHeaders: extraHeaders,
       },
       timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
@@ -237,17 +231,18 @@ export async function fetchLLMCompletion(
     runName: traceParams?.traceName,
   };
 
-  if (params.structuredOutputSchema) {
-    return {
-      completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
-        .withStructuredOutput(params.structuredOutputSchema)
-        .invoke(finalMessages, runConfig),
-      processTracedEvents,
-    };
-  }
+  try {
+    if (params.structuredOutputSchema) {
+      return {
+        completion: await (chatModel as ChatOpenAI) // Typecast necessary due to https://github.com/langchain-ai/langchainjs/issues/6795
+          .withStructuredOutput(params.structuredOutputSchema)
+          .invoke(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }
 
-  /*
-  Workaround OpenAI o1 while in beta:
+    /*
+  Workaround OpenAI reasoning models:
   
   This is a temporary workaround to avoid sending system messages to OpenAI's O1 models.
   O1 models do not support in beta:
@@ -258,43 +253,55 @@ export async function fetchLLMCompletion(
 
   Reference: https://platform.openai.com/docs/guides/reasoning/beta-limitations
   */
-  if (modelParams.model.startsWith("o1-")) {
-    return {
-      completion: await new ChatOpenAI({
-        openAIApiKey: apiKey,
-        modelName: modelParams.model,
-        temperature: 1,
-        maxTokens: undefined,
-        topP: undefined,
-        callbacks,
-        maxRetries,
-        configuration: {
-          baseURL,
-        },
-        timeout: 1000 * 60 * 2, // 2 minutes timeout
-      })
-        .pipe(new StringOutputParser())
-        .invoke(
-          finalMessages.filter((message) => message._getType() !== "system"),
-          runConfig,
-        ),
-      processTracedEvents,
-    };
-  }
+    if (
+      modelParams.model.startsWith("o1-") ||
+      modelParams.model.startsWith("o3-")
+    ) {
+      return {
+        completion: await new ChatOpenAI({
+          openAIApiKey: apiKey,
+          modelName: modelParams.model,
+          temperature: 1,
+          maxTokens: undefined,
+          topP: undefined,
+          callbacks,
+          maxRetries,
+          modelKwargs: {
+            max_completion_tokens: modelParams.max_tokens,
+          },
+          configuration: {
+            baseURL,
+          },
+          timeout: 1000 * 60 * 2, // 2 minutes timeout
+        })
+          .pipe(new StringOutputParser())
+          .invoke(
+            finalMessages.filter((message) => message._getType() !== "system"),
+            runConfig,
+          ),
+        processTracedEvents,
+      };
+    }
+
+    if (streaming) {
+      return {
+        completion: await chatModel
+          .pipe(new BytesOutputParser())
+          .stream(finalMessages, runConfig),
+        processTracedEvents,
+      };
+    }
 
-  if (streaming) {
     return {
       completion: await chatModel
-        .pipe(new BytesOutputParser())
-        .stream(finalMessages, runConfig),
+        .pipe(new StringOutputParser())
+        .invoke(finalMessages, runConfig),
       processTracedEvents,
     };
+  } catch (error) {
+    if (throwOnError) {
+      throw error;
+    }
+    return { completion: null, processTracedEvents };
   }
-
-  return {
-    completion: await chatModel
-      .pipe(new StringOutputParser())
-      .invoke(finalMessages, runConfig),
-    processTracedEvents,
-  };
 }

packages/shared/src/server/llm/types.ts

@@ -1,6 +1,8 @@
 import { LlmApiKeys } from "@prisma/client";
 import z from "zod";
 import { BedrockConfigSchema } from "../../interfaces/customLLMProviderConfigSchemas";
+import { TokenCountDelegate } from "../ingestion/processEventBatch";
+import { AuthHeaderValidVerificationResult } from "../auth/types";
 
 export type PromptVariable = { name: string; value: string; isUsed: boolean };
 
@@ -16,7 +18,7 @@ export enum LLMAdapter {
   OpenAI = "openai",
   Azure = "azure",
   Bedrock = "bedrock",
-  VertexAI = "vertex-ai",
+  VertexAI = "google-vertex-ai",
 }
 
 export enum ChatMessageRole {
@@ -71,6 +73,7 @@ export const ExperimentMetadataSchema = z
     provider: z.string(),
     model: z.string(),
     model_params: ZodModelConfig,
+    error: z.string().optional(),
   })
   .strict();
 export type ExperimentMetadata = z.infer<typeof ExperimentMetadataSchema>;
@@ -82,6 +85,8 @@ export const openAIModels = [
   "gpt-4o-2024-05-13",
   "gpt-4o-mini",
   "gpt-4o-mini-2024-07-18",
+  "o3-mini",
+  "o3-mini-2025-01-31",
   "o1-preview",
   "o1-preview-2024-09-12",
   "o1-mini",
@@ -116,6 +121,10 @@ export const anthropicModels = [
 ] as const;
 
 export const vertexAIModels = [
+  "gemini-2.0-pro-exp-02-05",
+  "gemini-2.0-flash-001",
+  "gemini-2.0-flash-lite-preview-02-05",
+  "gemini-2.0-flash-exp",
   "gemini-1.5-pro",
   "gemini-1.5-flash",
   "gemini-1.0-pro",
@@ -147,6 +156,8 @@ export const LLMApiKeySchema = z
     provider: z.string(),
     displaySecretKey: z.string(),
     secretKey: z.string(),
+    extraHeaders: z.string().nullish(),
+    extraHeaderKeys: z.array(z.string()),
     baseURL: z.string().nullable(),
     customModels: z.array(z.string()),
     withDefaultModels: z.boolean(),
@@ -160,3 +171,12 @@ export type LLMApiKey =
   z.infer<typeof LLMApiKeySchema> extends LlmApiKeys
     ? z.infer<typeof LLMApiKeySchema>
     : never;
+
+export type TraceParams = {
+  traceName: string;
+  traceId: string;
+  projectId: string;
+  tags: string[];
+  tokenCountDelegate: TokenCountDelegate;
+  authCheck: AuthHeaderValidVerificationResult;
+};

packages/shared/src/server/llm/utils.ts

@@ -0,0 +1,13 @@
+import { z } from "zod";
+
+import { decrypt } from "../../encryption";
+
+const ExtraHeaderSchema = z.record(z.string(), z.string());
+
+export function decryptAndParseExtraHeaders(
+  extraHeaders: string | null | undefined,
+) {
+  if (!extraHeaders) return;
+
+  return ExtraHeaderSchema.parse(JSON.parse(decrypt(extraHeaders)));
+}

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -383,6 +383,10 @@ export class FilterList {
     return this.filters.find(predicate);
   }
 
+  some(predicate: (filter: Filter) => boolean) {
+    return this.filters.some(predicate);
+  }
+
   length() {
     return this.filters.length;
   }

packages/shared/src/server/queries/createGenerationsQuery.ts

@@ -1,11 +1,4 @@
-import { ObservationView, Prisma } from "@prisma/client";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { observationsTableCols } from "../../observationsTable";
-import { TableFilters } from "./types";
+import { ObservationView } from "@prisma/client";
 
 type AdditionalObservationFields = {
   traceName: string | null;
@@ -26,147 +19,3 @@ export type IOAndMetadataOmittedObservations = Array<
   Omit<ObservationView, "input" | "output" | "metadata"> &
     AdditionalObservationFields
 >;
-
-export function parseGetAllGenerationsInput(filters: TableFilters) {
-  const searchCondition = filters.searchQuery
-    ? Prisma.sql`AND (
-        o."id" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."name" ILIKE ${`%${filters.searchQuery}%`} OR
-        o."model" ILIKE ${`%${filters.searchQuery}%`} OR
-        t."name" ILIKE ${`%${filters.searchQuery}%`}
-      )`
-    : Prisma.empty;
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filters.filter ?? [],
-    observationsTableCols,
-    "observations",
-  );
-
-  const orderByCondition = orderByToPrismaSql(
-    filters.orderBy,
-    observationsTableCols,
-  );
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const startTimeFilter = filters.filter?.find(
-    (f) => f.column === "Start Time" && f.type === "datetime",
-  );
-
-  const datetimeFilter =
-    startTimeFilter && startTimeFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          startTimeFilter.operator,
-          startTimeFilter.value,
-        )
-      : Prisma.empty;
-
-  return {
-    searchCondition,
-    filterCondition,
-    orderByCondition,
-    datetimeFilter,
-  };
-}
-
-export function createGenerationsQuery({
-  projectId,
-  datetimeFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectIOAndMetadata = false,
-  selectScoreValues = false,
-}: {
-  projectId: string;
-  datetimeFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectIOAndMetadata?: boolean;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  WITH scores_avg AS (
-        SELECT
-          trace_id,
-          observation_id,
-          ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-          jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-        FROM (
-          SELECT
-            trace_id,
-            observation_id,
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            avg(value) avg_value,
-            comment
-          FROM
-            scores
-          WHERE
-            project_id = ${projectId}
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-          GROUP BY
-            trace_id,
-            observation_id,
-            name,
-            comment
-          ORDER BY
-            trace_id
-          ) tmp
-        GROUP BY
-          trace_id, 
-          observation_id
-      )
-      SELECT
-        ${selectScoreValues ? Prisma.sql`s_avg."scores_values" AS "scores",` : Prisma.empty}
-        o.id,
-        o.name,
-        o.model,
-        o."modelParameters",
-        o.start_time as "startTime",
-        o.end_time as "endTime",
-        ${selectIOAndMetadata ? Prisma.sql`o.input, o.output, o.metadata,` : Prisma.empty} 
-        o.trace_id as "traceId",
-        t.name as "traceName",
-        o.completion_start_time as "completionStartTime",
-        o.time_to_first_token as "timeToFirstToken",
-        o.prompt_tokens as "promptTokens",
-        o.completion_tokens as "completionTokens",
-        o.total_tokens as "totalTokens",
-        o.unit,
-        o.level,
-        o.status_message as "statusMessage",
-        o.version,
-        o.model_id as "modelId",
-        o.input_price as "inputPrice",
-        o.output_price as "outputPrice",
-        o.total_price as "totalPrice",
-        o.calculated_input_cost as "calculatedInputCost",
-        o.calculated_output_cost as "calculatedOutputCost",
-        o.calculated_total_cost as "calculatedTotalCost",
-        o."latency",
-        o.prompt_id as "promptId",
-        p.name as "promptName",
-        p.version as "promptVersion",
-        t.tags as "traceTags"
-      FROM observations_view o
-      JOIN traces t ON t.id = o.trace_id AND t.project_id = ${projectId}
-      LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = t.id and s_avg.observation_id = o.id
-      LEFT JOIN prompts p ON p.id = o.prompt_id AND p.project_id = ${projectId}
-      WHERE
-        o.project_id = ${projectId}
-        AND o.type = 'GENERATION'
-        ${datetimeFilter}
-        ${searchCondition}
-        ${filterCondition}
-        ${orderByCondition}
-      ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-      ${page && limit ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-    `;
-}

packages/shared/src/server/queries/createTracesQuery.ts

@@ -1,135 +0,0 @@
-import { Prisma } from "@prisma/client";
-import { TableFilters } from "./types";
-import {
-  datetimeFilterToPrismaSql,
-  tableColumnsToSqlFilterAndPrefix,
-} from "../filterToPrisma";
-import { tracesTableCols } from "../../tableDefinitions/tracesTable";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-
-export function parseTraceAllFilters(input: TableFilters) {
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    input.filter ?? [],
-    tracesTableCols,
-    "traces",
-  );
-  const orderByCondition = orderByToPrismaSql(input.orderBy, tracesTableCols);
-
-  // to improve query performance, add timeseries filter to observation queries as well
-  const timeseriesFilter = input.filter?.find(
-    (f) => f.column === "Timestamp" && f.type === "datetime",
-  );
-
-  const observationTimeseriesFilter =
-    timeseriesFilter && timeseriesFilter.type === "datetime"
-      ? datetimeFilterToPrismaSql(
-          "start_time",
-          timeseriesFilter.operator,
-          timeseriesFilter.value,
-        )
-      : Prisma.empty;
-
-  const searchCondition = input.searchQuery
-    ? Prisma.sql`AND (
-    t."id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."external_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."user_id" ILIKE ${`%${input.searchQuery}%`} OR 
-    t."name" ILIKE ${`%${input.searchQuery}%`}
-  )`
-    : Prisma.empty;
-
-  return {
-    filterCondition,
-    orderByCondition,
-    observationTimeseriesFilter,
-    searchCondition,
-  };
-}
-
-export function createTracesQuery({
-  select,
-  projectId,
-  observationTimeseriesFilter = Prisma.empty,
-  page,
-  limit,
-  searchCondition = Prisma.empty,
-  filterCondition = Prisma.empty,
-  orderByCondition = Prisma.empty,
-  selectScoreValues = false,
-}: {
-  select: Prisma.Sql;
-  projectId: string;
-  observationTimeseriesFilter?: Prisma.Sql;
-  page?: number;
-  limit?: number;
-  searchCondition?: Prisma.Sql;
-  filterCondition?: Prisma.Sql;
-  orderByCondition?: Prisma.Sql;
-  selectScoreValues?: boolean;
-}) {
-  return Prisma.sql`
-  SELECT
-      ${select}
-  FROM
-    "traces" AS t
-  LEFT JOIN LATERAL (
-    SELECT
-      SUM(prompt_tokens) AS "promptTokens",
-      SUM(completion_tokens) AS "completionTokens",
-      SUM(total_tokens) AS "totalTokens",
-      SUM(calculated_total_cost) AS "calculatedTotalCost",
-      SUM(calculated_input_cost) AS "calculatedInputCost",
-      SUM(calculated_output_cost) AS "calculatedOutputCost"
-    FROM
-      "observations_view"
-    WHERE
-      trace_id = t.id
-      AND "type" = 'GENERATION'
-      AND "project_id" = ${projectId}
-      ${observationTimeseriesFilter}
-  ) AS generation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      COUNT(*) AS "observationCount",
-      EXTRACT(EPOCH FROM COALESCE(MAX("end_time"), MAX("start_time"))) - EXTRACT(EPOCH FROM MIN("start_time"))::double precision AS "latency",
-      COALESCE(  
-        MAX(CASE WHEN level = 'ERROR' THEN 'ERROR' END),  
-        MAX(CASE WHEN level = 'WARNING' THEN 'WARNING' END),  
-        MAX(CASE WHEN level = 'DEFAULT' THEN 'DEFAULT' END),  
-        'DEBUG'  
-      ) AS "level"
-    FROM
-        "observations"
-    WHERE
-        trace_id = t.id
-        AND "project_id" = ${projectId}
-         ${observationTimeseriesFilter}
-  ) AS observation_metrics ON true
-  LEFT JOIN LATERAL (
-    SELECT
-      ${selectScoreValues ? Prisma.sql`jsonb_object_agg(name::text, "values") AS "scores_values",` : Prisma.empty}
-      jsonb_object_agg(name::text, avg_value::double precision) AS "scores_avg"
-    FROM (
-        SELECT
-            name,
-            ${selectScoreValues ? Prisma.sql`array_agg(COALESCE(string_value, value::text)) AS "values",` : Prisma.empty}
-            AVG(value) avg_value
-        FROM
-            scores
-        WHERE
-            trace_id = t.id
-            AND t."project_id" = ${projectId}     
-            ${selectScoreValues ? Prisma.empty : Prisma.sql`AND scores."data_type" IN ('NUMERIC', 'BOOLEAN')`}
-        GROUP BY
-            name
-    ) tmp
-  ) AS s_avg ON true
-  WHERE 
-    t."project_id" = ${projectId}
-    ${searchCondition}
-    ${filterCondition}
-  ${orderByCondition}
-  ${limit ? Prisma.sql`LIMIT ${limit}` : Prisma.empty}
-  ${page !== undefined && limit !== undefined ? Prisma.sql`OFFSET ${page * limit}` : Prisma.empty}
-`;
-}

packages/shared/src/server/queries/index.ts

@@ -1,8 +1,5 @@
 export { createSessionsAllQuery } from "./createSessionsAllQuery";
-export { createTracesQuery, parseTraceAllFilters } from "./createTracesQuery";
 export {
-  createGenerationsQuery,
-  parseGetAllGenerationsInput,
   type FullObservations,
   type FullObservationsWithScores,
   type IOAndMetadataOmittedObservations,

packages/shared/src/server/queues.ts

@@ -1,45 +1,16 @@
 import { z } from "zod";
-import { eventTypes, ingestionBatchEvent } from ".";
-
-export const LegacyIngestionEventFull = z.object({
-  useS3EventStore: z.literal(false),
-  data: ingestionBatchEvent,
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEventMeta = z.object({
-  useS3EventStore: z.literal(true),
-  data: z.array(
-    z.object({
-      type: z.nativeEnum(eventTypes),
-      eventBodyId: z.string(),
-      eventId: z.string(),
-    }),
-  ),
-  authCheck: z.object({
-    validKey: z.literal(true),
-    scope: z.object({
-      projectId: z.string(),
-      accessLevel: z.enum(["all", "scores"]),
-    }),
-  }),
-});
-
-export const LegacyIngestionEvent = z.discriminatedUnion("useS3EventStore", [
-  LegacyIngestionEventFull,
-  LegacyIngestionEventMeta,
-]);
+import { eventTypes } from ".";
+import {
+  BatchActionQuerySchema,
+  BatchActionType,
+} from "../features/batchAction/types";
+import { BatchExportTableName } from "../features/batchExport/types";
 
 export const IngestionEvent = z.object({
   data: z.object({
     type: z.nativeEnum(eventTypes),
     eventBodyId: z.string(),
+    fileKey: z.string().optional(),
   }),
   authCheck: z.object({
     validKey: z.literal(true),
@@ -86,6 +57,19 @@ export const ExperimentCreateEventSchema = z.object({
   runId: z.string(),
   description: z.string().optional(),
 });
+export const DataRetentionProcessingEventSchema = z.object({
+  projectId: z.string(),
+  retention: z.number(),
+});
+export const BatchActionProcessingEventSchema = z.object({
+  projectId: z.string(),
+  actionId: z.string(),
+  query: BatchActionQuerySchema,
+  tableName: z.nativeEnum(BatchExportTableName),
+  cutoffCreatedAt: z.date(),
+  targetId: z.string().optional(),
+  type: z.nativeEnum(BatchActionType),
+});
 
 export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
@@ -95,7 +79,6 @@ export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
 >;
 export type EvalExecutionEventType = z.infer<typeof EvalExecutionEvent>;
-export type LegacyIngestionEventType = z.infer<typeof LegacyIngestionEvent>;
 export type IngestionEventQueueType = z.infer<typeof IngestionEvent>;
 export type ExperimentCreateEventType = z.infer<
   typeof ExperimentCreateEventSchema
@@ -103,6 +86,12 @@ export type ExperimentCreateEventType = z.infer<
 export type PostHogIntegrationProcessingEventType = z.infer<
   typeof PostHogIntegrationProcessingEventSchema
 >;
+export type DataRetentionProcessingEventType = z.infer<
+  typeof DataRetentionProcessingEventSchema
+>;
+export type BatchActionProcessingEventType = z.infer<
+  typeof BatchActionProcessingEventSchema
+>;
 
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
@@ -112,11 +101,16 @@ export enum QueueName {
   DatasetRunItemUpsert = "dataset-run-item-upsert-queue",
   BatchExport = "batch-export-queue",
   IngestionQueue = "ingestion-queue", // Process single events with S3-merge
-  LegacyIngestionQueue = "legacy-ingestion-queue", // Used for batch processing of Ingestion
+  IngestionSecondaryQueue = "secondary-ingestion-queue", // Separates high priority + high throughput projects from other projects.
   CloudUsageMeteringQueue = "cloud-usage-metering-queue",
   ExperimentCreate = "experiment-create-queue",
   PostHogIntegrationQueue = "posthog-integration-queue",
   PostHogIntegrationProcessingQueue = "posthog-integration-processing-queue",
+  CoreDataS3ExportQueue = "core-data-s3-export-queue",
+  MeteringDataPostgresExportQueue = "metering-data-postgres-export-queue",
+  DataRetentionQueue = "data-retention-queue",
+  DataRetentionProcessingQueue = "data-retention-processing-queue",
+  BatchActionQueue = "batch-action-queue",
 }
 
 export enum QueueJobs {
@@ -126,12 +120,17 @@ export enum QueueJobs {
   DatasetRunItemUpsert = "dataset-run-item-upsert",
   EvaluationExecution = "evaluation-execution-job",
   BatchExportJob = "batch-export-job",
-  LegacyIngestionJob = "legacy-ingestion-job",
   CloudUsageMeteringJob = "cloud-usage-metering-job",
   IngestionJob = "ingestion-job",
+  IngestionSecondaryJob = "secondary-ingestion-job",
   ExperimentCreateJob = "experiment-create-job",
   PostHogIntegrationJob = "posthog-integration-job",
   PostHogIntegrationProcessingJob = "posthog-integration-processing-job",
+  CoreDataS3ExportJob = "core-data-s3-export-job",
+  MeteringDataPostgresExportJob = "metering-data-postgres-export-job",
+  DataRetentionJob = "data-retention-job",
+  DataRetentionProcessingJob = "data-retention-processing-job",
+  BatchActionProcessingJob = "batch-action-processing-job",
 }
 
 export type TQueueJobTypes = {
@@ -171,13 +170,13 @@ export type TQueueJobTypes = {
     payload: BatchExportJobType;
     name: QueueJobs.BatchExportJob;
   };
-  [QueueName.LegacyIngestionQueue]: {
+  [QueueName.IngestionQueue]: {
     timestamp: Date;
     id: string;
-    payload: LegacyIngestionEventType;
-    name: QueueJobs.LegacyIngestionJob;
+    payload: IngestionEventQueueType;
+    name: QueueJobs.IngestionJob;
   };
-  [QueueName.IngestionQueue]: {
+  [QueueName.IngestionSecondaryQueue]: {
     timestamp: Date;
     id: string;
     payload: IngestionEventQueueType;
@@ -195,4 +194,16 @@ export type TQueueJobTypes = {
     payload: PostHogIntegrationProcessingEventType;
     name: QueueJobs.PostHogIntegrationProcessingJob;
   };
+  [QueueName.DataRetentionProcessingQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: DataRetentionProcessingEventType;
+    name: QueueJobs.DataRetentionProcessingJob;
+  };
+  [QueueName.BatchActionQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: BatchActionProcessingEventType;
+    name: QueueJobs.BatchActionProcessingJob;
+  };
 };

packages/shared/src/server/redis/batchActionQueue.ts

@@ -3,30 +3,30 @@ import { QueueName, TQueueJobTypes } from "../queues";
 import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
 import { logger } from "../logger";
 
-export class LegacyIngestionQueue {
+export class BatchActionQueue {
   private static instance: Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.BatchActionQueue]
   > | null = null;
 
   public static getInstance(): Queue<
-    TQueueJobTypes[QueueName.LegacyIngestionQueue]
+    TQueueJobTypes[QueueName.BatchActionQueue]
   > | null {
-    if (LegacyIngestionQueue.instance) return LegacyIngestionQueue.instance;
+    if (BatchActionQueue.instance) return BatchActionQueue.instance;
 
     const newRedis = createNewRedisInstance({
       enableOfflineQueue: false,
       ...redisQueueRetryOptions,
     });
 
-    LegacyIngestionQueue.instance = newRedis
-      ? new Queue<TQueueJobTypes[QueueName.LegacyIngestionQueue]>(
-          QueueName.LegacyIngestionQueue,
+    BatchActionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.BatchActionQueue]>(
+          QueueName.BatchActionQueue,
           {
             connection: newRedis,
             defaultJobOptions: {
               removeOnComplete: true,
-              removeOnFail: 500_000,
-              attempts: 5,
+              removeOnFail: 10_000,
+              attempts: 2,
               backoff: {
                 type: "exponential",
                 delay: 5000,
@@ -36,10 +36,10 @@ export class LegacyIngestionQueue {
         )
       : null;
 
-    LegacyIngestionQueue.instance?.on("error", (err) => {
-      logger.error("LegacyIngestionQueue error", err);
+    BatchActionQueue.instance?.on("error", (err) => {
+      logger.error("BatchActionQueue error", err);
     });
 
-    return LegacyIngestionQueue.instance;
+    return BatchActionQueue.instance;
   }
 }

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -45,6 +45,7 @@ export class CloudUsageMeteringQueue {
         QueueJobs.CloudUsageMeteringJob,
         {},
         {
+          // Run at minute 5 of every hour (e.g. 1:05, 2:05, 3:05, etc)
           repeat: { pattern: "5 * * * *" },
         },
       );

packages/shared/src/server/redis/coreDataS3ExportQueue.ts

@@ -0,0 +1,60 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class CoreDataS3ExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_S3_CORE_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (CoreDataS3ExportQueue.instance) {
+      return CoreDataS3ExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    CoreDataS3ExportQueue.instance = newRedis
+      ? new Queue(QueueName.CoreDataS3ExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    CoreDataS3ExportQueue.instance?.on("error", (err) => {
+      logger.error("CoreDataS3ExportQueue error", err);
+    });
+
+    if (CoreDataS3ExportQueue.instance) {
+      logger.debug("Scheduling jobs for CoreDataS3ExportQueue");
+      CoreDataS3ExportQueue.instance
+        .add(
+          QueueJobs.CoreDataS3ExportJob,
+          {},
+          {
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding CoreDataS3ExportJob schedule", err);
+        });
+    }
+
+    return CoreDataS3ExportQueue.instance;
+  }
+}

packages/shared/src/server/redis/dataRetentionProcessingQueue.ts

@@ -0,0 +1,40 @@
+import { Queue } from "bullmq";
+import { QueueName } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionProcessingQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionProcessingQueue.instance) {
+      return DataRetentionProcessingQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionProcessingQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionProcessingQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 10000,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionProcessingQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionProcessingQueue error", err);
+    });
+
+    return DataRetentionProcessingQueue.instance;
+  }
+}

packages/shared/src/server/redis/dataRetentionQueue.ts

@@ -0,0 +1,55 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+
+export class DataRetentionQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (DataRetentionQueue.instance) {
+      return DataRetentionQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DataRetentionQueue.instance = newRedis
+      ? new Queue(QueueName.DataRetentionQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    DataRetentionQueue.instance?.on("error", (err) => {
+      logger.error("DataRetentionQueue error", err);
+    });
+
+    if (DataRetentionQueue.instance) {
+      logger.debug("Scheduling jobs for DataRetentionQueue");
+      DataRetentionQueue.instance
+        .add(
+          QueueJobs.DataRetentionJob,
+          {},
+          {
+            repeat: { pattern: "15 3 * * *" }, // every day at 3:15am
+          },
+        )
+        .catch((err) => {
+          logger.error("Error adding DataRetentionQueue schedule", err);
+        });
+    }
+
+    return DataRetentionQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -5,18 +5,20 @@ import { CloudUsageMeteringQueue } from "./cloudUsageMeteringQueue";
 import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
-import { IngestionQueue } from "./ingestionQueue";
-import { LegacyIngestionQueue } from "./legacyIngestion";
+import { IngestionQueue, SecondaryIngestionQueue } from "./ingestionQueue";
 import { TraceUpsertQueue } from "./traceUpsert";
 import { TraceDeleteQueue } from "./traceDelete";
 import { ProjectDeleteQueue } from "./projectDelete";
 import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
 import { PostHogIntegrationProcessingQueue } from "./postHogIntegrationProcessingQueue";
+import { CoreDataS3ExportQueue } from "./coreDataS3ExportQueue";
+import { MeteringDataPostgresExportQueue } from "./meteringDataPostgresExportQueue";
+import { DataRetentionQueue } from "./dataRetentionQueue";
+import { DataRetentionProcessingQueue } from "./dataRetentionProcessingQueue";
+import { BatchActionQueue } from "./batchActionQueue";
 
 export function getQueue(queueName: QueueName): Queue | null {
   switch (queueName) {
-    case QueueName.LegacyIngestionQueue:
-      return LegacyIngestionQueue.getInstance();
     case QueueName.BatchExport:
       return BatchExportQueue.getInstance();
     case QueueName.CloudUsageMeteringQueue:
@@ -39,6 +41,18 @@ export function getQueue(queueName: QueueName): Queue | null {
       return PostHogIntegrationQueue.getInstance();
     case QueueName.PostHogIntegrationProcessingQueue:
       return PostHogIntegrationProcessingQueue.getInstance();
+    case QueueName.IngestionSecondaryQueue:
+      return SecondaryIngestionQueue.getInstance();
+    case QueueName.CoreDataS3ExportQueue:
+      return CoreDataS3ExportQueue.getInstance();
+    case QueueName.MeteringDataPostgresExportQueue:
+      return MeteringDataPostgresExportQueue.getInstance();
+    case QueueName.DataRetentionQueue:
+      return DataRetentionQueue.getInstance();
+    case QueueName.DataRetentionProcessingQueue:
+      return DataRetentionProcessingQueue.getInstance();
+    case QueueName.BatchActionQueue:
+      return BatchActionQueue.getInstance();
     default:
       const exhaustiveCheckDefault: never = queueName;
       throw new Error(`Queue ${queueName} not found`);

packages/shared/src/server/redis/ingestionQueue.ts

@@ -43,3 +43,45 @@ export class IngestionQueue {
     return IngestionQueue.instance;
   }
 }
+
+export class SecondaryIngestionQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.IngestionSecondaryQueue]
+  > | null {
+    if (SecondaryIngestionQueue.instance)
+      return SecondaryIngestionQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    SecondaryIngestionQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.IngestionSecondaryQueue]>(
+          QueueName.IngestionSecondaryQueue,
+          {
+            connection: newRedis,
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    SecondaryIngestionQueue.instance?.on("error", (err) => {
+      logger.error("SecondaryIngestionQueue error", err);
+    });
+
+    return SecondaryIngestionQueue.instance;
+  }
+}

packages/shared/src/server/redis/meteringDataPostgresExportQueue.ts

@@ -0,0 +1,63 @@
+import { Queue } from "bullmq";
+import { QueueName, QueueJobs } from "../queues";
+import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
+import { env } from "../../env";
+
+export class MeteringDataPostgresExportQueue {
+  private static instance: Queue | null = null;
+
+  public static getInstance(): Queue | null {
+    if (env.LANGFUSE_POSTGRES_METERING_DATA_EXPORT_IS_ENABLED !== "true") {
+      return null;
+    }
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      return MeteringDataPostgresExportQueue.instance;
+    }
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    MeteringDataPostgresExportQueue.instance = newRedis
+      ? new Queue(QueueName.MeteringDataPostgresExportQueue, {
+          connection: newRedis,
+          defaultJobOptions: {
+            removeOnComplete: true,
+            removeOnFail: 100,
+            attempts: 5,
+            backoff: {
+              type: "exponential",
+              delay: 5000,
+            },
+          },
+        })
+      : null;
+
+    MeteringDataPostgresExportQueue.instance?.on("error", (err) => {
+      logger.error("MeteringDataPostgresExportQueue error", err);
+    });
+
+    if (MeteringDataPostgresExportQueue.instance) {
+      logger.debug("Scheduling jobs for MeteringDataPostgresExportQueue");
+      MeteringDataPostgresExportQueue.instance
+        .add(
+          QueueJobs.MeteringDataPostgresExportJob,
+          {},
+          {
+            repeat: { pattern: "30 2 * * *" }, // every day at 2:30am UTC
+          },
+        )
+        .catch((err) => {
+          logger.error(
+            "Error adding MeteringDataPostgresExportJob schedule",
+            err,
+          );
+        });
+    }
+
+    return MeteringDataPostgresExportQueue.instance;
+  }
+}

packages/shared/src/server/repositories/clickhouse.ts

@@ -4,30 +4,12 @@ import {
   convertDateToClickhouseDateTime,
 } from "../clickhouse/client";
 import { logger } from "../logger";
-import { instrumentAsync } from "../instrumentation";
-import {
-  StorageService,
-  StorageServiceFactory,
-} from "../services/StorageService";
+import { getTracer, instrumentAsync } from "../instrumentation";
 import { randomUUID } from "crypto";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config";
-
-let s3StorageServiceClient: StorageService;
-
-const getS3StorageServiceClient = (bucketName: string): StorageService => {
-  if (!s3StorageServiceClient) {
-    s3StorageServiceClient = StorageServiceFactory.getInstance({
-      bucketName,
-      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
-      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
-      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
-      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
-      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
-    });
-  }
-  return s3StorageServiceClient;
-};
+import { context, trace } from "@opentelemetry/api";
+import { uploadEventToS3 } from "../utils/eventLog";
 
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
@@ -40,9 +22,6 @@ export async function upsertClickhouse<
     // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
     span.setAttribute("ch.query.table", opts.table);
 
-    const s3Client = getS3StorageServiceClient(
-      env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
-    );
     await Promise.all(
       opts.records.map((record) => {
         // drop trailing s and pretend it's always a create.
@@ -52,11 +31,19 @@ export async function upsertClickhouse<
           // @ts-ignore - If it's an observation we now that `type` is a string
           eventType = `${record["type"].toLowerCase()}-create`;
         }
-        s3Client.uploadJson(
-          `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${record.project_id}/${getClickhouseEntityType(eventType)}/${record.id}/${randomUUID()}.json`,
+
+        const eventId = randomUUID();
+        return uploadEventToS3(
+          {
+            projectId: record.project_id as string,
+            entityType: getClickhouseEntityType(eventType),
+            entityId: record.id as string,
+            eventId,
+            traceId: record?.trace_id as string,
+          },
           [
             {
-              id: randomUUID(),
+              id: eventId,
               timestamp: new Date().toISOString(),
               type: eventType,
               body: opts.eventBodyMapper(record),
@@ -101,6 +88,64 @@ export async function upsertClickhouse<
   });
 }
 
+export async function* queryClickhouseStream<T>(opts: {
+  query: string;
+  params?: Record<string, unknown> | undefined;
+  clickhouseConfigs?: NodeClickHouseClientConfigOptions;
+}): AsyncGenerator<T> {
+  const tracer = getTracer("clickhouse-query-stream");
+  const span = tracer.startSpan("clickhouse-query-stream");
+
+  try {
+    const res = await context.with(
+      trace.setSpan(context.active(), span),
+      async () => {
+        // https://opentelemetry.io/docs/specs/semconv/database/database-spans/
+        span.setAttribute("ch.query.text", opts.query);
+
+        const res = await clickhouseClient(opts.clickhouseConfigs).query({
+          query: opts.query,
+          format: "JSONEachRow",
+          query_params: opts.params,
+        });
+        // same logic as for prisma. we want to see queries in development
+        if (env.NODE_ENV === "development") {
+          logger.info(`clickhouse:query ${res.query_id} ${opts.query}`);
+        }
+
+        span.setAttribute("ch.queryId", res.query_id);
+
+        // add summary headers to the span. Helps to tune performance
+        const summaryHeader = res.response_headers["x-clickhouse-summary"];
+        if (summaryHeader) {
+          try {
+            const summary = Array.isArray(summaryHeader)
+              ? JSON.parse(summaryHeader[0])
+              : JSON.parse(summaryHeader);
+            for (const key in summary) {
+              span.setAttribute(`ch.${key}`, summary[key]);
+            }
+          } catch (error) {
+            logger.debug(
+              `Failed to parse clickhouse summary header ${summaryHeader}`,
+              error,
+            );
+          }
+        }
+        return res;
+      },
+    );
+
+    for await (const rows of res.stream<T>()) {
+      for (const row of rows) {
+        yield row.json();
+      }
+    }
+  } finally {
+    span.end();
+  }
+}
+
 export async function queryClickhouse<T>(opts: {
   query: string;
   params?: Record<string, unknown> | undefined;

packages/shared/src/server/repositories/dashboards.ts

@@ -59,7 +59,6 @@ export const getObservationsCostGroupedByName = async (
   const appliedFilter = chFilter.apply();
 
   const hasTraceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
-  // TODO: Validate whether we can filter traces on timestamp here.
 
   const query = `
     SELECT 
@@ -68,9 +67,10 @@ export const getObservationsCostGroupedByName = async (
       sumMap(usage_details)['total'] as sum_usage_details
     FROM observations o FINAL ${hasTraceFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
-    AND ${appliedFilter.query}
+    ${appliedFilter.query ? `AND ${appliedFilter.query}` : ""}
     GROUP BY provided_model_name
     ORDER BY sumMap(cost_details)['total'] DESC
+    LIMIT 50
     `;
 
   const result = await queryClickhouse<{
@@ -145,23 +145,26 @@ export const getScoreAggregate = async (
 export const groupTracesByTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
   ).apply();
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
   const query = `
     SELECT 
-      ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
       count(*) as count
     FROM traces t FINAL
     WHERE project_id = {projectId: String}
     AND ${chFilter.query}
     GROUP BY timestamp
-    ${orderByTimeSeries(groupBy, "timestamp")}
+    ${orderByQuery}
     `;
-
   const result = await queryClickhouse<{
     timestamp: string;
     count: string;
@@ -170,11 +173,12 @@ export const groupTracesByTime = async (
     params: {
       projectId,
       ...chFilter.params,
+      ...orderByParams,
     },
   });
 
   return result.map((row) => ({
-    timestamp: new Date(row.timestamp),
+    timestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
     countTraceId: Number(row.count),
   }));
 };
@@ -182,7 +186,6 @@ export const groupTracesByTime = async (
 export const getObservationUsageByTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -200,9 +203,14 @@ export const getObservationUsageByTime = async (
       ) as DateTimeFilter | undefined)
     : undefined;
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time",
+  );
+
   const query = `
     SELECT 
-      ${selectTimeseriesColumn(groupBy, "start_time", "start_time")},
+      ${selectTimeseriesColumn(bucketSizeInSeconds, "start_time", "start_time")},
       sumMap(usage_details) as units,
       sumMap(cost_details) as cost, 
       provided_model_name
@@ -212,7 +220,7 @@ export const getObservationUsageByTime = async (
     AND ${appliedFilter.query}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
     GROUP BY start_time, provided_model_name
-    ${orderByTimeSeries(groupBy, "start_time")}
+    ${orderByQuery}
     `;
 
   const result = await queryClickhouse<{
@@ -225,6 +233,7 @@ export const getObservationUsageByTime = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...orderByParams,
       ...(timeFilter
         ? { traceTimestamp: convertDateToClickhouseDateTime(timeFilter.value) }
         : {}),
@@ -232,7 +241,7 @@ export const getObservationUsageByTime = async (
   });
 
   return result.map((row) => ({
-    start_time: new Date(row.start_time),
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time),
     units: Object.fromEntries(
       Object.entries(row.units ?? {}).map(([key, value]) => [
         key,
@@ -271,12 +280,15 @@ export const getDistinctModels = async (
 
   // No need for final as duplicates are caught by distinct anyway.
   const query = `
-    SELECT distinct(provided_model_name) as model
+    SELECT distinct(provided_model_name) as model, count(*) as count
     FROM observations o
     ${tracesFilter ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
     WHERE project_id = {projectId: String}
     AND ${appliedFilter.query}
     ${timeFilter ? `AND t.timestamp >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+    GROUP BY provided_model_name
+    ORDER BY count(*) DESC
+    LIMIT 1000
     `;
 
   const result = await queryClickhouse<{ model: string }>({
@@ -296,7 +308,6 @@ export const getDistinctModels = async (
 export const getScoresAggregateOverTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -307,9 +318,14 @@ export const getScoresAggregateOverTime = async (
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
   // TODO: Validate whether we can filter traces on timestamp here.
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "timestamp",
+  );
+
   const query = `
   SELECT 
-    ${selectTimeseriesColumn(groupBy, "timestamp", "timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "timestamp", "timestamp")},
     name,
     data_type,
     source,
@@ -324,7 +340,7 @@ export const getScoresAggregateOverTime = async (
     name,
     data_type,
     source
-  ${orderByTimeSeries(groupBy, "timestamp")};
+  ${orderByQuery};
 `;
 
   const result = await queryClickhouse<{
@@ -338,11 +354,12 @@ export const getScoresAggregateOverTime = async (
     params: {
       projectId,
       ...appliedFilter.params,
+      ...orderByParams,
     },
   });
 
   return result.map((row) => ({
-    scoreTimestamp: new Date(row.timestamp),
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.timestamp),
     scoreName: row.name,
     scoreDataType: row.data_type,
     scoreSource: row.source,
@@ -418,7 +435,7 @@ export const getObservationLatencies = async (
   // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
     SELECT
-      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles,
+      quantiles(0.5, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles,
       name
     FROM observations o
     ${chFilter.find((f) => f.clickhouseTable === "traces") ? "LEFT JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
@@ -465,7 +482,7 @@ export const getTracesLatencies = async (
       select o.trace_id,
              t.name,
              o.project_id,
-             date_diff('milliseconds', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
+             date_diff('millisecond', min(o.start_time), coalesce(max(o.end_time), max(o.start_time))) as duration
       FROM traces t 
       JOIN observations o
       ON o.trace_id = t.id AND o.project_id = t.project_id
@@ -506,7 +523,6 @@ export const getTracesLatencies = async (
 export const getModelLatenciesOverTime = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -516,25 +532,33 @@ export const getModelLatenciesOverTime = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
   // Skipping FINAL here, as the quantiles are approximate to begin with.
   const query = `
   SELECT 
-    ${selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket")},
     provided_model_name,
-    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('milliseconds', o.start_time, o.end_time)) as quantiles
+    quantiles(0.5, 0.75, 0.9, 0.95, 0.99)(date_diff('millisecond', o.start_time, o.end_time)) as quantiles
   FROM observations o
   ${traceFilter ? "JOIN traces t ON o.trace_id = t.id AND o.project_id = t.project_id" : ""}
   WHERE project_id = {projectId: String}
   AND ${appliedFilter.query}
   GROUP BY provided_model_name, start_time_bucket
-  ${orderByTimeSeries(groupBy, "start_time_bucket")};
+  ${orderByQuery};
 `;
 
   const result = await queryClickhouse<{
     start_time_bucket: string;
     provided_model_name: string;
     quantiles: string[];
-  }>({ query, params: { projectId, ...appliedFilter.params } });
+  }>({
+    query,
+    params: { projectId, ...appliedFilter.params, ...orderByParams },
+  });
 
   return result.map((row) => ({
     p50: Number(row.quantiles[0]) / 1000,
@@ -543,14 +567,13 @@ export const getModelLatenciesOverTime = async (
     p95: Number(row.quantiles[3]) / 1000,
     p99: Number(row.quantiles[4]) / 1000,
     model: row.provided_model_name,
-    start_time: new Date(row.start_time_bucket),
+    start_time: parseClickhouseUTCDateTimeFormat(row.start_time_bucket),
   }));
 };
 
 export const getNumericScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -559,9 +582,14 @@ export const getNumericScoreTimeSeries = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
   const query = `
     SELECT
-    ${selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp")},
+    ${selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp")},
     s.name as score_name,
     AVG(s.value) as avg_value
     FROM scores s final
@@ -569,11 +597,11 @@ export const getNumericScoreTimeSeries = async (
     WHERE s.project_id = {projectId: String}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
     GROUP BY score_name, score_timestamp
-    ${orderByTimeSeries(groupBy, "score_timestamp")}
+    ${orderByQuery}
   `;
 
-  return queryClickhouse<{
-    score_timestamp: Date;
+  const result = await queryClickhouse<{
+    score_timestamp: string;
     score_name: string;
     avg_value: number;
   }>({
@@ -581,14 +609,20 @@ export const getNumericScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
     },
   });
+
+  return result.map((row) => ({
+    scoreTimestamp: parseClickhouseUTCDateTimeFormat(row.score_timestamp),
+    scoreName: row.score_name,
+    avgValue: Number(row.avg_value),
+  }));
 };
 
 export const getCategoricalScoreTimeSeries = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc | undefined,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -597,9 +631,14 @@ export const getCategoricalScoreTimeSeries = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "score_timestamp",
+  );
+
   const query = `
     SELECT
-    ${groupBy ? selectTimeseriesColumn(groupBy, "s.timestamp", "score_timestamp") + ", " : ""}
+    ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "s.timestamp", "score_timestamp") + ", " : ""}
     s.name as score_name,
     s.data_type as score_data_type,
     s.source as score_source,
@@ -609,12 +648,12 @@ export const getCategoricalScoreTimeSeries = async (
     ${traceFilter ? "JOIN traces t ON s.trace_id = t.id AND s.project_id = t.project_id" : ""}
     WHERE s.project_id = {projectId: String}
     ${chFilterRes?.query ? `AND ${chFilterRes.query}` : ""}
-    GROUP BY score_name, score_data_type, score_source, score_value ${groupBy ? ", score_timestamp" : ""}
-    ${groupBy ? orderByTimeSeries(groupBy, "score_timestamp") : ""}
+    GROUP BY score_name, score_data_type, score_source, score_value ${bucketSizeInSeconds ? ", score_timestamp" : ""}
+      ${orderByQuery}
   `;
 
-  return queryClickhouse<{
-    score_timestamp?: Date;
+  const result = await queryClickhouse<{
+    score_timestamp?: string;
     score_name: string;
     score_data_type: string;
     score_source: string;
@@ -625,14 +664,25 @@ export const getCategoricalScoreTimeSeries = async (
     params: {
       projectId,
       ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
     },
   });
+
+  return result.map((row) => ({
+    scoreTimestamp: row.score_timestamp
+      ? parseClickhouseUTCDateTimeFormat(row.score_timestamp)
+      : undefined,
+    scoreName: row.score_name,
+    scoreDataType: row.score_data_type,
+    scoreSource: row.score_source,
+    scoreValue: row.score_value,
+    count: Number(row.count),
+  }));
 };
 
 export const getObservationsStatusTimeSeries = async (
   projectId: string,
   filter: FilterState,
-  groupBy: DateTrunc | undefined,
 ) => {
   const chFilter = new FilterList(
     createFilterFromFilterState(filter, dashboardColumnDefinitions),
@@ -641,9 +691,14 @@ export const getObservationsStatusTimeSeries = async (
 
   const traceFilter = chFilter.find((f) => f.clickhouseTable === "traces");
 
+  const [orderByQuery, orderByParams, bucketSizeInSeconds] = orderByTimeSeries(
+    filter,
+    "start_time_bucket",
+  );
+
   const query = `
     SELECT 
-      ${groupBy ? selectTimeseriesColumn(groupBy, "o.start_time", "start_time_bucket") + ", " : ""}
+      ${bucketSizeInSeconds ? selectTimeseriesColumn(bucketSizeInSeconds, "o.start_time", "start_time_bucket") + ", " : ""}
       count(*) as observation_count,
       level as level
     FROM observations o
@@ -651,8 +706,8 @@ export const getObservationsStatusTimeSeries = async (
     WHERE project_id = {projectId: String}
     AND o.level IS NOT NULL
     AND ${chFilterRes?.query}
-    GROUP BY level ${groupBy ? ", start_time_bucket" : ""}
-    ${groupBy ? orderByTimeSeries(groupBy, "start_time_bucket") : ""}
+    GROUP BY level ${bucketSizeInSeconds ? ", start_time_bucket" : ""}
+    ${orderByQuery}
   `;
 
   const result = await queryClickhouse<{
@@ -661,7 +716,11 @@ export const getObservationsStatusTimeSeries = async (
     level: string;
   }>({
     query,
-    params: { projectId, ...(chFilterRes ? chFilterRes.params : {}) },
+    params: {
+      projectId,
+      ...(chFilterRes ? chFilterRes.params : {}),
+      ...orderByParams,
+    },
   });
 
   return result.map((row) => ({
@@ -673,61 +732,70 @@ export const getObservationsStatusTimeSeries = async (
   }));
 };
 
-const orderByTimeSeries = (dateTrunc: DateTrunc, col: string) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toIntervalYear(1)";
-      break;
-    case "month":
-      interval = "toIntervalMonth(1)";
-      break;
-    case "week":
-      interval = "toIntervalWeek(1)";
-      break;
-    case "day":
-      interval = "toIntervalDay(1)";
-      break;
-    case "hour":
-      interval = "toIntervalHour(1)";
-      break;
-    case "minute":
-      interval = "toIntervalMinute(1)";
-      break;
-    default:
-      return undefined;
+export const orderByTimeSeries = (
+  filter: FilterState,
+  col: string,
+): [string, { fromTime: number; toTime: number }, number] => {
+  const potentialBucketSizesSeconds = [
+    5, 10, 30, 60, 300, 600, 1800, 3600, 18000, 36000, 86400, 604800, 2592000,
+  ];
+
+  // Calculate time difference in seconds
+  const [from, to] = extractFromAndToTimestampsFromFilter(filter);
+
+  if (!from || !to) {
+    throw new Error("Time Filter is required for time series queries");
   }
 
-  return `ORDER BY ${col} ASC WITH FILL STEP ${interval}`;
+  const fromDate = new Date(from.value as Date);
+  const toDate = new Date(to.value as Date);
+
+  const diffInSeconds = Math.abs(toDate.getTime() - fromDate.getTime()) / 1000;
+
+  // choose the bucket size that is the closest to the desired number of buckets
+  const bucketSizeInSeconds = potentialBucketSizesSeconds.reduce(
+    (closest, size) => {
+      const diffFromDesiredBuckets = Math.abs(diffInSeconds / size - 50);
+      return diffFromDesiredBuckets < closest.diffFromDesiredBuckets
+        ? { size, diffFromDesiredBuckets }
+        : closest;
+    },
+    { size: 0, diffFromDesiredBuckets: Infinity },
+  ).size;
+
+  // Convert to interval string
+  const interval = `toIntervalSecond(${bucketSizeInSeconds})`;
+
+  return [
+    `ORDER BY ${col} ASC 
+    WITH FILL
+    FROM toStartOfInterval(toDateTime({fromTime: DateTime64(3)}), INTERVAL ${bucketSizeInSeconds} SECOND)
+    TO toDateTime({toTime: DateTime64(3)}) + INTERVAL ${bucketSizeInSeconds} SECOND
+    STEP ${interval}`,
+    { fromTime: fromDate.getTime(), toTime: toDate.getTime() },
+    bucketSizeInSeconds,
+  ];
 };
 
-const selectTimeseriesColumn = (
-  dateTrunc: DateTrunc,
+export const selectTimeseriesColumn = (
+  bucketSizeInSeconds: number,
   col: string,
   as: String,
 ) => {
-  let interval;
-  switch (dateTrunc) {
-    case "year":
-      interval = "toStartOfYear";
-      break;
-    case "month":
-      interval = "toStartOfMonth";
-      break;
-    case "week":
-      interval = "toStartOfWeek";
-      break;
-    case "day":
-      interval = "toStartOfDay";
-      break;
-    case "hour":
-      interval = "toStartOfHour";
-      break;
-    case "minute":
-      interval = "toStartOfMinute";
-      break;
-    default:
-      return undefined;
-  }
-  return `${interval}(${col}) as ${as}`;
+  return `toStartOfInterval(${col}, INTERVAL ${bucketSizeInSeconds} SECOND) as ${as}`;
+};
+
+export const extractFromAndToTimestampsFromFilter = (filter?: FilterState) => {
+  if (!filter)
+    throw new Error("Time Filter is required for time series queries");
+
+  const fromTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === ">" || f.operator === ">="),
+  );
+
+  const toTimestamp = filter.filter(
+    (f) => f.type === "datetime" && (f.operator === "<" || f.operator === "<="),
+  );
+
+  return [fromTimestamp[0], toTimestamp[0]];
 };

packages/shared/src/server/repositories/observations.ts

@@ -2,6 +2,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  queryClickhouseStream,
   upsertClickhouse,
 } from "./clickhouse";
 import { ObservationLevel } from "@prisma/client";
@@ -414,7 +415,7 @@ export const getObservationsTable = async (
   const observationRecords = await getObservationsTableInternal<
     Omit<
       ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
     >
   >({
     ...opts,
@@ -431,7 +432,7 @@ export const getObservationsTable = async (
   return observationRecords.map((o) => {
     const trace = traces.find((t) => t.id === o.trace_id);
     return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
       latency: o.latency ? Number(o.latency) / 1000 : null,
       timeToFirstToken: o.time_to_first_token
         ? Number(o.time_to_first_token) / 1000
@@ -449,7 +450,7 @@ export const getObservationsTableWithModelData = async (
   const observationRecords = await getObservationsTableInternal<
     Omit<
       ObservationsTableQueryResult,
-      "trace_tags" | "trace_name" | "trace_user_id" | "type"
+      "trace_tags" | "trace_name" | "trace_user_id"
     >
   >({
     ...opts,
@@ -490,7 +491,7 @@ export const getObservationsTableWithModelData = async (
     const trace = traces.find((t) => t.id === o.trace_id);
     const model = models.find((m) => m.id === o.internal_model_id);
     return {
-      ...convertObservationToView({ ...o, type: "GENERATION" }),
+      ...convertObservationToView(o),
       latency: o.latency ? Number(o.latency) / 1000 : null,
       timeToFirstToken: o.time_to_first_token
         ? Number(o.time_to_first_token) / 1000
@@ -517,6 +518,7 @@ const getObservationsTableInternal = async <T>(
       ? "count(*) as count"
       : `
         o.id as id,
+        o.type as type,
         o.project_id as "project_id",
         o.name as name,
         o."model_parameters" as model_parameters,
@@ -536,9 +538,12 @@ const getObservationsTableInternal = async <T>(
         o.updated_at as "updated_at",
         o.provided_model_name as "provided_model_name",
         o.total_cost as "total_cost",
+        o.prompt_id as "prompt_id",
+        o.prompt_name as "prompt_name",
+        o.prompt_version as "prompt_version",
         internal_model_id as "internal_model_id",
-        if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time)) as latency,
-        if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time)) as "time_to_first_token"`;
+        if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time)) as latency,
+        if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time)) as "time_to_first_token"`;
 
   const { projectId, filter, selectIOAndMetadata, limit, offset, orderBy } =
     opts;
@@ -677,7 +682,7 @@ const getObservationsTableInternal = async <T>(
         ${traceTableFilter.length > 0 || orderByTraces || search.query ? "LEFT JOIN traces t FINAL ON t.id = o.trace_id AND t.project_id = o.project_id" : ""}
         ${hasScoresFilter ? `LEFT JOIN scores_avg AS s_avg ON s_avg.trace_id = o.trace_id and s_avg.observation_id = o.id` : ""}
       WHERE ${appliedObservationsFilter.query}
-        AND o.type = 'GENERATION'
+        
         ${timeFilter && (traceTableFilter.length > 0 || orderByTraces) ? `AND t.timestamp > {tracesTimestampFilter: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
         ${search.query}
       ${chOrderBy}
@@ -747,6 +752,50 @@ export const getObservationsGroupedByModel = async (
   return res.map((r) => ({ model: r.name }));
 };
 
+export const getObservationsGroupedByModelId = async (
+  projectId: string,
+  filter: FilterState,
+) => {
+  const observationsFilter = new FilterList([
+    new StringFilter({
+      clickhouseTable: "observations",
+      field: "project_id",
+      operator: "=",
+      value: projectId,
+      tablePrefix: "o",
+    }),
+  ]);
+
+  observationsFilter.push(
+    ...createFilterFromFilterState(
+      filter,
+      observationsTableUiColumnDefinitions,
+    ),
+  );
+
+  const appliedObservationsFilter = observationsFilter.apply();
+
+  // We mainly use queries like this to retrieve filter options.
+  // Therefore, we can skip final as some inaccuracy in count is acceptable.
+  const query = `
+    SELECT o.internal_model_id as modelId
+    FROM observations o
+    WHERE ${appliedObservationsFilter.query}
+    AND o.type = 'GENERATION'
+    GROUP BY o.internal_model_id
+    ORDER BY count() DESC
+    LIMIT 1000;
+  `;
+
+  const res = await queryClickhouse<{ modelId: string }>({
+    query,
+    params: {
+      ...appliedObservationsFilter.params,
+    },
+  });
+  return res.map((r) => ({ modelId: r.modelId }));
+};
+
 export const getObservationsGroupedByName = async (
   projectId: string,
   filter: FilterState,
@@ -859,6 +908,7 @@ export const getObservationsGroupedByPromptName = async (
 
 export const getCostForTraces = async (
   projectId: string,
+  timestamp: Date,
   traceIds: string[],
 ) => {
   // Wrapping the query in a CTE allows us to skip FINAL which allows Clickhouse to use skip indexes.
@@ -868,6 +918,7 @@ export const getCostForTraces = async (
       FROM observations o
       WHERE o.project_id = {projectId: String}
       AND o.trace_id IN ({traceIds: Array(String)})
+      AND o.start_time >= {timestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}
       ORDER BY o.event_ts DESC
       LIMIT 1 BY o.id, o.project_id
     )
@@ -881,6 +932,7 @@ export const getCostForTraces = async (
     params: {
       projectId,
       traceIds,
+      timestamp: convertDateToClickhouseDateTime(timestamp),
     },
   });
   return res.length > 0 ? Number(res[0].total_cost) : undefined;
@@ -923,6 +975,27 @@ export const deleteObservationsByProjectId = async (projectId: string) => {
   });
 };
 
+export const deleteObservationsOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM observations
+    WHERE project_id = {projectId: String}
+    AND start_time < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const getObservationsWithPromptName = async (
   projectId: string,
   promptNames: string[],
@@ -963,7 +1036,7 @@ export const getObservationMetricsForPrompts = async (
                   end_time,
                   usage_details,
                   cost_details,
-                  dateDiff('milliseconds', start_time, end_time) AS latency_ms
+                  dateDiff('millisecond', start_time, end_time) AS latency_ms
               FROM observations
               FINAL
               WHERE (type = 'GENERATION') 
@@ -1026,7 +1099,7 @@ export const getLatencyAndTotalCostForObservations = async (
     SELECT
         id,
         cost_details['total'] AS total_cost,
-        dateDiff('milliseconds', start_time, end_time) AS latency_ms
+        dateDiff('millisecond', start_time, end_time) AS latency_ms
     FROM observations FINAL 
     WHERE project_id = {projectId: String} 
     AND id IN ({observationIds: Array(String)})
@@ -1058,7 +1131,7 @@ export const getLatencyAndTotalCostForObservationsByTraces = async (
     SELECT
         trace_id,
         sumMap(cost_details)['total'] AS total_cost,
-        dateDiff('milliseconds', min(start_time), max(end_time)) AS latency_ms
+        dateDiff('millisecond', min(start_time), max(end_time)) AS latency_ms
     FROM observations FINAL
     WHERE project_id = {projectId: String} 
     AND trace_id IN ({traceIds: Array(String)})
@@ -1167,23 +1240,23 @@ export const getTraceIdsForObservations = async (
   }));
 };
 
-export const getGenerationsForPostHog = async (
+export const getGenerationsForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
   maxTimestamp: Date,
-) => {
+) {
   const query = `
     SELECT
       o.name as name,
       o.start_time as start_time,
       o.id as id,
       o.total_cost as total_cost,
-      if(isNull(completion_start_time), NULL, date_diff('milliseconds', start_time, completion_start_time)) as time_to_first_token,
+      if(isNull(completion_start_time), NULL, date_diff('millisecond', start_time, completion_start_time)) as time_to_first_token,
       o.usage_details['total'] as input_tokens,
       o.usage_details['output'] as output_tokens,
       o.cost_details['total'] as total_tokens,
       o.project_id as project_id,
-      if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time) / 1000) as latency,
+      if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000) as latency,
       o.provided_model_name as model,
       o.level as level,
       o.version as version,
@@ -1205,7 +1278,7 @@ export const getGenerationsForPostHog = async (
     AND o.type = 'GENERATION'
   `;
 
-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
     query,
     params: {
       projectId,
@@ -1215,32 +1288,34 @@ export const getGenerationsForPostHog = async (
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.start_time,
-    langfuse_generation_name: record.name,
-    langfuse_trace_name: record.trace_name,
-    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
-    langfuse_id: record.id,
-    langfuse_cost_usd: record.total_cost,
-    langfuse_input_units: record.input_tokens,
-    langfuse_output_units: record.output_tokens,
-    langfuse_total_units: record.total_tokens,
-    langfuse_session_id: record.trace_session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
-    langfuse_latency: record.latency,
-    langfuse_time_to_first_token: record.time_to_first_token,
-    langfuse_release: record.trace_release,
-    langfuse_version: record.version,
-    langfuse_model: record.model,
-    langfuse_level: record.level,
-    langfuse_tags: record.trace_tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.start_time,
+      langfuse_generation_name: record.name,
+      langfuse_trace_name: record.trace_name,
+      langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.trace_id as string)}?observation=${encodeURIComponent(record.id as string)}`,
+      langfuse_id: record.id,
+      langfuse_cost_usd: record.total_cost,
+      langfuse_input_units: record.input_tokens,
+      langfuse_output_units: record.output_tokens,
+      langfuse_total_units: record.total_tokens,
+      langfuse_session_id: record.trace_session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+      langfuse_latency: record.latency,
+      langfuse_time_to_first_token: record.time_to_first_token,
+      langfuse_release: record.trace_release,
+      langfuse_version: record.version,
+      langfuse_model: record.model,
+      langfuse_level: record.level,
+      langfuse_tags: record.trace_tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };

packages/shared/src/server/repositories/observations_converters.ts

@@ -9,7 +9,6 @@ import Decimal from "decimal.js";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { ObservationRecordReadType } from "./definitions";
 import { parseJsonPrioritised } from "../../utils/json";
-import { jsonSchema } from "../../utils/zod";
 
 export const convertObservationToView = (
   record: ObservationRecordReadType,
@@ -62,15 +61,22 @@ export const convertObservation = (
       ? parseClickhouseUTCDateTimeFormat(record.end_time)
       : null,
     name: record.name ?? null,
-    metadata: record.metadata,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
     level: record.level as ObservationLevel,
     statusMessage: record.status_message ?? null,
     version: record.version ?? null,
     input: (record.input
-      ? jsonSchema.parse(parseJsonPrioritised(record.input))
+      ? parseJsonPrioritised(record.input)
       : null) as Prisma.JsonValue | null,
     output: (record.output
-      ? jsonSchema.parse(parseJsonPrioritised(record.output))
+      ? parseJsonPrioritised(record.output)
       : null) as Prisma.JsonValue | null,
     modelParameters: record.model_parameters
       ? JSON.parse(record.model_parameters)
@@ -126,10 +132,11 @@ export const convertObservation = (
         parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
       : null,
     timeToFirstToken: record.completion_start_time
-      ? parseClickhouseUTCDateTimeFormat(
+      ? (parseClickhouseUTCDateTimeFormat(
           record.completion_start_time,
         ).getTime() -
-        parseClickhouseUTCDateTimeFormat(record.start_time).getTime()
+          parseClickhouseUTCDateTimeFormat(record.start_time).getTime()) /
+        1000
       : null,
   };
 };

packages/shared/src/server/repositories/scores.ts

@@ -3,6 +3,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  queryClickhouseStream,
   upsertClickhouse,
 } from "./clickhouse";
 import { FilterList } from "../queries/clickhouse-sql/clickhouse-filter";
@@ -135,13 +136,16 @@ export const upsertScore = async (score: Partial<ScoreRecordReadType>) => {
   });
 };
 
-export const getScoresForTraces = async (
-  projectId: string,
-  traceIds: string[],
-  timestamp?: Date,
-  limit?: number,
-  offset?: number,
-) => {
+export type GetScoresForTracesProps = {
+  projectId: string;
+  traceIds: string[];
+  timestamp?: Date;
+  limit?: number;
+  offset?: number;
+};
+
+export const getScoresForTraces = async (props: GetScoresForTracesProps) => {
+  const { projectId, traceIds, timestamp, limit, offset } = props;
   const query = `
       select 
         *
@@ -200,20 +204,24 @@ export const getScoresForObservations = async (
   return rows.map(convertToScore);
 };
 
-export const getScoresGroupedByNameSourceType = async (projectId: string) => {
+export const getScoresGroupedByNameSourceType = async (
+  projectId: string,
+  timestamp: Date | undefined,
+) => {
   // We mainly use queries like this to retrieve filter options.
   // Therefore, we can skip final as some inaccuracy in count is acceptable.
   const query = `
-      select 
-        name,
-        source,
-        data_type
-      from scores s
-      WHERE s.project_id = {projectId: String}
-      GROUP BY name, source, data_type
-      ORDER BY count() desc
-      LIMIT 1000;
-    `;
+    select 
+      name,
+      source,
+      data_type
+    from scores s
+    WHERE s.project_id = {projectId: String}
+    ${timestamp ? `AND s.timestamp >= {timestamp: DateTime64(3)}` : ""}
+    GROUP BY name, source, data_type
+    ORDER BY count() desc
+    LIMIT 1000;
+  `;
 
   const rows = await queryClickhouse<{
     name: string;
@@ -223,6 +231,9 @@ export const getScoresGroupedByNameSourceType = async (projectId: string) => {
     query: query,
     params: {
       projectId: projectId,
+      ...(timestamp
+        ? { timestamp: convertDateToClickhouseDateTime(timestamp) }
+        : {}),
     },
   });
 
@@ -525,6 +536,27 @@ export const deleteScoresByProjectId = async (projectId: string) => {
   });
 };
 
+export const deleteScoresOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM scores
+    WHERE project_id = {projectId: String}
+    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const getNumericScoreHistogram = async (
   projectId: string,
   filter: FilterState,
@@ -666,11 +698,11 @@ export const getDistinctScoreNames = async (
   return rows.map((row) => row.name);
 };
 
-export const getScoresForPostHog = async (
+export const getScoresForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
   maxTimestamp: Date,
-) => {
+) {
   const query = `
     SELECT
       s.id as id,
@@ -694,7 +726,7 @@ export const getScoresForPostHog = async (
     AND t.timestamp <= {maxTimestamp: DateTime64(3)}
   `;
 
-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
     query,
     params: {
       projectId,
@@ -704,24 +736,26 @@ export const getScoresForPostHog = async (
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.timestamp,
-    langfuse_score_name: record.name,
-    langfuse_score_value: record.value,
-    langfuse_score_comment: record.comment,
-    langfuse_trace_name: record.trace_name,
-    langfuse_id: record.id,
-    langfuse_session_id: record.trace_session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
-    langfuse_release: record.trace_release,
-    langfuse_tags: record.trace_tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.timestamp,
+      langfuse_score_name: record.name,
+      langfuse_score_value: record.value,
+      langfuse_score_comment: record.comment,
+      langfuse_trace_name: record.trace_name,
+      langfuse_id: record.id,
+      langfuse_session_id: record.trace_session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.trace_user_id || "langfuse_unknown_user",
+      langfuse_release: record.trace_release,
+      langfuse_tags: record.trace_tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };

packages/shared/src/server/repositories/traces.ts

@@ -2,6 +2,7 @@ import {
   commandClickhouse,
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
+  queryClickhouseStream,
   upsertClickhouse,
 } from "./clickhouse";
 import {
@@ -16,10 +17,7 @@ import {
 } from "../queries/clickhouse-sql/clickhouse-filter";
 import { TraceRecordReadType } from "./definitions";
 import { tracesTableUiColumnDefinitions } from "../../tableDefinitions/mapTracesTable";
-import { OrderByState } from "../../interfaces/orderBy";
-import { orderByToClickhouseSql } from "../queries/clickhouse-sql/orderby-factory";
 import { UiColumnMapping } from "../../tableDefinitions";
-import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
 import { convertClickhouseToDomain } from "./traces_converters";
 import { clickhouseSearchCondition } from "../queries/clickhouse-sql/search";
@@ -32,7 +30,7 @@ import { env } from "../../env";
 export const checkTraceExists = async (
   projectId: string,
   traceId: string,
-  timestamp: Date | undefined,
+  timestamp: Date,
   filter: FilterState,
 ): Promise<boolean> => {
   const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
@@ -63,18 +61,22 @@ export const checkTraceExists = async (
   const query = `
     WITH observations_agg AS (
         SELECT
-          
             multiIf(
               arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
               arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
               arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
               'DEBUG'
-            ) AS level,
+            ) AS aggregated_level,
+            countIf(level = 'ERROR') as error_count,
+            countIf(level = 'WARNING') as warning_count,
+            countIf(level = 'DEFAULT') as default_count,
+            countIf(level = 'DEBUG') as debug_count,
             trace_id,
             project_id
         FROM observations o FINAL 
         WHERE o.project_id = {projectId: String}
         ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        AND o.start_time >= {timestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}
         GROUP BY trace_id, project_id
       )
     SELECT 
@@ -84,7 +86,7 @@ export const checkTraceExists = async (
     ${observationFilterRes ? `INNER JOIN observations_agg o ON t.id = o.trace_id AND t.project_id = o.project_id` : ""}
     WHERE ${tracesFilterRes.query}
     AND t.project_id = {projectId: String}
-    ${timestamp ? `AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+    AND timestamp >= {timestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
     GROUP BY t.id, t.project_id
   `;
 
@@ -383,196 +385,6 @@ export const getTracesGroupedByTags = async (props: GroupedTracesQueryProp) => {
   return rows;
 };
 
-export type SessionDataReturnType = {
-  session_id: string;
-  max_timestamp: string;
-  min_timestamp: string;
-  trace_ids: string[];
-  user_ids: string[];
-  trace_count: number;
-  trace_tags: string[];
-  total_observations: number;
-  duration: number;
-  session_usage_details: Record<string, number>;
-  session_cost_details: Record<string, number>;
-  session_input_cost: string;
-  session_output_cost: string;
-  session_total_cost: string;
-  session_input_usage: string;
-  session_output_usage: string;
-  session_total_usage: string;
-};
-
-export const getSessionsTableCount = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<{ count: string }>({
-    select: `
-      count(session_id) as count
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows.length > 0 ? Number(rows[0].count) : 0;
-};
-
-export const getSessionsTable = async (props: {
-  projectId: string;
-  filter: FilterState;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-}) => {
-  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
-    select: `
-    session_id, 
-    max_timestamp, 
-    min_timestamp, 
-    trace_ids, 
-    user_ids, 
-    trace_count, 
-    trace_tags,
-    total_observations,
-    duration,
-    session_usage_details,
-    session_cost_details,
-    session_input_cost,
-    session_output_cost,
-    session_total_cost,
-    session_input_usage,
-    session_output_usage,
-    session_total_usage
-    `,
-    projectId: props.projectId,
-    filter: props.filter,
-    orderBy: props.orderBy,
-    limit: props.limit,
-    page: props.page,
-  });
-
-  return rows;
-};
-
-export type FetchSessionsTableProps = {
-  select: string;
-  projectId: string;
-  filter: FilterState;
-  searchQuery?: string;
-  orderBy?: OrderByState;
-  limit?: number;
-  page?: number;
-};
-
-const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
-  const { select, projectId, filter, orderBy, limit, page } = props;
-
-  const { tracesFilter, scoresFilter, observationsFilter } =
-    getProjectIdDefaultFilter(projectId, { tracesPrefix: "s" });
-
-  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
-
-  const tracesFilterRes = tracesFilter.apply();
-  const scoresAvgFilterRes = scoresFilter.apply();
-  const observationsStatsRes = observationsFilter.apply();
-
-  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
-    (f) =>
-      f.field === "min_timestamp" &&
-      (f.operator === ">=" || f.operator === ">"),
-  ) as DateTimeFilter | undefined;
-
-  const singleTraceFilter = traceTimestampFilter
-    ? new FilterList([
-        new DateTimeFilter({
-          clickhouseTable: "traces",
-          field: "timestamp",
-          operator: traceTimestampFilter.operator,
-          value: traceTimestampFilter.value,
-        }),
-      ]).apply()
-    : undefined;
-
-  const query = `
-      WITH observations_agg AS (
-        SELECT o.trace_id,
-              count(*) as obs_count,
-              min(o.start_time) as min_start_time,
-              max(o.end_time) as max_end_time,
-              sumMap(usage_details) as sum_usage_details,
-              sumMap(cost_details) as sum_cost_details,
-              anyLast(project_id) as project_id
-        FROM observations o FINAL
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            -- Aggregate observations data at session level
-            sum(o.obs_count) as total_observations,
-            date_diff('milliseconds', min(min_start_time), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage
-        FROM traces t FINAL
-        LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id
-        WHERE t.session_id IS NOT NULL
-            AND t.project_id = {projectId: String}
-            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${select}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
-
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...observationsStatsRes.params,
-      ...scoresAvgFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
-    },
-  });
-
-  return res;
-};
-
 export const getTracesIdentifierForSession = async (
   projectId: string,
   sessionId: string,
@@ -630,6 +442,27 @@ export const deleteTraces = async (projectId: string, traceIds: string[]) => {
   });
 };
 
+export const deleteTracesOlderThanDays = async (
+  projectId: string,
+  days: number,
+) => {
+  const query = `
+    DELETE FROM traces
+    WHERE project_id = {projectId: String}
+    AND timestamp < now() - INTERVAL {numDays: Int} DAYS;
+  `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+      numDays: days,
+    },
+    clickhouseConfigs: {
+      request_timeout: 120_000, // 2 minutes
+    },
+  });
+};
+
 export const deleteTracesByProjectId = async (projectId: string) => {
   const query = `
     DELETE FROM traces
@@ -680,10 +513,25 @@ export const getTotalUserCount = async (
   });
 };
 
-export const getUserMetrics = async (projectId: string, userIds: string[]) => {
+export const getUserMetrics = async (
+  projectId: string,
+  userIds: string[],
+  filter: FilterState,
+) => {
   if (userIds.length === 0) {
     return [];
   }
+
+  // filter state contains date range filter for traces so far.
+  const chFilter = new FilterList(
+    createFilterFromFilterState(filter, tracesTableUiColumnDefinitions),
+  );
+  const chFilterRes = chFilter.apply();
+
+  const timestampFilter = chFilter.find(
+    (f) => f.field === "timestamp" && f.operator === ">=",
+  );
+
   // this query uses window functions on observations + traces to always get only the first row and thereby remove deduplicates
   // we filter wherever possible by project id and user id
   const query = `
@@ -713,6 +561,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                     observations o
                 WHERE
                     o.project_id = {projectId: String }
+                    ${timestampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
                     AND o.trace_id in (
                         SELECT
                             distinct id
@@ -721,6 +570,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                         where
                             user_id IN ({userIds: Array(String) })
                             AND project_id = {projectId: String }
+                            ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
                     )
                     AND o.type = 'GENERATION'
             ) as o
@@ -740,6 +590,7 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
                 WHERE
                     t.user_id IN ({userIds: Array(String) })
                     AND t.project_id = {projectId: String }
+                    ${filter.length > 0 ? `AND ${chFilterRes.query}` : ""}
             ) as t on t.id = o.trace_id
             and t.project_id = o.project_id
         WHERE
@@ -778,8 +629,17 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
     params: {
       projectId,
       userIds,
+      ...chFilterRes.params,
+      ...(timestampFilter
+        ? {
+            traceTimestamp: convertDateToClickhouseDateTime(
+              (timestampFilter as DateTimeFilter).value,
+            ),
+          }
+        : {}),
     },
   });
+
   return rows.map((row) => ({
     userId: row.user_id,
     maxTimestamp: parseClickhouseUTCDateTimeFormat(row.max_timestamp),
@@ -793,18 +653,18 @@ export const getUserMetrics = async (projectId: string, userIds: string[]) => {
   }));
 };
 
-export const getTracesForPostHog = async (
+export const getTracesForPostHog = async function* (
   projectId: string,
   minTimestamp: Date,
   maxTimestamp: Date,
-) => {
+) {
   const query = `
     WITH observations_agg AS (
       SELECT o.project_id,
              o.trace_id,
              sum(total_cost) as total_cost,
              count(*) as observation_count,
-             date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds
+             date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds
       FROM observations o FINAL
       WHERE o.project_id = {projectId: String}
       AND o.start_time >= {minTimestamp: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}
@@ -831,7 +691,7 @@ export const getTracesForPostHog = async (
     AND t.timestamp <= {maxTimestamp: DateTime64(3)}
   `;
 
-  const records = await queryClickhouse<Record<string, unknown>>({
+  const records = queryClickhouseStream<Record<string, unknown>>({
     query,
     params: {
       projectId,
@@ -841,28 +701,30 @@ export const getTracesForPostHog = async (
   });
 
   const baseUrl = env.NEXTAUTH_URL?.replace("/api/auth", "");
-  return records.map((record) => ({
-    timestamp: record.timestamp,
-    langfuse_id: record.id,
-    langfuse_trace_name: record.name,
-    langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.id as string)}`,
-    langfuse_cost_usd: record.total_cost,
-    langfuse_count_observations: record.observation_count,
-    langfuse_session_id: record.session_id,
-    langfuse_project_id: projectId,
-    langfuse_user_id: record.user_id || "langfuse_unknown_user",
-    langfuse_latency: record.latency,
-    langfuse_release: record.release,
-    langfuse_version: record.version,
-    langfuse_tags: record.tags,
-    langfuse_event_version: "1.0.0",
-    $session_id: record.posthog_session_id ?? null,
-    $set: {
-      langfuse_user_url: record.user_id
-        ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
-        : null,
-    },
-  }));
+  for await (const record of records) {
+    yield {
+      timestamp: record.timestamp,
+      langfuse_id: record.id,
+      langfuse_trace_name: record.name,
+      langfuse_url: `${baseUrl}/project/${projectId}/traces/${encodeURIComponent(record.id as string)}`,
+      langfuse_cost_usd: record.total_cost,
+      langfuse_count_observations: record.observation_count,
+      langfuse_session_id: record.session_id,
+      langfuse_project_id: projectId,
+      langfuse_user_id: record.user_id || "langfuse_unknown_user",
+      langfuse_latency: record.latency,
+      langfuse_release: record.release,
+      langfuse_version: record.version,
+      langfuse_tags: record.tags,
+      langfuse_event_version: "1.0.0",
+      $session_id: record.posthog_session_id ?? null,
+      $set: {
+        langfuse_user_url: record.user_id
+          ? `${baseUrl}/project/${projectId}/users/${encodeURIComponent(record.user_id as string)}`
+          : null,
+      },
+    };
+  }
 };
 
 export const getTracesByIdsForAnyProject = async (traceIds: string[]) => {

packages/shared/src/server/repositories/traces_converters.ts

@@ -1,7 +1,8 @@
-import { Trace } from "@prisma/client";
+import { Prisma, Trace } from "@prisma/client";
 import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
 import { TraceRecordReadType } from "./definitions";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { parseJsonPrioritised } from "../../utils/json";
 
 export const convertTraceDomainToClickhouse = (
   trace: Trace,
@@ -43,9 +44,20 @@ export const convertClickhouseToDomain = (
     userId: record.user_id ?? null,
     sessionId: record.session_id ?? null,
     public: record.public,
-    input: record.input ?? null,
-    output: record.output ?? null,
-    metadata: record.metadata,
+    input: (record.input
+      ? parseJsonPrioritised(record.input)
+      : null) as Prisma.JsonValue | null,
+    output: (record.output
+      ? parseJsonPrioritised(record.output)
+      : null) as Prisma.JsonValue | null,
+    metadata:
+      record.metadata &&
+      Object.fromEntries(
+        Object.entries(record.metadata ?? {}).map(([key, val]) => [
+          key,
+          val && parseJsonPrioritised(val),
+        ]),
+      ),
     createdAt: parseClickhouseUTCDateTimeFormat(record.created_at),
     updatedAt: parseClickhouseUTCDateTimeFormat(record.updated_at),
     externalId: null,

packages/shared/src/server/services/StorageService.ts

@@ -1,5 +1,6 @@
 import { Readable } from "stream";
 import {
+  DeleteObjectsCommand,
   GetObjectCommand,
   ListObjectsV2Command,
   PutObjectCommand,
@@ -30,7 +31,7 @@ export interface StorageService {
 
   download(path: string): Promise<string>;
 
-  listFiles(prefix: string): Promise<string[]>;
+  listFiles(prefix: string): Promise<{ file: string; createdAt: Date }[]>;
 
   getSignedUrl(
     fileName: string,
@@ -45,6 +46,8 @@ export interface StorageService {
     contentType: string;
     contentLength: number;
   }): Promise<string>;
+
+  deleteFiles(paths: string[]): Promise<void>;
 }
 
 export class StorageServiceFactory {
@@ -201,7 +204,28 @@ class AzureBlobStorageService implements StorageService {
     }
   }
 
-  public async listFiles(prefix: string): Promise<string[]> {
+  public async deleteFiles(paths: string[]): Promise<void> {
+    try {
+      await this.createContainerIfNotExists();
+
+      await Promise.all(
+        paths.map(async (path) => {
+          const blobClient = this.client.getBlobClient(path);
+          await blobClient.deleteIfExists();
+        }),
+      );
+    } catch (err) {
+      logger.error(
+        `Failed to delete files from Azure Blob Storage ${paths}`,
+        err,
+      );
+      throw Error("Failed to delete files from Azure Blob Storage");
+    }
+  }
+
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
     try {
       await this.createContainerIfNotExists();
 
@@ -209,7 +233,10 @@ class AzureBlobStorageService implements StorageService {
       const files = [];
       for await (const blob of result) {
         if (blob.name.startsWith(prefix)) {
-          files.push(blob.name);
+          files.push({
+            file: blob.name,
+            createdAt: blob?.properties?.createdOn ?? new Date(),
+          });
         }
       }
       return files;
@@ -364,7 +391,9 @@ class S3StorageService implements StorageService {
     }
   }
 
-  public async listFiles(prefix: string): Promise<string[]> {
+  public async listFiles(
+    prefix: string,
+  ): Promise<{ file: string; createdAt: Date }[]> {
     const listCommand = new ListObjectsV2Command({
       Bucket: this.bucketName,
       Prefix: prefix,
@@ -373,7 +402,11 @@ class S3StorageService implements StorageService {
     try {
       const response = await this.client.send(listCommand);
       return (
-        response.Contents?.flatMap((file) => (file.Key ? [file.Key] : [])) ?? []
+        response.Contents?.flatMap((file) =>
+          file.Key
+            ? [{ file: file.Key, createdAt: file.LastModified ?? new Date() }]
+            : [],
+        ) ?? []
       );
     } catch (err) {
       logger.error(`Failed to list files from S3 ${prefix}`, err);
@@ -404,6 +437,37 @@ class S3StorageService implements StorageService {
     }
   }
 
+  public async deleteFiles(paths: string[]): Promise<void> {
+    const chunkSize = 900;
+    const chunks = [];
+
+    for (let i = 0; i < paths.length; i += chunkSize) {
+      chunks.push(paths.slice(i, i + chunkSize));
+    }
+
+    try {
+      for (const chunk of chunks) {
+        const command = new DeleteObjectsCommand({
+          Bucket: this.bucketName,
+          Delete: {
+            Objects: chunk.map((path) => ({ Key: path })),
+            Quiet: true,
+          },
+        });
+        const result = await this.client.send(command);
+        if (result?.Errors && result?.Errors?.length > 0) {
+          logger.error("Failed to delete files from S3", {
+            errors: result.Errors,
+          });
+          throw new Error("Failed to delete files from S3");
+        }
+      }
+    } catch (err) {
+      logger.error(`Failed to delete files from S3`, err);
+      throw new Error("Failed to delete files from S3");
+    }
+  }
+
   public async getSignedUploadUrl(params: {
     path: string;
     ttlSeconds: number;

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -0,0 +1,319 @@
+import { OrderByState } from "../../interfaces/orderBy";
+import { sessionCols } from "../../tableDefinitions/mapSessionTable";
+import { FilterState } from "../../types";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
+import {
+  getProjectIdDefaultFilter,
+  createFilterFromFilterState,
+} from "../queries/clickhouse-sql/factory";
+import {
+  TRACE_TO_OBSERVATIONS_INTERVAL,
+  queryClickhouse,
+} from "../repositories";
+
+export type SessionDataReturnType = {
+  session_id: string;
+  max_timestamp: string;
+  min_timestamp: string;
+  trace_ids: string[];
+  user_ids: string[];
+  trace_count: number;
+  trace_tags: string[];
+};
+
+export type SessionWithMetricsReturnType = SessionDataReturnType & {
+  total_observations: number;
+  duration: number;
+  session_usage_details: Record<string, number>;
+  session_cost_details: Record<string, number>;
+  session_input_cost: string;
+  session_output_cost: string;
+  session_total_cost: string;
+  session_input_usage: string;
+  session_output_usage: string;
+  session_total_usage: string;
+};
+
+export const getSessionsTableCount = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<{ count: string }>({
+    select: "count",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.length > 0 ? Number(rows[0].count) : 0;
+};
+
+export const getSessionsTable = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionDataReturnType>({
+    select: "rows",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+  }));
+};
+
+export const getSessionsWithMetrics = async (props: {
+  projectId: string;
+  filter: FilterState;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+}) => {
+  const rows = await getSessionsTableGeneric<SessionWithMetricsReturnType>({
+    select: "metrics",
+    projectId: props.projectId,
+    filter: props.filter,
+    orderBy: props.orderBy,
+    limit: props.limit,
+    page: props.page,
+  });
+
+  return rows.map((row) => ({
+    ...row,
+    trace_count: Number(row.trace_count),
+    total_observations: Number(row.total_observations),
+  }));
+};
+
+export type FetchSessionsTableProps = {
+  select: "count" | "rows" | "metrics";
+  projectId: string;
+  filter: FilterState;
+  searchQuery?: string;
+  orderBy?: OrderByState;
+  limit?: number;
+  page?: number;
+};
+
+const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
+  const { select, projectId, filter, orderBy, limit, page } = props;
+
+  let sqlSelect: string;
+  switch (select) {
+    case "count":
+      sqlSelect = "count(session_id) as count";
+      break;
+    case "rows":
+      sqlSelect = `
+          session_id, 
+          max_timestamp, 
+          min_timestamp, 
+          trace_ids, 
+          user_ids, 
+          trace_count, 
+          trace_tags`;
+      break;
+    case "metrics":
+      sqlSelect = `
+        session_id, 
+        max_timestamp, 
+        min_timestamp, 
+        trace_ids, 
+        user_ids, 
+        trace_count, 
+        trace_tags,
+        total_observations,
+        duration,
+        session_usage_details,
+        session_cost_details,
+        session_input_cost,
+        session_output_cost,
+        session_total_cost,
+        session_input_usage,
+        session_output_usage,
+        session_total_usage`;
+      break;
+    default:
+      const exhaustiveCheckDefault: never = select;
+      throw new Error(`Unknown select type: ${select}`);
+  }
+
+  const { tracesFilter } = getProjectIdDefaultFilter(projectId, {
+    tracesPrefix: "s",
+  });
+
+  tracesFilter.push(...createFilterFromFilterState(filter, sessionCols));
+
+  const tracesFilterRes = tracesFilter.apply();
+
+  const traceTimestampFilter: DateTimeFilter | undefined = tracesFilter.find(
+    (f) =>
+      f.field === "min_timestamp" &&
+      (f.operator === ">=" || f.operator === ">"),
+  ) as DateTimeFilter | undefined;
+
+  const filters = [];
+  if (traceTimestampFilter) {
+    filters.push(
+      new DateTimeFilter({
+        clickhouseTable: "traces",
+        field: "timestamp",
+        operator: traceTimestampFilter.operator,
+        value: traceTimestampFilter.value,
+      }),
+    );
+  }
+
+  const additionalSingleTraceFilter = tracesFilter.find(
+    (f) => f.field === "bookmarked" || f.field === "session_id",
+  );
+
+  if (additionalSingleTraceFilter) {
+    filters.push(additionalSingleTraceFilter);
+  }
+
+  const singleTraceFilter =
+    filters.length > 0 ? new FilterList(filters).apply() : undefined;
+
+  const hasMetricsFilter =
+    tracesFilter.find((f) =>
+      [
+        "session_total_cost",
+        "session_input_cost",
+        "session_output_cost",
+        "duration",
+        "session_total_usage",
+        "session_output_usage",
+        "session_input_usage",
+      ].includes(f.field),
+    ) ||
+    (orderBy &&
+      [
+        "totalCost",
+        "inputCost",
+        "outputCost",
+        "sessionDuration",
+        "totalTokens",
+        "outputTokens",
+        "inputTokens",
+        "usage",
+      ].includes(orderBy?.column));
+
+  const selectMetrics = select === "metrics" || hasMetricsFilter;
+
+  // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
+  const query = `
+    WITH deduplicated_traces AS (
+      SELECT * EXCEPT input, output, metadata
+      FROM traces t
+      WHERE t.session_id IS NOT NULL 
+        AND t.project_id = {projectId: String}
+        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    deduplicated_observations AS (
+        SELECT * 
+        FROM observations o
+        WHERE o.project_id = {projectId: String}
+        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+        AND o.trace_id IN (
+          SELECT id
+          FROM deduplicated_traces
+        )
+        ORDER BY event_ts DESC
+        LIMIT 1 BY id, project_id
+    ),
+    observations_agg AS (
+      SELECT o.trace_id,
+            count(*) as obs_count,
+            min(o.start_time) as min_start_time,
+            max(o.end_time) as max_end_time,
+            sumMap(usage_details) as sum_usage_details,
+            sumMap(cost_details) as sum_cost_details,
+            anyLast(project_id) as project_id
+      FROM deduplicated_observations o
+      WHERE o.project_id = {projectId: String}
+      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+      GROUP BY o.trace_id
+    ),
+    session_data AS (
+        SELECT
+            t.session_id,
+            anyLast(t.project_id) as project_id,
+            max(t.timestamp) as max_timestamp,
+            min(t.timestamp) as min_timestamp,
+            groupArray(t.id) AS trace_ids,
+            groupUniqArray(t.user_id) AS user_ids,
+            count(*) as trace_count,
+            groupUniqArrayArray(t.tags) as trace_tags
+            -- Aggregate observations data at session level
+            ${
+              selectMetrics
+                ? `
+            ,
+            sum(o.obs_count) as total_observations,
+            date_diff('millisecond', min(min_start_time), max(max_end_time)) as duration,
+            sumMap(o.sum_usage_details) as session_usage_details,
+            sumMap(o.sum_cost_details) as session_cost_details,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+            sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                : ""
+            }
+        FROM deduplicated_traces t
+        ${
+          selectMetrics
+            ? `LEFT JOIN observations_agg o
+        ON t.id = o.trace_id AND t.project_id = o.project_id`
+            : ""
+        }
+        WHERE t.session_id IS NOT NULL
+            AND t.project_id = {projectId: String}
+            ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+        GROUP BY t.session_id
+    )
+    SELECT ${sqlSelect}
+    FROM session_data s
+    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+    `;
+
+  const obsStartTimeValue = traceTimestampFilter
+    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
+    : null;
+
+  const res = await queryClickhouse<T>({
+    query: query,
+    params: {
+      projectId,
+      limit: limit,
+      offset: limit && page ? limit * page : 0,
+      ...tracesFilterRes.params,
+      ...singleTraceFilter?.params,
+      ...(obsStartTimeValue
+        ? { observationsStartTime: obsStartTimeValue }
+        : {}),
+    },
+  });
+
+  return res;
+};

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -70,6 +70,10 @@ export type TracesMetricsUiReturnType = {
   scores: ScoreAggregate;
   usageDetails: Record<string, number>;
   costDetails: Record<string, number>;
+  errorCount: bigint;
+  warningCount: bigint;
+  defaultCount: bigint;
+  debugCount: bigint;
 };
 
 export const convertToUiTableRows = (
@@ -125,6 +129,10 @@ export const convertToUITableMetrics = (
       ? new Decimal(row.cost_details.output)
       : null,
     level: row.level,
+    debugCount: BigInt(row.debug_count ?? 0),
+    warningCount: BigInt(row.warning_count ?? 0),
+    errorCount: BigInt(row.error_count ?? 0),
+    defaultCount: BigInt(row.default_count ?? 0),
   };
 };
 
@@ -138,6 +146,10 @@ export type TracesTableMetricsClickhouseReturnType = {
   usage_details: Record<string, number>;
   cost_details: Record<string, number>;
   scores_avg: Array<{ name: string; avg_value: number }>;
+  error_count: number | null;
+  warning_count: number | null;
+  default_count: number | null;
+  debug_count: number | null;
 };
 
 export type FetchTracesTableProps = {
@@ -225,7 +237,11 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
         os.latency_milliseconds / 1000 as latency,
         os.cost_details as cost_details,
         os.usage_details as usage_details,
-        os.level as level,
+        os.aggregated_level as level,
+        os.error_count as error_count,
+        os.warning_count as warning_count,
+        os.default_count as default_count,
+        os.debug_count as debug_count,
         os.observation_count as observation_count,
         s.scores_avg as scores_avg,
         t.public as public`;
@@ -245,7 +261,6 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
         t.public as public`;
       break;
     default:
-      const exhaustiveCheckDefault: never = select;
       throw new Error(`Unknown select type: ${select}`);
   }
 
@@ -322,19 +337,29 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
       clickhouseSelect: "toDate(t.timestamp)",
       uiTableName: "timestamp_to_date",
       uiTableId: "timestamp_to_date",
-      clickhouseTableName: "observations",
+      clickhouseTableName: "traces",
+    },
+    {
+      clickhouseSelect: "t.event_ts",
+      uiTableName: "event_ts",
+      uiTableId: "event_ts",
+      clickhouseTableName: "traces",
     },
   ];
   const chOrderBy = orderByToClickhouseSql(
     [
       defaultOrder
-        ? {
-            column: "timestamp_to_date",
-            order: orderBy.order,
-          }
+        ? [
+            {
+              column: "timestamp_to_date",
+              order: orderBy.order,
+            },
+            { column: "timestamp", order: orderBy.order },
+            { column: "event_ts", order: "DESC" as "DESC" },
+          ]
         : null,
       orderBy ?? null,
-    ],
+    ].flat(),
     orderByCols,
   );
 
@@ -342,7 +367,7 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
   // - we only join scores and observations if we really need them to speed up default views
   // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
   // - we filter the observations and scores as much as possible before joining them to traces.
-  // - we order by todate(timestamp), timestamp per default and do not use FINAL.
+  // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
   //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
 
   const query = `
@@ -351,13 +376,17 @@ const getTracesTableGeneric = async <T>(props: FetchTracesTableProps) => {
         COUNT(*) AS observation_count,
           sumMap(usage_details) as usage_details,
           SUM(total_cost) AS total_cost,
-          date_diff('milliseconds', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+          countIf(level = 'ERROR') as error_count,
+          countIf(level = 'WARNING') as warning_count,
+          countIf(level = 'DEFAULT') as default_count,
+          countIf(level = 'DEBUG') as debug_count,
           multiIf(
             arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
             arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
             arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
             'DEBUG'
-          ) AS level,
+          ) AS aggregated_level,
           sumMap(cost_details) as cost_details,
           trace_id,
           project_id

packages/shared/src/server/utils/eventLog.ts

@@ -0,0 +1,52 @@
+import { EventLog } from "@prisma/client";
+import { prisma } from "../../db";
+import { logger } from "../logger";
+import { env } from "../../env";
+import {
+  StorageService,
+  StorageServiceFactory,
+} from "../services/StorageService";
+
+let s3StorageServiceClient: StorageService;
+
+const getS3StorageServiceClient = (bucketName: string): StorageService => {
+  if (!s3StorageServiceClient) {
+    s3StorageServiceClient = StorageServiceFactory.getInstance({
+      bucketName,
+      accessKeyId: env.LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID,
+      secretAccessKey: env.LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY,
+      endpoint: env.LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT,
+      region: env.LANGFUSE_S3_EVENT_UPLOAD_REGION,
+      forcePathStyle: env.LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE === "true",
+    });
+  }
+  return s3StorageServiceClient;
+};
+
+export const uploadEventToS3 = async (
+  event: Omit<
+    EventLog,
+    "createdAt" | "updatedAt" | "bucketPath" | "bucketName" | "id"
+  >,
+  data: Record<string, unknown>[],
+) => {
+  const bucketPath = `${env.LANGFUSE_S3_EVENT_UPLOAD_PREFIX}${event.projectId}/${event.entityType}/${event.entityId}/${event.eventId}.json`;
+  if (env.LANGFUSE_S3_EVENT_UPLOAD_POSTGRES_LOG_ENABLED === "true") {
+    try {
+      await prisma.eventLog.create({
+        data: {
+          ...event,
+          bucketPath,
+          bucketName: env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+        },
+      });
+    } catch (e) {
+      logger.error("Failed to write event log to Postgres", e);
+      // Fallthrough as this shouldn't block further execution right now.
+    }
+  }
+
+  return getS3StorageServiceClient(
+    env.LANGFUSE_S3_EVENT_UPLOAD_BUCKET,
+  ).uploadJson(bucketPath, data);
+};

packages/shared/src/tableDefinitions/mapObservationsTable.ts

@@ -26,6 +26,12 @@ export const observationsTableTraceUiColumnDefinitions: UiColumnMapping[] = [
 
 export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
   ...observationsTableTraceUiColumnDefinitions,
+  {
+    uiTableName: "type",
+    uiTableId: "type",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."type"',
+  },
   {
     uiTableName: "ID",
     uiTableId: "id",
@@ -68,7 +74,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "timeToFirstToken",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "if(isNull(completion_start_time), NULL,  date_diff('milliseconds', start_time, completion_start_time) / 1000)",
+      "if(isNull(completion_start_time), NULL,  date_diff('millisecond', start_time, completion_start_time) / 1000)",
     // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
@@ -77,7 +83,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "latency",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "if(isNull(end_time), NULL, date_diff('milliseconds', start_time, end_time) / 1000)",
+      "if(isNull(end_time), NULL, date_diff('millisecond', start_time, end_time) / 1000)",
     // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
     clickhouseTypeOverwrite: "Decimal64(3)",
   },
@@ -86,7 +92,7 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableId: "tokensPerSecond",
     clickhouseTableName: "observations",
     clickhouseSelect:
-      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('milliseconds', start_time, end_time) / 1000))",
+      "(arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, usage_details))) / (date_diff('millisecond', start_time, end_time) / 1000))",
   },
   {
     uiTableName: "Input Cost ($)",
@@ -127,6 +133,12 @@ export const observationsTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "observations",
     clickhouseSelect: 'o."provided_model_name"',
   },
+  {
+    uiTableName: "Model ID",
+    uiTableId: "modelId",
+    clickhouseTableName: "observations",
+    clickhouseSelect: 'o."internal_model_id"',
+  },
   {
     uiTableName: "Input Tokens",
     uiTableId: "inputTokens",

packages/shared/src/tableDefinitions/mapSessionTable.ts

@@ -1,7 +1,7 @@
 import { UiColumnMapping } from ".";
 
 export const sessionCols: UiColumnMapping[] = [
-  // we do not access the traces scores in clichouse. We default back to the trace timestamps.
+  // we do not access the traces scores in ClickHouse. We default back to the trace timestamps.
 
   {
     uiTableName: "⭐️",
@@ -26,6 +26,8 @@ export const sessionCols: UiColumnMapping[] = [
     uiTableId: "sessionDuration",
     clickhouseTableName: "traces",
     clickhouseSelect: "duration",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Count Traces",
@@ -86,6 +88,8 @@ export const sessionCols: UiColumnMapping[] = [
     uiTableId: "sessionDuration",
     clickhouseTableName: "traces",
     clickhouseSelect: "duration",
+    // If we use the default of Decimal64(12), we cannot filter for more than ~40min due to an overflow
+    clickhouseTypeOverwrite: "Decimal64(3)",
   },
   {
     uiTableName: "Traces Count",

packages/shared/src/tableDefinitions/mapTracesTable.ts

@@ -12,7 +12,7 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     uiTableName: "Level",
     uiTableId: "level",
     clickhouseTableName: "observations",
-    clickhouseSelect: "level",
+    clickhouseSelect: "aggregated_level",
   },
   {
     uiTableName: "ID",
@@ -68,6 +68,30 @@ export const tracesTableUiColumnDefinitions: UiColumnMapping[] = [
     clickhouseTableName: "traces",
     clickhouseSelect: "tags",
   },
+  {
+    uiTableName: "Warning Level Count",
+    uiTableId: "warningCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "warning_count",
+  },
+  {
+    uiTableName: "Error Level Count",
+    uiTableId: "errorCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "error_count",
+  },
+  {
+    uiTableName: "Default Level Count",
+    uiTableId: "defaultCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "default_count",
+  },
+  {
+    uiTableName: "Debug Level Count",
+    uiTableId: "debugCount",
+    clickhouseTableName: "observations",
+    clickhouseSelect: "debug_count",
+  },
   {
     uiTableName: "Input Tokens",
     uiTableId: "inputTokens",

packages/shared/src/tableDefinitions/tracesTable.ts

@@ -88,6 +88,30 @@ export const tracesTableCols: ColumnDefinition[] = [
     internal: 'generation_metrics."completionTokens"',
     nullable: true,
   },
+  {
+    name: "Error Level Count",
+    id: "errorCount",
+    type: "number",
+    internal: 'generation_metrics."errorCount"',
+  },
+  {
+    name: "Warning Level Count",
+    id: "warningCount",
+    type: "number",
+    internal: 'generation_metrics."warningCount"',
+  },
+  {
+    name: "Default Level Count",
+    id: "defaultCount",
+    type: "number",
+    internal: 'generation_metrics."defaultCount"',
+  },
+  {
+    name: "Debug Level Count",
+    id: "debugCount",
+    type: "number",
+    internal: 'generation_metrics."debugCount"',
+  },
   {
     name: "Total Tokens",
     id: "totalTokens",
@@ -102,7 +126,6 @@ export const tracesTableCols: ColumnDefinition[] = [
     internal: 'generation_metrics."totalTokens"',
     nullable: true,
   },
-
   {
     name: "Scores",
     id: "scores_avg",

packages/shared/src/utils/json.ts

@@ -1,6 +1,4 @@
-import { z } from "zod";
-import { merge } from "lodash";
-import { JsonNested, jsonSchema, jsonSchemaNullable } from "./zod";
+import { JsonNested } from "./zod";
 
 /**
  * Deeply parses a JSON string or object for nested stringified JSON
@@ -10,7 +8,6 @@ import { JsonNested, jsonSchema, jsonSchemaNullable } from "./zod";
 export function deepParseJson(json: unknown): unknown {
   if (typeof json === "string") {
     try {
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
       const parsed = JSON.parse(json);
       if (typeof parsed === "number") return json; // numbers that were strings in the input should remain as strings
       return deepParseJson(parsed); // Recursively parse parsed value
@@ -40,61 +37,12 @@ export function deepParseJson(json: unknown): unknown {
   return json;
 }
 
-export const mergeJson = (
-  json1?: z.infer<typeof jsonSchema>,
-  json2?: z.infer<typeof jsonSchema>,
-) => {
-  if (json1 === undefined) {
-    return json2;
-  }
-  return merge(json1, json2);
-};
-
 export const parseJsonPrioritised = (
   json: string,
-): z.infer<typeof jsonSchema> | string | undefined => {
+): JsonNested | string | undefined => {
   try {
-    const parsedJson = JSON.parse(json);
-
-    if (Object.keys(parsedJson).length === 0) {
-      return parsedJson;
-    }
-
-    const parsedArray = z.array(jsonSchemaNullable).safeParse(parsedJson);
-    if (parsedArray.success) {
-      return parsedArray.data;
-    }
-
-    const parsedObject = z.record(jsonSchemaNullable).safeParse(parsedJson);
-    if (parsedObject.success) {
-      return parsedObject.data;
-    }
-
-    return jsonSchema.parse(parsedJson);
+    return JSON.parse(json);
   } catch (error) {
-    const parsed = jsonSchema.safeParse(json);
-
-    return parsed.success ? parsed.data : json;
+    return json;
   }
 };
-
-export const convertRecordToJsonSchema = (
-  record: Record<string, string>,
-): JsonNested | undefined => {
-  const jsonSchema: JsonNested = {};
-
-  // if record is empty, return undefined
-  if (Object.keys(record).length === 0) {
-    return undefined;
-  }
-
-  for (const key in record) {
-    try {
-      jsonSchema[key] = JSON.parse(record[key]);
-    } catch (e) {
-      jsonSchema[key] = record[key];
-    }
-  }
-
-  return jsonSchema;
-};

packages/shared/src/utils/stringChecks.ts

@@ -4,22 +4,27 @@ export function getIsCharOrUnderscore(value: string): boolean {
   return charOrUnderscore.test(value);
 }
 
+// Regex for valid variable names (letters, underscores, starting with letter)
+export const VARIABLE_REGEX = /^[a-zA-Z][a-zA-Z_]*$/;
+
+// Regex to find variables in mustache syntax
+export const MUSTACHE_REGEX = /{{([^{}]*)}}+/g;
+
+// Regex to find multiline variables
+export const MULTILINE_VARIABLE_REGEX = /{{[^}]*\n[^}]*}}/g;
+
+// Regex to find unclosed variables
+export const UNCLOSED_VARIABLE_REGEX = /{{(?![^{]*}})/g;
+
+export function isValidVariableName(variable: string): boolean {
+  return VARIABLE_REGEX.test(variable);
+}
+
 export function extractVariables(mustacheString: string): string[] {
-  const mustacheRegex = /\{\{(.*?)\}\}/g;
-  const uniqueVariables = new Set<string>();
-
-  for (const match of mustacheString.matchAll(mustacheRegex)) {
-    uniqueVariables.add(match[1]);
-  }
-
-  for (const variable of uniqueVariables) {
-    // if validated fails, remove from set
-    if (!getIsCharOrUnderscore(variable)) {
-      uniqueVariables.delete(variable);
-    }
-  }
-
-  return Array.from(uniqueVariables);
+  const matches = Array.from(mustacheString.matchAll(MUSTACHE_REGEX))
+    .map((match) => match[1])
+    .filter(isValidVariableName);
+  return [...new Set(matches)];
 }
 
 export function stringifyValue(value: unknown) {

packages/shared/src/utils/zod.ts

@@ -23,18 +23,18 @@ type Json = Root | { [key: string]: JsonNested } | JsonNested[];
 // Here, you define the schema recursively
 export const jsonSchemaNullable: z.ZodType<JsonNested> = z.lazy(() =>
   z.union([
-    nestedLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
+    nestedLiteralSchema,
   ]),
 );
 
 // Root schema that does not allow nulls at the root level
 export const jsonSchema: z.ZodType<Json> = z.lazy(() =>
   z.union([
-    rootLiteralSchema,
     z.array(jsonSchemaNullable),
     z.record(jsonSchemaNullable),
+    rootLiteralSchema,
   ]),
 );
 

patches/next-auth@4.24.11.patch

@@ -0,0 +1,26 @@
+diff --git a/core/lib/oauth/client.js b/core/lib/oauth/client.js
+index 52c51eb6ff422dc0899ccec31baf3fa39e42eeae..bc50c35bb617d0e86b68ca42f64d44b475ba4abb 100644
+--- a/core/lib/oauth/client.js
++++ b/core/lib/oauth/client.js
+@@ -1,5 +1,7 @@
+ "use strict";
+ 
++var HttpsProxyAgent = require('https-proxy-agent').HttpsProxyAgent;
++
+ Object.defineProperty(exports, "__esModule", {
+   value: true
+ });
+@@ -7,7 +9,12 @@ exports.openidClient = openidClient;
+ var _openidClient = require("openid-client");
+ async function openidClient(options) {
+   const provider = options.provider;
+-  if (provider.httpOptions) _openidClient.custom.setHttpOptionsDefaults(provider.httpOptions);
++  let httpOptions = {};
++  if (provider.httpOptions) httpOptions = { ...provider.httpOptions };
++  if (process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY) {
++    httpOptions.agent = new HttpsProxyAgent(process.env.AUTH_HTTPS_PROXY || process.env.AUTH_HTTP_PROXY);
++  }
++  _openidClient.custom.setHttpOptionsDefaults(httpOptions);
+   let issuer;
+   if (provider.wellKnown) {
+     issuer = await _openidClient.Issuer.discover(provider.wellKnown);

turbo.json

@@ -16,9 +16,6 @@
     "db:reset": {},
     "db:push": {},
     "db:seed": {},
-    "models:migrate": {
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
-    },
     "db:seed:examples": {},
     "dev": {
       "cache": false,

web/Dockerfile

@@ -95,6 +95,9 @@ ARG BUILDPLATFORM
 
 WORKDIR /app
 
+ARG NEXT_PUBLIC_BUILD_ID
+ENV BUILD_ID=$NEXT_PUBLIC_BUILD_ID
+
 ENV NODE_ENV production
 # Uncomment the following line in case you want to disable telemetry during runtime.
 ENV NEXT_TELEMETRY_DISABLED 1
@@ -105,13 +108,16 @@ ENV NEXT_MANUAL_SIG_HANDLE true
 
 RUN apk add --no-cache dumb-init tzdata
 
-RUN addgroup --system --gid 1001 nodejs
-RUN adduser --system --uid 1001 nextjs
+# Don't run production as root
+ARG UID=1001
+ARG GID=1001
+RUN addgroup --system --gid ${GID} nodejs
+RUN adduser --system --uid ${UID} nextjs
 
-RUN npm install -g --no-package-lock --no-save prisma@5.22.0
+RUN npm install -g --no-package-lock --no-save prisma@6.3.0
 
 RUN MIGRATE_TARGET_ARCH=$(echo ${TARGETPLATFORM:-linux/amd64} | sed 's/\//-/g') && \
-    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.0/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
+    wget -q -O- https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.$MIGRATE_TARGET_ARCH.tar.gz | tar xvz && \
     mv migrate /usr/bin/migrate
 
 COPY --from=builder --chown=nextjs:nodejs /app/web/next.config.mjs .

web/entrypoint.sh

@@ -12,6 +12,11 @@ if [ -z "$DATABASE_URL" ]; then
         echo "Error: Required database environment variables are not set. Provide a postgres url for DATABASE_URL."
         exit 1
     fi
+    if [ -n "$DATABASE_ARGS" ]; then
+        # Append ARGS to DATABASE_URL
+        DATABASE_URL="${DATABASE_URL}?$DATABASE_ARGS"
+        export DATABASE_URL
+    fi
 fi
 
 # Check if CLICKHOUSE_URL is not set
@@ -22,7 +27,7 @@ fi
 
 # Set DIRECT_URL to the value of DATABASE_URL if it is not set, required for migrations
 if [ -z "$DIRECT_URL" ]; then
-    export DIRECT_URL=$DATABASE_URL
+    export DIRECT_URL="${DATABASE_URL}"
 fi
 
 # Always execute the postgres migration, except when disabled.

web/next.config.mjs

@@ -12,8 +12,8 @@ import { env } from "./src/env.mjs";
  */
 const cspHeader = `
   default-src 'self' https://*.langfuse.com https://*.langfuse.dev https://*.posthog.com https://*.sentry.io wss://*.crisp.chat https://*.crisp.chat;
-  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://settings.crisp.chat https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com;
-  style-src 'self' 'unsafe-inline' https://client.crisp.chat;
+  script-src 'self' 'unsafe-eval' 'unsafe-inline' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://settings.crisp.chat https://challenges.cloudflare.com https://*.sentry.io  https://static.cloudflareinsights.com https://*.stripe.com https://uptime.betterstack.com;
+  style-src 'self' 'unsafe-inline' https://client.crisp.chat https://uptime.betterstack.com;
   img-src 'self' https: blob: data: http://localhost:* https://client.crisp.chat https://image.crisp.chat https://storage.crisp.chat;
   font-src 'self' https://client.crisp.chat;
   frame-src 'self' https://challenges.cloudflare.com https://*.stripe.com https://game.crisp.chat;
@@ -22,7 +22,7 @@ const cspHeader = `
   base-uri 'self';
   form-action 'self';
   frame-ancestors 'none';
-  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://storage.crisp.chat wss://client.relay.crisp.chat wss://stream.relay.crisp.chat https://*.ingest.us.sentry.io;
+  connect-src 'self' https://*.langfuse.com https://*.langfuse.dev https://client.crisp.chat https://storage.crisp.chat wss://client.relay.crisp.chat wss://stream.relay.crisp.chat https://*.ingest.us.sentry.io https://uptime.betterstack.com;
   media-src 'self' https: http://localhost:* https://client.crisp.chat;
   ${env.LANGFUSE_CSP_ENFORCE_HTTPS === "true" ? "upgrade-insecure-requests; block-all-mixed-content;" : ""}
   ${env.SENTRY_CSP_REPORT_URI ? `report-uri ${env.SENTRY_CSP_REPORT_URI}; report-to csp-endpoint;` : ""}
@@ -128,22 +128,22 @@ const nextConfig = {
       // Required to check authentication status from langfuse.com
       ...(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION !== undefined
         ? [
-          {
-            source: "/api/auth/session",
-            headers: [
-              {
-                key: "Access-Control-Allow-Origin",
-                value: "https://langfuse.com",
-              },
-              { key: "Access-Control-Allow-Credentials", value: "true" },
-              { key: "Access-Control-Allow-Methods", value: "GET,POST" },
-              {
-                key: "Access-Control-Allow-Headers",
-                value: "Content-Type, Authorization",
-              },
-            ],
-          },
-        ]
+            {
+              source: "/api/auth/session",
+              headers: [
+                {
+                  key: "Access-Control-Allow-Origin",
+                  value: "https://langfuse.com",
+                },
+                { key: "Access-Control-Allow-Credentials", value: "true" },
+                { key: "Access-Control-Allow-Methods", value: "GET,POST" },
+                {
+                  key: "Access-Control-Allow-Headers",
+                  value: "Content-Type, Authorization",
+                },
+              ],
+            },
+          ]
         : []),
       // all files in /public/generated are public and can be accessed from any origin, e.g. to render an API reference based on our openapi schema
       {

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.2.0",
+  "version": "3.26.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -19,15 +19,15 @@
     "test-client": "dotenv -e ../.env -- jest --silent false --verbose false --runInBand --detectOpenHandles --selectProjects client",
     "test:watch": "dotenv -e ../.env -- jest --watch --runInBand",
     "test:e2e": "dotenv -e ../.env -- playwright test",
-    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server",
-    "models:migrate": "dotenv -e ../.env -- tsx scripts/model-match.ts"
+    "test:e2e:server": "dotenv -e ../.env -- jest --runInBand --detectOpenHandles --verbose --selectProjects e2e-server"
   },
   "dependencies": {
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.3",
     "@baselime/trpc-opentelemetry-middleware": "^0.1.2",
     "@codemirror/lang-json": "^6.0.1",
-    "@codemirror/lint": "^6.8.0",
+    "@codemirror/language": "^6.10.1",
+    "@codemirror/lint": "^6.4.2",
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
@@ -41,6 +41,7 @@
     "@langchain/openai": "^0.3.14",
     "@langfuse/ee": "workspace:*",
     "@langfuse/shared": "workspace:*",
+    "@lezer/highlight": "^1.2.1",
     "@marsidev/react-turnstile": "^0.5.4",
     "@mui/x-tree-view": "^7.19.0",
     "@next-auth/prisma-adapter": "^1.0.7",
@@ -58,7 +59,7 @@
     "@opentelemetry/sdk-node": "^0.53.0",
     "@opentelemetry/sdk-trace-base": "^1.26.0",
     "@opentelemetry/sdk-trace-node": "^1.26.0",
-    "@prisma/instrumentation": "^5.22.0",
+    "@prisma/instrumentation": "^6.3.0",
     "@radix-ui/react-accordion": "^1.2.1",
     "@radix-ui/react-alert-dialog": "^1.1.2",
     "@radix-ui/react-avatar": "^1.1.1",
@@ -70,6 +71,7 @@
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.1.0",
     "@radix-ui/react-popover": "^1.1.2",
+    "@radix-ui/react-progress": "^1.1.1",
     "@radix-ui/react-radio-group": "^1.2.1",
     "@radix-ui/react-scroll-area": "^1.2.0",
     "@radix-ui/react-select": "^2.1.2",
@@ -84,7 +86,7 @@
     "@remixicon/react": "^4.2.0",
     "@repo/eslint-config": "workspace:*",
     "@repo/typescript-config": "workspace:*",
-    "@sentry/nextjs": "^8.39.0",
+    "@sentry/nextjs": "^8.52.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
     "@tanstack/react-query": "^4.36.1",
@@ -94,37 +96,40 @@
     "@trpc/next": "^10.45.0",
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
-    "@uiw/codemirror-theme-github": "^4.23.0",
-    "@uiw/codemirror-theme-tokyo-night": "^4.23.5",
-    "@uiw/react-codemirror": "^4.21.25",
+    "@uiw/codemirror-themes": "^4.23.7",
+    "@uiw/react-codemirror": "^4.21.24",
     "ai": "^3.4.9",
     "bcryptjs": "^2.4.3",
-    "bullmq": "^5.12.10",
+    "bullmq": "^5.34.10",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
-    "cmdk": "^1.0.0",
+    "cmdk": "^1.0.4",
     "core-js": "^3.38.1",
     "cors": "^2.8.5",
+    "csv-parse": "^5.6.0",
     "date-fns": "^3.3.1",
     "dd-trace": "^5.23.1",
     "decimal.js": "^10.4.3",
+    "diff": "^7.0.0",
     "dompurify": "^3.1.5",
+    "fastest-levenshtein": "^1.0.16",
     "graphql": "^16.9.0",
+    "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "ip-address": "^9.0.5",
-    "js-tiktoken": "^1.0.15",
     "kysely": "^0.27.4",
     "langchain": "^0.3.6",
     "lodash": "^4.17.21",
     "lucide-react": "^0.462.0",
-    "next": "^14.2.15",
+    "next": "^14.2.21",
     "next-auth": "^4.24.11",
     "next-query-params": "^5.0.1",
     "next-themes": "^0.3.0",
     "posthog-js": "^1.176.0",
     "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
-    "prisma": "^5.22.0",
+    "prisma": "^6.3.0",
+    "protobufjs": "^7.4.0",
     "rate-limiter-flexible": "^5.0.3",
     "react": "18.2.0",
     "react-day-picker": "^8.10.1",
@@ -138,11 +143,15 @@
     "react18-json-view": "^0.2.8-canary.6",
     "remark-gfm": "^4.0.0",
     "remark-math": "^6.0.0",
+    "remark-parse": "^11.0.0",
+    "remark-rehype": "^11.1.1",
+    "remark-stringify": "^11.0.0",
     "sonner": "^1.4.41",
     "stripe": "^17.4.0",
     "superjson": "2.2.1",
     "tailwind-merge": "^2.5.2",
     "tailwindcss-animate": "^1.0.7",
+    "unified": "^11.0.5",
     "use-query-params": "^2.2.1",
     "uuid": "^9.0.1",
     "vaul": "^0.9.1",
@@ -157,6 +166,7 @@
     "@testing-library/react": "^15.0.7",
     "@types/bcryptjs": "^2.4.6",
     "@types/cors": "^2.8.17",
+    "@types/diff": "^7.0.0",
     "@types/dompurify": "^3.0.5",
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
@@ -175,10 +185,9 @@
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "node-mocks-http": "^1.14.1",
-    "postcss": "^8.4.47",
     "prettier": "^3.3.3",
     "prettier-plugin-tailwindcss": "^0.6.6",
-    "tailwindcss": "^3.4.13",
+    "tailwindcss": "^3.4.17",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
     "tsx": "^4.19.1",
@@ -193,7 +202,9 @@
   },
   "pnpm": {
     "overrides": {
-      "jsonpath-plus": "10.0.7"
+      "jsonpath-plus": "10.2.0",
+      "nanoid": "^3.3.8",
+      "katex": "0.16.21"
     }
   }
 }

web/public/generated/api-client/openapi.yml

@@ -53,6 +53,7 @@ components:
       properties:
         id:
           type: string
+          nullable: true
         traceId:
           type: string
           example: cdef-1234-5678-90ab
@@ -67,15 +68,19 @@ components:
             values must equal either 1 or 0 (true or false)
         observationId:
           type: string
+          nullable: true
         comment:
           type: string
+          nullable: true
         dataType:
           $ref: '#/components/schemas/ScoreDataType'
+          nullable: true
           description: >-
             When set, must match the score value's type. If not set, will be
             inferred from the score value or config
         configId:
           type: string
+          nullable: true
           description: >-
             Reference a score config on a score. When set, the score name must
             equal the config name and scores must comply with the config's range
@@ -100,6 +105,7 @@ components:
           $ref: '#/components/schemas/ScoreSource'
         observationId:
           type: string
+          nullable: true
         timestamp:
           type: string
           format: date-time
@@ -111,10 +117,13 @@ components:
           format: date-time
         authorUserId:
           type: string
+          nullable: true
         comment:
           type: string
+          nullable: true
         configId:
           type: string
+          nullable: true
           description: >-
             Reference a score config on a score. When set, config and score name
             must be equal and value must comply to optionally defined numerical
@@ -166,6 +175,7 @@ components:
         value:
           type: number
           format: double
+          nullable: true
           description: >-
             Only defined if a config is linked. Represents the numeric category
             mapping of the stringValue

web/public/generated/api/openapi.yml

@@ -723,6 +723,9 @@ paths:
         Notes:
 
 
+        - Introduction to data model:
+        https://langfuse.com/docs/tracing-data-model
+
         - Batch sizes are limited to 3.5 MB in total. You need to adjust the
         number of events per batch accordingly.
 
@@ -734,7 +737,7 @@ paths:
         - Ingestion
       parameters: []
       responses:
-        '200':
+        '207':
           description: ''
           content:
             application/json:
@@ -1423,6 +1426,76 @@ paths:
               schema: {}
       security:
         - BasicAuth: []
+  /api/public/v2/prompts/{name}/versions/{version}:
+    patch:
+      description: Update labels for a specific prompt version
+      operationId: promptVersion_update
+      tags:
+        - PromptVersion
+      parameters:
+        - name: name
+          in: path
+          description: The name of the prompt
+          required: true
+          schema:
+            type: string
+        - name: version
+          in: path
+          description: Version of the prompt to update
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Prompt'
+        '400':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '401':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '403':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '404':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+        '405':
+          description: ''
+          content:
+            application/json:
+              schema: {}
+      security:
+        - BasicAuth: []
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                newLabels:
+                  type: array
+                  items:
+                    type: string
+                  description: >-
+                    New labels for the prompt version. Labels are unique across
+                    versions. The "latest" label is reserved and managed by
+                    Langfuse.
+              required:
+                - newLabels
   /api/public/v2/prompts/{promptName}:
     get:
       description: Get a prompt
@@ -3183,11 +3256,12 @@ components:
             to exact match, use `(?i)^modelname$`
         startDate:
           type: string
-          format: date
+          format: date-time
           nullable: true
           description: Apply only to generations which are newer than this ISO date.
         unit:
           $ref: '#/components/schemas/ModelUsageUnit'
+          nullable: true
           description: Unit used by this model.
         inputPrice:
           type: number
@@ -3223,7 +3297,6 @@ components:
         - id
         - modelName
         - matchPattern
-        - unit
         - isLangfuseManaged
     ModelUsageUnit:
       title: ModelUsageUnit
@@ -4276,6 +4349,7 @@ components:
           description: Apply only to generations which are newer than this ISO date.
         unit:
           $ref: '#/components/schemas/ModelUsageUnit'
+          nullable: true
           description: Unit used by this model.
         inputPrice:
           type: number
@@ -4308,7 +4382,6 @@ components:
       required:
         - modelName
         - matchPattern
-        - unit
     Observations:
       title: Observations
       type: object
@@ -4450,6 +4523,10 @@ components:
             type: string
           nullable: true
           description: List of tags to apply to all versions of this prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
       required:
         - name
         - prompt
@@ -4475,6 +4552,10 @@ components:
             type: string
           nullable: true
           description: List of tags to apply to all versions of this prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
       required:
         - name
         - prompt
@@ -4524,6 +4605,10 @@ components:
           description: >-
             List of tags. Used to filter via UI and API. The same across
             versions of a prompt.
+        commitMessage:
+          type: string
+          nullable: true
+          description: Commit message for this prompt version.
       required:
         - name
         - version

web/public/generated/postman/collection.json

@@ -558,7 +558,7 @@
           "_type": "endpoint",
           "name": "Batch",
           "request": {
-            "description": "Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.\n\nNotes:\n\n- Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.\n- The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.",
+            "description": "Batched ingestion for Langfuse Tracing. If you want to use tracing via the API, such as to build your own Langfuse client implementation, this is the only API route you need to implement.\n\nNotes:\n\n- Introduction to data model: https://langfuse.com/docs/tracing-data-model\n- Batch sizes are limited to 3.5 MB in total. You need to adjust the number of events per batch accordingly.\n- The API does not return a 4xx status code for input errors. Instead, it responds with a 207 status code, which includes a list of the encountered errors.",
             "url": {
               "raw": "{{baseUrl}}/api/public/ingestion",
               "host": [
@@ -1057,6 +1057,61 @@
         }
       ]
     },
+    {
+      "_type": "container",
+      "description": null,
+      "name": "Prompt Version",
+      "item": [
+        {
+          "_type": "endpoint",
+          "name": "Update",
+          "request": {
+            "description": "Update labels for a specific prompt version",
+            "url": {
+              "raw": "{{baseUrl}}/api/public/v2/prompts/:name/versions/:version",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "public",
+                "v2",
+                "prompts",
+                ":name",
+                "versions",
+                ":version"
+              ],
+              "query": [],
+              "variable": [
+                {
+                  "key": "name",
+                  "value": "",
+                  "description": "The name of the prompt"
+                },
+                {
+                  "key": "version",
+                  "value": "",
+                  "description": "Version of the prompt to update"
+                }
+              ]
+            },
+            "header": [],
+            "method": "PATCH",
+            "auth": null,
+            "body": {
+              "mode": "raw",
+              "raw": "{\n    \"newLabels\": [\n        \"example\"\n    ]\n}",
+              "options": {
+                "raw": {
+                  "language": "json"
+                }
+              }
+            }
+          },
+          "response": []
+        }
+      ]
+    },
     {
       "_type": "container",
       "description": null,
@@ -1192,7 +1247,7 @@
             "auth": null,
             "body": {
               "mode": "raw",
-              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ]\n}",
+              "raw": "{\n    \"type\": \"chat\",\n    \"name\": \"example\",\n    \"prompt\": [\n        {\n            \"role\": \"example\",\n            \"content\": \"example\"\n        }\n    ],\n    \"config\": \"UNKNOWN\",\n    \"labels\": [\n        \"example\"\n    ],\n    \"tags\": [\n        \"example\"\n    ],\n    \"commitMessage\": \"example\"\n}",
               "options": {
                 "raw": {
                   "language": "json"

web/scripts/model-match.ts

@@ -1,260 +0,0 @@
-// Description: New model definitions in Langfuse are automatically applied to new observations.
-// You can optionally run this script to apply new model definitions to existing observations.
-// See docs: https://langfuse.com/docs/deployment/self-host#migrate-models
-// Execute: `npm run models:migrate`
-
-import "dotenv/config";
-
-import { findModel } from "@langfuse/shared/src/server";
-import { prisma } from "@langfuse/shared/src/db";
-import lodash from "lodash";
-import { type Prisma } from "@langfuse/shared/src/db";
-import { tokenCount } from "@/src/features/ingest/usage";
-
-async function main() {
-  return await modelMatch();
-}
-
-// Call the function
-main().catch((err) => {
-  console.error("An error occurred:", err);
-});
-
-export async function modelMatch() {
-  console.log("Starting model match");
-  const start = Date.now();
-
-  const BATCH_SIZE = 10_000;
-  let continueLoop = true;
-  let index = 0;
-  let totalObservations = 0;
-
-  while (continueLoop) {
-    type ObservationSelect = {
-      model: string | null;
-      id: string;
-      projectId: string;
-      startTime: Date;
-      unit: string | null;
-      promptTokens: number;
-      completionTokens: number;
-      totalTokens: number;
-      input: Prisma.JsonValue;
-      output: Prisma.JsonValue;
-    };
-
-    const observations = await prisma.observation.findMany({
-      select: {
-        id: true,
-        startTime: true,
-        model: true,
-        unit: true,
-        projectId: true,
-        promptTokens: true,
-        completionTokens: true,
-        totalTokens: true,
-        input: true,
-        output: true,
-      },
-      orderBy: {
-        startTime: "desc",
-      },
-      where: {
-        internalModel: null,
-        type: "GENERATION",
-        model: { not: null },
-      },
-      take: BATCH_SIZE,
-      skip: index * BATCH_SIZE,
-    });
-
-    console.log(`Found ${observations.length} observations to migrate`);
-
-    type Config = {
-      startTime: Date;
-      model: string;
-      unit: string;
-      projectId: string | null;
-    };
-
-    interface GroupedObservations {
-      [key: string]: ObservationSelect[];
-    }
-
-    const groupedObservations = observations.reduce<GroupedObservations>(
-      (acc, observation) => {
-        const config = {
-          startTime: observation.startTime.toISOString().slice(0, 10),
-          model: observation.model,
-          unit: observation.unit,
-          projectId: observation.projectId,
-        };
-
-        const key = JSON.stringify(config);
-
-        // Ensure the array is initialized before using it
-        acc[key] = acc[key] ?? [];
-        acc[key]?.push(observation);
-
-        return acc;
-      },
-      {},
-    );
-
-    let updatedObservations = 0;
-    const dbPromises = [];
-
-    for (const [key, observationsGroup] of Object.entries(
-      groupedObservations,
-    )) {
-      const { startTime, model, unit, projectId } = JSON.parse(key) as Config;
-
-      console.log("Execute key: ", startTime, model, unit, projectId);
-
-      if (!projectId) {
-        throw new Error("No project id");
-      }
-
-      const foundModel = await findModel({
-        event: { projectId, model, unit, startTime: startTime },
-      });
-
-      console.log(
-        "Found model: ",
-        foundModel?.id,
-        " for key: ",
-        key,
-        " with observations: ",
-        observationsGroup.length,
-      );
-
-      if (foundModel) {
-        // find all the observations with all tokens 0 and tokenize them individually
-        const observationsWithAllTokensZero = observationsGroup.filter(
-          (observation) =>
-            observation.promptTokens === 0 &&
-            observation.completionTokens === 0 &&
-            observation.totalTokens === 0,
-        );
-
-        for (const observation of observationsWithAllTokensZero) {
-          console.log("Tokenizing observation: ", observation.id);
-          const newInputCount = tokenCount({
-            model: foundModel,
-            text: observation.input,
-          });
-          const newOutputCount = tokenCount({
-            model: foundModel,
-            text: observation.output,
-          });
-
-          dbPromises.push(
-            prisma.observation.update({
-              where: {
-                id: observation.id,
-              },
-              data: {
-                promptTokens: newInputCount,
-                completionTokens: newOutputCount,
-                totalTokens: (newInputCount ?? 0) + (newOutputCount ?? 0),
-                internalModel: foundModel.modelName,
-              },
-            }),
-          );
-        }
-
-        // for all remaining observations, batch update them with the model id
-        const observationsWithTokens = observationsGroup.filter(
-          (observation) =>
-            observation.promptTokens !== 0 ||
-            observation.completionTokens !== 0 ||
-            observation.totalTokens !== 0,
-        );
-
-        // Push the promise for updating observations into the array
-        lodash.chunk(observationsWithTokens, 32000).map((chunk) => {
-          dbPromises.push(
-            prisma.observation.updateMany({
-              where: {
-                id: {
-                  in: chunk.map((observation) => observation.id),
-                },
-              },
-              data: {
-                internalModel: foundModel.modelName,
-              },
-            }),
-          );
-        });
-
-        updatedObservations += observationsGroup.length;
-      } else {
-        lodash.chunk(observationsGroup, 32000).map((chunk) => {
-          dbPromises.push(
-            prisma.observation.updateMany({
-              where: {
-                id: {
-                  in: chunk.map((observation) => observation.id),
-                },
-              },
-              data: {
-                internalModel: "LANGFUSETMPNOMODEL",
-              },
-            }),
-          );
-        });
-
-        updatedObservations += observationsGroup.length;
-      }
-    }
-
-    totalObservations += updatedObservations;
-    // Wait for all update operations to complete
-    const promiseChunk = lodash.chunk(dbPromises, 10);
-
-    for (const promises of promiseChunk) {
-      console.log("Waiting for promises to complete", promises.length);
-      await Promise.all(promises);
-    }
-
-    console.log(
-      "Updated observations count: ",
-      updatedObservations,
-      " in total: ",
-      totalObservations,
-    );
-
-    console.log(updatedObservations, observations.length);
-
-    if (updatedObservations === 0) {
-      index++;
-    }
-
-    if (observations.length === 0) {
-      console.log("No more observations to migrate");
-      continueLoop = false;
-    }
-  }
-
-  let updatedCount;
-  do {
-    console.log(`Updating LANGFUSETMPNOMODEL ${updatedCount}`);
-    const result = await prisma.$queryRaw<[{ id: string }]>`
-    WITH to_update AS (
-      SELECT id 
-      FROM observations 
-      WHERE internal_model = 'LANGFUSETMPNOMODEL'
-      AND "type" = 'GENERATION'
-      LIMIT 50000
-    )
-    UPDATE observations
-    set internal_model = NULL
-    WHERE id IN (SELECT id FROM to_update)
-    RETURNING id;`;
-    updatedCount = result.length;
-  } while (updatedCount > 0);
-
-  const end = Date.now();
-
-  console.log(`Model match took ${end - start} ms`);
-}

web/src/__e2e__/auth.spec.ts

@@ -123,7 +123,7 @@ test("Unauthenticated user should not be redirected to non-relative URLs after l
 test("Unauthenticated user should be redirected to relative URL after login", async ({
   page,
 }) => {
-  const relativeUrl = "/support";
+  const relativeUrl = "/setup";
   await page.goto(
     `/auth/sign-in?targetPath=${encodeURIComponent(relativeUrl)}`,
   );

web/src/__tests__/async/daily-metrics-api.servertest.ts

@@ -23,11 +23,9 @@ describe("/api/public/metrics/daily API Endpoint", () => {
       createObservation({
         trace_id: createdTrace.id,
         project_id: createdTrace.project_id,
-        name: "observation-name",
+        name: "observation-name-1",
         end_time: new Date().getTime(),
         start_time: new Date().getTime() - 1000,
-        input: "input",
-        output: "output",
         provided_model_name: "model-1",
       }),
       createObservation({
@@ -35,9 +33,23 @@ describe("/api/public/metrics/daily API Endpoint", () => {
         project_id: createdTrace.project_id,
         name: "observation-name-2",
         end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 1000,
+        provided_model_name: "model-1",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-3",
+        end_time: new Date().getTime(),
+        start_time: new Date().getTime() - 100000,
+        provided_model_name: "model-2",
+      }),
+      createObservation({
+        trace_id: createdTrace.id,
+        project_id: createdTrace.project_id,
+        name: "observation-name-4",
+        end_time: new Date().getTime(),
         start_time: new Date().getTime() - 100000,
-        input: "input-2",
-        output: "output-2",
         provided_model_name: "model-2",
       }),
     ];
@@ -56,17 +68,17 @@ describe("/api/public/metrics/daily API Endpoint", () => {
 
     const metric = metrics.body.data[0];
     expect(metric.countTraces).toBe(1);
-    expect(metric.countObservations).toBe(2);
+    expect(metric.countObservations).toBe(4);
     expect(metric.usage).toHaveLength(2);
-    expect(metric.totalCost).toBe(600);
+    expect(metric.totalCost).toBe(1200);
     for (const usage of metric.usage) {
       expect(usage.model).toMatch(/model-\d/g);
-      expect(usage.inputUsage).toBe(1234);
-      expect(usage.outputUsage).toBe(5678);
-      expect(usage.totalUsage).toBe(6912);
-      expect(usage.countObservations).toBe(1);
+      expect(usage.inputUsage).toBe(1234 * 2);
+      expect(usage.outputUsage).toBe(5678 * 2);
+      expect(usage.totalUsage).toBe(6912 * 2);
+      expect(usage.countObservations).toBe(2);
       expect(usage.countTraces).toBe(1);
-      expect(usage.totalCost).toBe(300);
+      expect(usage.totalCost).toBe(600);
     }
   });
 });

web/src/__tests__/async/dataset-service.servertest.ts

@@ -192,7 +192,6 @@ describe("Fetch datasets for UI presentation", () => {
     const runs = await createDatasetRunsTable({
       projectId,
       datasetId,
-      queryClickhouse: false,
       page: 0,
       limit: 10,
     });
@@ -238,13 +237,14 @@ describe("Fetch datasets for UI presentation", () => {
     expect(secondRun.run_id).toEqual(datasetRun2Id);
     expect(secondRun.run_description).toBeNull();
     expect(secondRun.run_metadata).toEqual({});
-    expect(secondRun.avgLatency).toEqual(1);
+    expect(secondRun.avgLatency).toBeGreaterThanOrEqual(1);
+    expect(secondRun.avgLatency).toBeLessThanOrEqual(1.002);
     expect(secondRun.avgTotalCost.toString()).toStrictEqual("300");
 
     expect(JSON.stringify(secondRun.scores)).toEqual(JSON.stringify({}));
   });
 
-  it.only("should test that dataset runs can link to the same traces", async () => {
+  it("should test that dataset runs can link to the same traces", async () => {
     const datasetId = v4();
 
     await prisma.dataset.create({
@@ -328,7 +328,6 @@ describe("Fetch datasets for UI presentation", () => {
     const runs = await createDatasetRunsTable({
       projectId,
       datasetId,
-      queryClickhouse: false,
       page: 0,
       limit: 10,
     });