chore: release v3.87.0

* feat(annotation-queues): add support for batch adding sessions to annotation queues

* chore: eslint types

.claude/settings.json

@@ -4,8 +4,11 @@
       "Bash(find:*)",
       "Bash(rg:*)",
       "Bash(grep:*)",
-      "Bash(pnpm run test:*)"
+      "Bash(ls:*)",
+      "Bash(cat:*)",
+      "Bash(head:*)",
+      "Bash(tail:*)"
     ],
     "deny": []
   }
-}
+}

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx,notIn
+ignore-words-list = afterall,vertx,notIn,alue
 

.devcontainer.json

@@ -1,6 +1,6 @@
 {
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+  "name": "langfuse-development",
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
 }

.github/dependabot.yml

@@ -36,6 +36,7 @@ updates:
         patterns:
           - "express"
           - "@types/express"
+          - "@types/express-serve-static-core"
       observability:
         patterns:
           - "dd-trace"

.github/workflows/_deploy_ecs_service.yml

@@ -15,10 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - name: Get app name
         uses: winterjung/split@v2
         id: split

.github/workflows/pipeline.yml

@@ -52,6 +52,49 @@ jobs:
       - name: lint web
         run: pnpm run lint
 
+  prettier-check:
+    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm i
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+      - name: Check formatting on changed files
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            BASE_SHA=${{ github.event.pull_request.base.sha }}
+          else
+            BASE_SHA=$(git merge-base origin/main HEAD)
+          fi
+          
+          echo "Checking files changed from $BASE_SHA to HEAD"
+          
+          # Get changed files
+          CHANGED_FILES=$(git diff --name-only $BASE_SHA HEAD -- '*.js' '*.jsx' '*.ts' '*.tsx' '*.css' | tr '\n' ' ')
+          
+          if [ -n "$CHANGED_FILES" ] && [ "$CHANGED_FILES" != " " ]; then
+            echo "Files to check: $CHANGED_FILES"
+            pnpm prettier --check --experimental-cli $CHANGED_FILES
+          else
+            echo "No JS/TS/CSS files changed - skipping prettier check"
+          fi
+
   test-docker-build:
     timeout-minutes: 20
     runs-on: ubuntu-latest
@@ -129,6 +172,7 @@ jobs:
           cp .env.dev.example .env
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=false" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run dev containers
         run: |
@@ -391,8 +435,6 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
-          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -430,6 +472,7 @@ jobs:
     needs:
       [
         lint,
+        prettier-check,
         tests-web-sync,
         tests-worker,
         e2e-tests,
@@ -440,11 +483,11 @@ jobs:
     if: always()
     steps:
       - name: Successful deploy
-        if: ${{ !(contains(needs.*.result, 'failure')) }}
+        if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
         run: exit 0
         working-directory: .
       - name: Failing deploy
-        if: ${{ contains(needs.*.result, 'failure') }}
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
 

.github/workflows/snyk.yml

@@ -1,65 +1,28 @@
 name: Snyk Container
 on:
-  pull_request:
-    branches:
-      - "**"
   push:
-    branches:
-      - main
-  # Snyk cannot upload results in merge group. Hence, we only run on PRs and when pushingon the main branch https://github.com/github/codeql-action/issues/1572
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    branches: ["main"]
 
 jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Build a Docker image
-        run: docker compose -f docker-compose.build.yml up -d
+      - uses: actions/checkout@v3
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
-        continue-on-error: true
+      - name: Scan web image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-web
-          args: --file=web/Dockerfile
+          image: langfuse/langfuse # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)
 
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: web
-
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
-        continue-on-error: true
+      - name: Scan worker image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-worker
-          args: --file=worker/Dockerfile
-
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: worker
+          image: langfuse/langfuse-worker # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)

.gitignore

@@ -67,4 +67,7 @@ node_modules
 .yarn
 .turbo
 
-web/test-results/*
+web/test-results/*
+
+# local config files
+*.local.*

.husky/pre-commit

@@ -11,15 +11,13 @@ if [ "$current_branch" = "$protected_branch" ]; then
     echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
     read -r answer < /dev/tty
     if [ "$answer" != "${answer#[Yy]}" ]; then
-        # Commit approved, run formatting
-        pnpm run format-fast
-        exit 0 # Commit will proceed
+        # Commit approved, check formatting. On files changed, block commit
+        pnpm run format:check
     else
         echo "Commit to $protected_branch branch has been canceled."
         exit 1 # Commit will be blocked
     fi
 fi
 
-# If not the protected branch, run formatting and proceed with the commit
-pnpm run format-fast
-exit 0
+# If not the protected branch, check formatting (on changed files, block)
+pnpm run format:check

CLAUDE.md

@@ -189,3 +189,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## TypeScript Best Practices
 - In TypeScript, if possible, don't use the `any` type
+
+## General Coding Guidelines
+- For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so

CONTRIBUTING.md

@@ -137,7 +137,7 @@ Requirements
     cp .env.dev.example .env
    ```
 
-4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it.
+4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it. Also, this will fail on the very first run. Please run it again.
 
    ```bash
    pnpm run dx # first run only (resets db, docker containers, etc...)

README.cn.md

@@ -94,7 +94,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
   
 - [提示管理](https://langfuse.com/docs/prompts/get-started) 帮助你集中管理、版本控制并协作迭代提示。得益于服务器和客户端的高效缓存，你可以在不增加延迟的情况下反复迭代提示。
 
-- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为“裁判”、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
+- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为"裁判"、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
 
 - [数据集](https://langfuse.com/docs/datasets/overview) 为评估你的 LLM 应用提供测试集和基准。它们支持持续改进、部署前测试、结构化实验、灵活评估，并能与 LangChain、LlamaIndex 等框架无缝整合。
 
@@ -135,7 +135,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 
 - [虚拟机](https://langfuse.com/self-hosting/docker-compose)：使用 Docker Compose 在单台虚拟机上部署 Langfuse。
 
-- 【计划中】：针对各云平台的部署指南，欢迎在以下讨论中投票和评论：[AWS](https://github.com/orgs/langfuse/discussions/4645)、[Google Cloud](https://github.com/langfuse/discussions/4646)、[Azure](https://github.com/orgs/langfuse/discussions/4647)。
+- Terraform 模板: [AWS](https://langfuse.com/self-hosting/aws)、[Azure](https://langfuse.com/self-hosting/azure)、[GCP](https://langfuse.com/self-hosting/gcp)
 
 请参阅 [自托管文档](https://langfuse.com/self-hosting) 了解更多关于架构和配置选项的信息。
 
@@ -155,6 +155,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | [LiteLLM](https://langfuse.com/docs/integrations/litellm)               | Python, JS/TS (仅代理)     | 允许使用任何 LLM 替代 GPT。支持 Azure、OpenAI、Cohere、Anthropic、Ollama、VLLM、Sagemaker、HuggingFace、Replicate（100+ LLMs）。         |
 | [Vercel AI SDK](https://langfuse.com/docs/integrations/vercel-ai-sdk)     | JS/TS                    | 基于 TypeScript 的工具包，帮助开发者使用 React、Next.js、Vue、Svelte 和 Node.js 构建 AI 驱动的应用。                                           |
 | [API](https://langfuse.com/docs/api)                                    |                          | 直接调用公共 API。提供 OpenAPI 规格。                                                                                                       |
+| [Google VertexAI 和 Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 模型                | 在 Google 上运行基础模型和微调模型。                                                                         |
 
 ### 与 Langfuse 集成的软件包：
 
@@ -162,9 +163,9 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | ---------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------- |
 | [Instructor](https://langfuse.com/docs/integrations/instructor)         | 库                  | 用于获取结构化 LLM 输出（JSON、Pydantic）的库。                                                         |
 | [DSPy](https://langfuse.com/docs/integrations/dspy)                     | 库                  | 一个系统性优化语言模型提示和权重的框架。                                                                 |
+| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 库                  | 构建 LLM 应用的 Python 工具包。                                                                           |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 模型（本地）         | 在你的机器上轻松运行开源 LLM。                                                                            |
-| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 代理框架             | 用于构建分布式代理的开源 LLM 平台。                                                                       |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 聊天/代理界面         | 基于 JS/TS 的无代码构建器，用于定制化 LLM 流程。                                                          |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 聊天/代理界面         | 基于 Python 的 LangChain 用户界面，采用 react-flow 设计，提供便捷的实验与原型构建体验。                   |
@@ -253,7 +254,7 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 
 - 我们的 [文档](https://langfuse.com/docs) 是查找答案的最佳起点。内容全面，我们投入大量时间进行维护。你也可以通过 GitHub 提出文档修改建议。
 - [Langfuse 常见问题](https://langfuse.com/faq) 解答了最常见的问题。
-- 使用 “[Ask AI](https://langfuse.com/docs/ask-ai)” 立即获取问题答案。
+- 使用 "Ask AI" 立即获取问题答案。
 
 支持渠道：
 
@@ -351,4 +352,4 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 所有数据均不会与第三方共享，也不包含任何敏感信息。我们对这一过程保持高度透明，你可以在 [此处](/web/src/features/telemetry/index.ts) 查看我们收集的具体数据。
 
 你可以通过设置 `TELEMETRY_ENABLED=false` 来选择退出。
-````
+```

README.ja.md

@@ -141,9 +141,7 @@ Langfuseチームによるマネージドデプロイメント。充実した無
 - **[VM](https://langfuse.com/self-hosting/docker-compose):**  
   Docker Composeを使用して、単一の仮想マシン上でLangfuseを実行します。
 
-- **Planned:**  
-  クラウド固有のデプロイガイドは計画中です。以下のスレッドに対して投票やコメントをお願いします:  
-  [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform テンプレート: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 [セルフホスティングのドキュメント](https://langfuse.com/self-hosting)を参照し、アーキテクチャや設定オプションの詳細をご確認ください。
 

README.kr.md

@@ -128,7 +128,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 
 - [Kubernetes (Helm)](https://langfuse.com/self-hosting/kubernetes-helm): Helm을 사용해 Kubernetes 클러스터에서 Langfuse를 실행합니다. 이는 권장되는 프로덕션 배포 방식입니다.
 - [VM](https://langfuse.com/self-hosting/docker-compose): Docker Compose를 사용해 단일 가상 머신에서 Langfuse를 실행합니다.
-- 예정: 클라우드별 배포 가이드 – 아래 스레드에서 투표 및 댓글을 남겨주세요: [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform 템플릿: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 자세한 내용은 [자체 호스팅 문서](https://langfuse.com/self-hosting)를 참조하세요.
 
@@ -158,6 +158,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 라이브러리         | LLM 애플리케이션 구축을 위한 Python 툴킷입니다.                                                                    |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 모델 (로컬)        | 자신의 컴퓨터에서 오픈 소스 LLM을 손쉽게 실행할 수 있습니다.                                                      |
 | [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 모델               | AWS에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
+| [Google VertexAI and Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 모델               | Google에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 에이전트 프레임워크 | 분산 에이전트 구축을 위한 오픈 소스 LLM 플랫폼입니다.                                                              |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 채팅/에이전트 UI    | 맞춤형 LLM 플로우를 위한 JS/TS 코드 없는(no-code) 빌더입니다.                                                       |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 채팅/에이전트 UI    | react-flow를 활용하여 실험 및 프로토타이핑을 손쉽게 할 수 있도록 디자인된 LangChain용 Python 기반 UI입니다.          |

docker-compose.yml

@@ -62,6 +62,8 @@ services:
       REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
       REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
       REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
+      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-}
+      SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-}
 
   langfuse-web:
     image: docker.io/langfuse/langfuse:3

ee/package.json

@@ -45,10 +45,5 @@
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

fern/apis/server/definition/observations.yml

@@ -32,6 +32,9 @@ service:
           userId: optional<string>
           type: optional<string>
           traceId: optional<string>
+          level:
+            type: optional<commons.ObservationLevel>
+            docs: Optional filter for observations with a specific level (e.g. "DEBUG", "DEFAULT", "WARNING", "ERROR").
           parentObservationId: optional<string>
           environment:
             type: optional<string>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.79.0",
+  "version": "3.87.0",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -27,8 +27,8 @@
     "dev:web": "turbo run dev --filter=web",
     "dev:web-turbo": "turbo run dev --filter=web -- --turbo",
     "lint": "turbo run lint",
-    "format": "prettier --write \"**/*.{js,jsx,ts,tsx,css}\"",
-    "format-fast": "prettier --write \"**/*.{js,jsx,ts,tsx,css}\" --experimental-cli",
+    "format": "prettier --write \"**/*.{js,jsx,ts,tsx,css}\" --experimental-cli",
+    "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx,css}\" --experimental-cli",
     "test": "turbo run test",
     "release": "dotenv -e ../.env -- release-it",
     "prepare": "husky"
@@ -40,7 +40,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/up.sh

@@ -5,8 +5,30 @@
 
 # Check if CLICKHOUSE_URL is configured
 if [ -z "${CLICKHOUSE_URL}" ]; then
-  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
-  exit 0
+  echo "Error: CLICKHOUSE_URL is not configured."
+  echo "Please set CLICKHOUSE_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_MIGRATION_URL is configured
+if [ -z "${CLICKHOUSE_MIGRATION_URL}" ]; then
+  echo "Error: CLICKHOUSE_MIGRATION_URL is not configured."
+  echo "Please set CLICKHOUSE_MIGRATION_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_USER is set
+if [ -z "${CLICKHOUSE_USER}" ]; then
+  echo "Error: CLICKHOUSE_USER is not set."
+  echo "Please set CLICKHOUSE_USER in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_PASSWORD is set
+if [ -z "${CLICKHOUSE_PASSWORD}" ]; then
+  echo "Error: CLICKHOUSE_PASSWORD is not set."
+  echo "Please set CLICKHOUSE_PASSWORD in your environment variables."
+  exit 1
 fi
 
 # Check if golang-migrate is installed

packages/shared/package.json

@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -79,13 +79,13 @@
     "bullmq": "^5.34.10",
     "dd-trace": "^5.36.0",
     "decimal.js": "^10.4.3",
-    "exponential-backoff": "^3.1.1",
+    "exponential-backoff": "^3.1.2",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "jsonpath-plus": "10.3.0",
     "kysely": "^0.27.4",
     "langchain": "^0.3.28",
-    "langfuse-langchain": "3.38.2",
+    "langfuse-langchain": "3.38.4",
     "lodash": "^4.17.21",
     "lossless-json": "^4.1.1",
     "next-auth": "^4.24.11",
@@ -118,17 +118,10 @@
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
-    "tsx": "^4.19.1",
-    "typescript": "^5.4.5",
-    "vitest": "^2.1.2"
+    "typescript": "^5.4.5"
   },
   "peerDependencies": {
     "@types/react": "~18.2.79",
     "react": "~18.2.0"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -55,6 +55,7 @@ export type AnnotationQueueStatus =
 export const AnnotationQueueObjectType = {
   TRACE: "TRACE",
   OBSERVATION: "OBSERVATION",
+  SESSION: "SESSION",
 } as const;
 export type AnnotationQueueObjectType =
   (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
@@ -109,6 +110,13 @@ export const BlobStorageIntegrationType = {
 } as const;
 export type BlobStorageIntegrationType =
   (typeof BlobStorageIntegrationType)[keyof typeof BlobStorageIntegrationType];
+export const BlobStorageExportMode = {
+  FULL_HISTORY: "FULL_HISTORY",
+  FROM_TODAY: "FROM_TODAY",
+  FROM_CUSTOM_DATE: "FROM_CUSTOM_DATE",
+} as const;
+export type BlobStorageExportMode =
+  (typeof BlobStorageExportMode)[keyof typeof BlobStorageExportMode];
 export const DashboardWidgetViews = {
   TRACES: "TRACES",
   OBSERVATIONS: "OBSERVATIONS",
@@ -298,6 +306,8 @@ export type BlobStorageIntegration = {
   enabled: boolean;
   export_frequency: string;
   file_type: Generated<BlobStorageIntegrationFileType>;
+  export_mode: Generated<BlobStorageExportMode>;
+  export_start_date: Timestamp | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -350,6 +360,8 @@ export type Dataset = {
   name: string;
   description: string | null;
   metadata: unknown | null;
+  remote_experiment_url: string | null;
+  remote_experiment_payload: unknown | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -628,6 +640,7 @@ export type Price = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
   model_id: string;
+  project_id: string | null;
   usage_type: string;
   price: string;
 };

packages/shared/prisma/migrations/20250709113103_add_blob_export_schedule_type/migration.sql

@@ -0,0 +1,6 @@
+-- CreateEnum
+CREATE TYPE "BlobStorageExportMode" AS ENUM ('FULL_HISTORY', 'FROM_TODAY', 'FROM_CUSTOM_DATE');
+
+-- AlterTable
+ALTER TABLE "blob_storage_integrations" ADD COLUMN     "export_mode" "BlobStorageExportMode" NOT NULL DEFAULT 'FULL_HISTORY',
+ADD COLUMN     "export_start_date" TIMESTAMP(3);

packages/shared/prisma/migrations/20250711105322_prices_add_project_id/migration.sql

@@ -0,0 +1,13 @@
+-- AlterTable
+ALTER TABLE "prices"
+    ADD COLUMN "project_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "prices"
+    ADD CONSTRAINT "prices_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- BackfillData
+UPDATE "prices"
+SET "project_id" = (SELECT "models"."project_id"
+                    FROM "models"
+                    WHERE "models"."id" = "prices"."model_id");

packages/shared/prisma/migrations/20250711134738_add_patch_llm_tool_schema_audit_logs_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('3445cac4-d9d5-4750-8b65-351135c1b85e', '20250711_1347_patch_llm_tool_schema_audit_logs', 'patchLLMToolAndLLLMSchemaAuditLogs', '{}');

packages/shared/prisma/migrations/20250714151410_add_trace_session_combined_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "trace_sessions_project_id_created_at_idx" ON "trace_sessions"("project_id", "created_at" DESC);

packages/shared/prisma/migrations/20250714151410_remove_trace_session_created_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_created_at_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_project_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_project_id_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_updated_at_idx";

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/schema.prisma

@@ -162,6 +162,7 @@ model Project {
   automationExecutions   AutomationExecution[]
   Automation             Automation[]
   DefaultLlmModel        DefaultLlmModel[]
+  Price                  Price[]
 
   @@index([orgId])
   @@map("projects")
@@ -310,9 +311,7 @@ model TraceSession {
   environment String   @default("default")
 
   @@id([id, projectId])
-  @@index([projectId])
-  @@index([createdAt])
-  @@index([updatedAt])
+  @@index([projectId, createdAt(sort: Desc)])
   @@map("trace_sessions")
 }
 
@@ -540,6 +539,7 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
 }
 
 model CronJobs {
@@ -552,16 +552,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -755,6 +757,8 @@ model Price {
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
   modelId   String   @map("model_id") // Model is already linked to project (or default), so we don't need projectId here
   Model     Model    @relation(fields: [modelId], references: [id], onDelete: Cascade)
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
   usageType String   @map("usage_type")
   price     Decimal
 
@@ -967,6 +971,8 @@ model BlobStorageIntegration {
   enabled         Boolean
   exportFrequency String                         @map("export_frequency")
   fileType        BlobStorageIntegrationFileType @default(CSV) @map("file_type")
+  exportMode      BlobStorageExportMode          @default(FULL_HISTORY) @map("export_mode")
+  exportStartDate DateTime?                      @map("export_start_date")
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
@@ -990,6 +996,14 @@ enum BlobStorageIntegrationType {
   @@map("BlobStorageIntegrationType")
 }
 
+enum BlobStorageExportMode {
+  FULL_HISTORY
+  FROM_TODAY
+  FROM_CUSTOM_DATE
+
+  @@map("BlobStorageExportMode")
+}
+
 model BatchExport {
   id        String   @id @default(cuid())
   createdAt DateTime @default(now()) @map("created_at")

packages/shared/scripts/seeder/seed-postgres.ts

@@ -23,6 +23,7 @@ import {
   SEED_TEXT_PROMPTS,
 } from "./utils/postgres-seed-constants";
 import {
+  generateDatasetItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -517,6 +518,7 @@ export async function createDatasets(
             description: data.description,
             projectId,
             metadata: data.metadata,
+            id: `${datasetName}-${projectId.slice(-8)}`,
           },
         }));
 
@@ -532,13 +534,23 @@ export async function createDatasets(
         const datasetItem = await prisma.datasetItem.upsert({
           where: {
             id_projectId: {
-              id: `${dataset.id}-${index}`,
+              id: generateDatasetItemId(
+                datasetName,
+                index,
+                projectId,
+                SEED_DATASETS.indexOf(data) || 0,
+              ),
               projectId,
             },
           },
           create: {
             projectId,
-            id: `${dataset.id}-${index}`,
+            id: generateDatasetItemId(
+              datasetName,
+              index,
+              projectId,
+              SEED_DATASETS.indexOf(data) || 0,
+            ),
             datasetId: dataset.id,
             sourceTraceId: sourceTraceId ?? null,
             sourceObservationId: null,
@@ -554,14 +566,14 @@ export async function createDatasets(
       for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.upsert({
           where: {
-            datasetId_projectId_name: {
-              datasetId: dataset.id,
+            id_projectId: {
+              id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
               projectId,
-              name: `demo-dataset-run-${datasetRunNumber}`,
             },
           },
           create: {
             projectId,
+            id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -2,6 +2,8 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItemsCh,
 } from "../../../src/server";
 import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
 import {
@@ -42,6 +44,16 @@ export class ClickHouseQueryBuilder {
     return await createObservationsCh(observations);
   }
 
+  /**
+   * Creates INSERT query for dataset run items data using VALUES syntax.
+   * Use for: Small datasets, dataset run items that link to postgres data (e.g. dataset runs)
+   */
+  async executeDatasetRunItemsInsert(
+    datasetRunItems: DatasetRunItemRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createDatasetRunItemsCh(datasetRunItems);
+  }
+
   /**
    * Creates INSERT query for score data using VALUES syntax.
    * Use for: Small datasets, scores with custom values and metadata.

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -6,6 +6,8 @@ import {
   REALISTIC_MODELS,
 } from "./clickhouse-seed-constants";
 import {
+  generateDatasetItemId,
+  generateDatasetRunItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -23,6 +25,8 @@ import {
   ObservationRecordInsertType,
   ScoreRecordInsertType,
   TraceRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItem,
 } from "../../../src/server";
 
 /**
@@ -60,6 +64,49 @@ export class DataGenerator {
     return Math.floor(Math.random() * (max - min + 1)) + min;
   }
 
+  /**
+   * Creates dataset run items for dataset runs.
+   * Use for: Dataset experiment scenarios.
+   */
+  generateDatasetRunItem(
+    input: DatasetItemInput & { runCreatedAt: number },
+    projectId: string,
+  ): DatasetRunItemRecordInsertType {
+    const datasetRunItemId = generateDatasetRunItemId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    // TODO: there are too many dataset run items in the postgres database?
+    return createDatasetRunItem({
+      id: datasetRunItemId,
+      project_id: projectId,
+      trace_id: generateDatasetRunTraceId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_id: `${input.datasetName}-${projectId.slice(-8)}`,
+      dataset_run_id: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_name: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_created_at: input.runCreatedAt,
+      dataset_run_description:
+        (input.runNumber || 0) % 2 === 0 ? "Dataset run description" : "",
+      dataset_run_metadata: { key: "value" },
+      dataset_item_id: generateDatasetItemId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_item_input: input.item.input,
+      dataset_item_expected_output: input.item.output,
+    });
+  }
+
   /**
    * Creates traces from dataset items for experiment runs.
    * Use for: Dataset experiments scenarios.

packages/shared/scripts/seeder/utils/postgres-seed-constants.ts

@@ -271,6 +271,68 @@ export const SEED_TEXT_PROMPTS = [
     labels: ["production", "latest"],
     tags: ["tag1", "tag2"],
   },
+  {
+    id: `prompt-with-many-labels`,
+    createdBy: "user-1",
+    prompt:
+      "This is a comprehensive prompt for testing multiple label scenarios. It demonstrates how prompts can be tagged with numerous labels for organization, categorization, and filtering purposes. Use this prompt to understand how label management works at scale. Variables: {{input}}",
+    name: "prompt-with-many-labels",
+    version: 1,
+    labels: [
+      "production",
+      "latest",
+      "v1",
+      "v2",
+      "stable",
+      "beta",
+      "alpha",
+      "test",
+      "development",
+      "staging",
+      "experimental",
+      "feature",
+      "bugfix",
+      "hotfix",
+      "critical",
+      "high-priority",
+      "medium-priority",
+      "low-priority",
+      "urgent",
+      "customer-facing",
+      "internal",
+      "public",
+      "private",
+      "confidential",
+      "ai",
+      "nlp",
+      "chatbot",
+      "assistant",
+      "automation",
+      "ml",
+      "data",
+      "analytics",
+      "monitoring",
+      "logging",
+      "debug",
+      "performance",
+      "security",
+      "compliance",
+      "audit",
+      "review",
+      "approved",
+      "rejected",
+      "pending",
+      "archived",
+      "deprecated",
+      "legacy",
+      "migration",
+      "upgrade",
+      "downgrade",
+      "template",
+      "example",
+    ],
+    tags: [],
+  },
 ];
 
 export const SEED_CHAT_ML_PROMPTS = [

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -1,3 +1,21 @@
+export const generateDatasetRunItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-run-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateDatasetItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
 export const generateDatasetRunTraceId = (
   datasetName: string,
   itemIndex: number,

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -92,12 +92,26 @@ export class SeederOrchestrator {
         logger.info(
           `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
         );
+        // const now = Date.now();
 
         const traces: TraceRecordInsertType[] = [];
         const observations: ObservationRecordInsertType[] = [];
+        // const datasetRunItems: DatasetRunItemRecordInsertType[] = [];
 
         for (const seedDataset of SEED_DATASETS) {
           for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // // Generate dataset run item data
+            // const datasetRunItem = this.dataGenerator.generateDatasetRunItem(
+            //   {
+            //     datasetName: seedDataset.name,
+            //     itemIndex,
+            //     item: datasetItem,
+            //     runNumber,
+            //     runCreatedAt: now,
+            //   },
+            //   projectId,
+            // );
+
             // Generate trace data
             const trace = this.dataGenerator.generateDatasetTrace(
               {
@@ -123,12 +137,14 @@ export class SeederOrchestrator {
 
             traces.push(trace);
             observations.push(observation);
+            // datasetRunItems.push(datasetRunItem);
           }
         }
 
         try {
           await this.queryBuilder.executeTracesInsert(traces);
           await this.queryBuilder.executeObservationsInsert(observations);
+          // await this.queryBuilder.executeDatasetRunItemsInsert(datasetRunItems);
         } catch (error) {
           logger.error(`✗ Insert failed:`, error);
           throw error;

packages/shared/src/domain/automations.ts

@@ -44,17 +44,27 @@ export const AvailableWebhookApiSchema = z.record(
   z.enum(["v1"]),
 );
 
+export const RequestHeaderSchema = z.object({
+  secret: z.boolean(),
+  value: z.string(),
+});
+
 export const WebhookActionConfigSchema = z.object({
   type: z.literal("WEBHOOK"),
   url: z.url(),
   headers: z.record(z.string(), z.string()),
+  requestHeaders: z.record(z.string(), RequestHeaderSchema),
+  displayHeaders: z.record(z.string(), RequestHeaderSchema),
   apiVersion: AvailableWebhookApiSchema,
   secretKey: z.string(),
   displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
 });
 
 export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
   secretKey: true,
+  headers: true,
+  requestHeaders: true,
 });
 
 export type SafeWebhookActionConfig = z.infer<
@@ -64,6 +74,8 @@ export type SafeWebhookActionConfig = z.infer<
 export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
   secretKey: true,
   displaySecretKey: true,
+  headers: true, // don't use legacy field anymore
+  displayHeaders: true,
 });
 
 export const ActionConfigSchema = z.discriminatedUnion("type", [

packages/shared/src/domain/dataset-run-items.ts

@@ -0,0 +1,28 @@
+import z from "zod/v4";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const DatasetRunItemSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  datasetRunId: z.string(),
+  datasetItemId: z.string(),
+  datasetId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().nullable(),
+  error: z.string().nullable(),
+  // timestamps
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  // dataset run fields
+  datasetRunName: z.string(),
+  datasetRunDescription: z.string().nullable(),
+  datasetRunMetadata: MetadataDomain,
+  datasetRunCreatedAt: z.date(),
+  // dataset item fields
+  datasetItemInput: jsonSchema,
+  datasetItemExpectedOutput: jsonSchema,
+  datasetItemMetadata: MetadataDomain,
+});
+
+export type DatasetRunItemDomain = z.infer<typeof DatasetRunItemSchema>;

packages/shared/src/domain/webhooks.ts

@@ -3,8 +3,8 @@ import { jsonSchema } from "../utils/zod";
 import { EventActionSchema } from "./automations";
 
 export const WebhookDefaultHeaders = {
-  "Content-Type": "application/json",
-  "User-Agent": "Langfuse/1.0",
+  "content-type": "application/json",
+  "user-agent": "Langfuse/1.0",
 };
 
 export const WebhookOutboundBaseSchema = z.object({

packages/shared/src/env.ts

@@ -16,6 +16,7 @@ const EnvSchema = z.object({
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
+  REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
   REDIS_TLS_CA_PATH: z.string().optional(),
   REDIS_TLS_CERT_PATH: z.string().optional(),
@@ -31,8 +32,10 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
     )
     .optional(),
-  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
+  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(300), // 5 minutes
   CLICKHOUSE_URL: z.string().url(),
   CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
   CLICKHOUSE_DB: z.string().default("default"),
@@ -107,7 +110,13 @@ const EnvSchema = z.object({
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
   LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
   LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
-
+  // Dataset Run Items Migration Environment Variables
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH: z
+    .enum(["true", "false"])
+    .default("false"),
   LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES: z
     .enum(["true", "false"])
     .default("false"),
@@ -131,6 +140,38 @@ const EnvSchema = z.object({
   LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/annotation/types.ts

@@ -1,5 +1,6 @@
 import z from "zod/v4";
 import { type ScoreDataType } from "../../db";
+import { StringNoHTML, StringNoHTMLNonEmpty } from "../../utils/zod";
 
 const NUMERIC: ScoreDataType = "NUMERIC";
 const CATEGORICAL: ScoreDataType = "CATEGORICAL";
@@ -47,12 +48,12 @@ export type ScoreTargetSession = z.infer<typeof ScoreTargetSession>;
 export type ScoreTarget = z.infer<typeof ScoreTarget>;
 
 const CreateAnnotationScoreBase = z.object({
-  name: z.string(),
+  name: StringNoHTMLNonEmpty,
   projectId: z.string(),
   environment: z.string().default("default"),
   scoreTarget: ScoreTarget,
   configId: z.string().optional(),
-  comment: z.string().nullish(),
+  comment: StringNoHTML.nullish(),
   queueId: z.string().nullish(),
 });
 
@@ -83,8 +84,8 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
 // annotation queues
 
 export const CreateQueueData = z.object({
-  name: z.string().min(1).max(35),
-  description: z.string().max(1000).optional(),
+  name: StringNoHTMLNonEmpty.max(35),
+  description: StringNoHTML.max(1000).optional(),
   scoreConfigIds: z.array(z.string()).min(1, {
     message: "At least 1 score config must be selected",
   }),

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,7 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/features/prompts/validation.ts

@@ -1,12 +1,12 @@
-import { z } from "zod/v4";
+import { StringNoHTMLNonEmpty } from "../../utils/zod";
 
 /**
  * Prompt name validation schema for API, tRPC and client
  */
-export const PromptNameSchema = z
-  .string()
-  .min(1, "Enter a name")
-  .regex(/^[^|]*$/, "Prompt name cannot contain '|' character")
+export const PromptNameSchema = StringNoHTMLNonEmpty.regex(
+  /^[^|]*$/,
+  "Prompt name cannot contain '|' character",
+)
   .regex(/^[^/]/, "Name cannot start with a slash")
   .regex(/^(?!.*\/\/)/, "Name cannot contain consecutive slashes")
   .regex(/^.*[^/]$/, "Name cannot end with a slash")

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -1,5 +1,9 @@
 import { z } from "zod/v4";
 
+// Sentinel value for Bedrock default credential provider chain
+export const BEDROCK_USE_DEFAULT_CREDENTIALS =
+  "__BEDROCK_DEFAULT_CREDENTIALS__";
+
 export const BedrockConfigSchema = z.object({ region: z.string() });
 export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
 

packages/shared/src/server/clickhouse/schemaUtils.ts

@@ -11,7 +11,8 @@ export type IngestionEntityTypes =
   | "trace"
   | "observation"
   | "score"
-  | "sdk_log";
+  | "sdk_log"
+  | "dataset_run_item";
 
 export const getClickhouseEntityType = (
   eventType: string,
@@ -29,6 +30,8 @@ export const getClickhouseEntityType = (
       return "observation";
     case eventTypes.SCORE_CREATE:
       return "score";
+    case eventTypes.DATASET_RUN_ITEM_CREATE:
+      return "dataset_run_item";
     case eventTypes.SDK_LOG:
       return "sdk_log";
     default:

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/dataset-run-items/datasetExecution.ts

@@ -0,0 +1,94 @@
+import { env } from "../../env";
+import { logger } from "../../server/logger";
+import {
+  DatasetRunItemsExecutionStrategy,
+  DatasetRunItemsOperationType,
+} from "./types";
+/**
+ * Returns the execution strategy for dataset run items based on environment variables.
+ *
+ * Two-phase migration approach:
+ * 1. Dual-write phase: DATASET_RUN_ITEMS_WRITE_TO_CLICKHOUSE=true (write to both databases)
+ * 2. Read migration phase: DATASET_RUN_ITEMS_READ_FROM_CLICKHOUSE=true (read from ClickHouse)
+ */
+function getDatasetRunItemsExecutionStrategy(): DatasetRunItemsExecutionStrategy {
+  return {
+    shouldWriteToClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH === "true",
+    shouldReadFromClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH === "true",
+  };
+}
+
+// Re-export the enum for backward compatibility
+
+/**
+ * Executes the appropriate database operation based on the execution strategy.
+ *
+ * @param postgresExecution - Function to execute PostgreSQL operation
+ * @param clickhouseExecution - Function to execute ClickHouse operation
+ * @param operationType - Type of operation ("read" or "write")
+ * @returns Result from the selected execution strategy
+ */
+export async function executeWithDatasetRunItemsStrategy<TInput, TOutput>({
+  input,
+  operationType,
+  postgresExecution,
+  clickhouseExecution,
+}: {
+  input: TInput;
+  operationType: DatasetRunItemsOperationType;
+  // eslint-disable-next-line no-unused-vars
+  postgresExecution: (input: TInput) => Promise<TOutput>;
+  // eslint-disable-next-line no-unused-vars
+  clickhouseExecution: (input: TInput) => Promise<TOutput>;
+}): Promise<TOutput> {
+  const strategy = getDatasetRunItemsExecutionStrategy();
+
+  if (operationType === DatasetRunItemsOperationType.WRITE) {
+    // For write operations, implement dual-write strategy
+    if (strategy.shouldWriteToClickHouse) {
+      // Dual-write phase: write to both databases
+      const postgresResult = await postgresExecution(input);
+
+      try {
+        await clickhouseExecution(input);
+        logger.debug("Successfully wrote to both PostgreSQL and ClickHouse", {
+          operation: `dataset_run_items_${operationType}`,
+        });
+      } catch (error) {
+        logger.error("ClickHouse write failed during dual-write phase", {
+          error: error instanceof Error ? error.message : String(error),
+          operation: `dataset_run_items_${operationType}`,
+        });
+        // Continue with PostgreSQL result since it succeeded
+      }
+
+      return postgresResult;
+    } else {
+      // Write only to PostgreSQL
+      return await postgresExecution(input);
+    }
+  } else {
+    // For read operations, rely on the strategy
+    const shouldExecuteClickhouse = strategy.shouldReadFromClickHouse;
+
+    if (shouldExecuteClickhouse) {
+      try {
+        return await clickhouseExecution(input);
+      } catch (error) {
+        logger.error(
+          "ClickHouse execution failed, falling back to PostgreSQL",
+          {
+            error: error instanceof Error ? error.message : String(error),
+            operation: `dataset_run_items_${operationType}`,
+          },
+        );
+        // Fallback to PostgreSQL for reliability
+        return await postgresExecution(input);
+      }
+    } else {
+      return await postgresExecution(input);
+    }
+  }
+}

packages/shared/src/server/dataset-run-items/types.ts

@@ -0,0 +1,16 @@
+/**
+ * Types and enums for dataset run items execution.
+ * This file is frontend-safe and doesn't import server-side dependencies.
+ */
+
+export enum DatasetRunItemsOperationType {
+  // eslint-disable-next-line no-unused-vars
+  READ = "read",
+  // eslint-disable-next-line no-unused-vars
+  WRITE = "write",
+}
+
+export type DatasetRunItemsExecutionStrategy = {
+  shouldWriteToClickHouse: boolean;
+  shouldReadFromClickHouse: boolean;
+};

packages/shared/src/server/index.ts

@@ -20,8 +20,8 @@ export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
+export * from "../server/ingestion/modelMatch";
 export * from "./ingestion/processEventBatch";
-export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
@@ -32,6 +32,7 @@ export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -63,11 +64,14 @@ export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
 export * from "./clickhouse/measureAndReturn";
-
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
-export * from "./automations/webhooks";
+// dataset run items
+export * from "./dataset-run-items/datasetExecution";
+export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
 
 // test utils
 export * from "./test-utils";
+export * from "./utils/headerUtils";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -1,13 +1,15 @@
-import { Model, Prisma } from "@langfuse/shared";
+import { Model, Prisma } from "../../";
 import {
   instrumentAsync,
   logger,
   recordIncrement,
-} from "@langfuse/shared/src/server";
-import { env } from "../env";
-import { redis } from "@langfuse/shared/src/server";
+  redis,
+  safeMultiDel,
+} from "../";
+import { type Cluster } from "ioredis";
+import { env } from "../../env";
 import { Decimal } from "decimal.js";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma } from "../../db";
 
 export type ModelMatchProps = {
   projectId: string;
@@ -81,7 +83,7 @@ const getModelFromRedis = async (
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;
@@ -178,6 +180,11 @@ export const getRedisModelKey = (p: ModelMatchProps) => {
 };
 
 const getModelMatchKeyPrefix = () => {
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // Use hash tags for Redis cluster compatibility
+    // This ensures all model cache keys are placed on the same hash slot
+    return "{model-match}";
+  }
   return "model-match";
 };
 
@@ -205,3 +212,35 @@ export const redisModelToPrismaModel = (redisModel: string): Model => {
         : null,
   };
 };
+
+export async function clearModelCacheForProject(
+  projectId: string,
+): Promise<void> {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    const pattern = `${getModelMatchKeyPrefix()}:${projectId}:*`;
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared ${keys.length} model cache entries for project ${projectId}`,
+      );
+    }
+  } catch (error) {
+    logger.error(`Error clearing model cache for project ${projectId}`, error);
+  }
+}

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -21,11 +21,16 @@ import { logger } from "../logger";
 import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
 import { redis } from "../redis/redis";
-import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+import {
+  eventTypes,
+  createIngestionEventSchema,
+  IngestionEventType,
+} from "./types";
 import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -57,7 +62,7 @@ export type TokenCountDelegate = (p: {
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -69,24 +74,38 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
   return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
 };
 
+/**
+ * Options for event batch processing.
+ * @property delay - Delay in ms to wait before processing events in the batch.
+ * @property source - Source of the events for metrics tracking (e.g., "otel", "api").
+ * @property isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ */
+type ProcessEventBatchOptions = {
+  delay?: number | null;
+  source?: "api" | "otel";
+  isLangfuseInternal?: boolean;
+};
+
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
  * @param authCheck - AuthHeaderValidVerificationResult
- * @param delay - (Optional) Delay in ms to wait before processing events in the batch.
- * @param source - (Optional) Source of the events for metrics tracking (e.g., "otel", "api").
+ * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
   authCheck: AuthHeaderValidVerificationResult,
-  delay: number | null = null,
-  source: "api" | "otel" = "api",
+  options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
   errors: {
@@ -96,6 +115,8 @@ export const processEventBatch = async (
     error?: string;
   }[];
 }> => {
+  const { delay = null, source = "api", isLangfuseInternal = false } = options;
+
   // add context of api call to the span
   const currentSpan = getCurrentSpan();
   recordIncrement("langfuse.ingestion.event", input.length, { source });
@@ -121,9 +142,10 @@ export const processEventBatch = async (
   const validationErrors: { id: string; error: unknown }[] = [];
   const authenticationErrors: { id: string; error: unknown }[] = [];
 
-  const batch: z.infer<typeof ingestionEvent>[] = input
+  const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
+  const batch: z.infer<typeof ingestionSchema>[] = input
     .flatMap((event) => {
-      const parsed = ingestionEvent.safeParse(event);
+      const parsed = ingestionSchema.safeParse(event);
       if (!parsed.success) {
         validationErrors.push({
           id:
@@ -239,11 +261,39 @@ export const processEventBatch = async (
       const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
       const queue = IngestionQueue.getInstance({ shardingKey });
 
-      const shouldSkipS3List =
-        getClickhouseEntityType(eventData.type) === "observation" &&
+      const isDatasetRunItemEvent =
+        getClickhouseEntityType(eventData.type) === "dataset_run_item";
+      const isObservationEvent =
+        getClickhouseEntityType(eventData.type) === "observation";
+
+      const isOtelOrSkipS3Project =
         authCheck.scope.projectId !== null &&
-        (projectIdsToSkipS3List.includes(authCheck.scope.projectId) ||
-          source === "otel");
+        (source === "otel" ||
+          projectIdsToSkipS3List.includes(authCheck.scope.projectId));
+
+      const shouldSkipS3List =
+        isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
+
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
 
       return queue
         ? queue.add(
@@ -268,7 +318,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -302,7 +352,7 @@ const isAuthorized = (
 /**
  * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
  */
-const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+const sortBatch = (batch: IngestionEventType[]) => {
   const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
     eventTypes.GENERATION_UPDATE,
     eventTypes.SPAN_UPDATE,

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/ingestion/types.ts

@@ -193,178 +193,42 @@ export const UsageDetails = z
   ])
   .nullish();
 
-export const EnvironmentName = z
+const INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  "Only alphanumeric lower case characters, hyphens, and underscores are allowed";
+
+const ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE +
+  " and it must not start with 'langfuse'";
+
+const PublicEnvironmentName = z
   .string()
   .max(40, "Maximum length is 40 characters")
-  .regex(
-    /^(?!langfuse)[a-z0-9-_]+$/,
-    "Only alphanumeric lower case characters, hyphens, and underscores are allowed, and it must not start with 'langfuse'",
-  )
+  .regex(/^(?!langfuse)[a-z0-9-_]+$/, ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
   .default("default");
 
-// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
-// With this setup parsing should be more lightweight and doesn't block other requests.
-// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
-export const TraceBody = z.object({
-  id: idSchema.nullish(),
-  timestamp: stringDateTime,
-  name: z.string().max(1000).nullish(),
-  externalId: z.string().nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  sessionId: z.string().nullish(),
-  userId: z.string().nullish(),
-  environment: EnvironmentName,
-  metadata: jsonSchema.nullish(),
-  release: z.string().nullish(),
-  version: z.string().nullish(),
-  public: z.boolean().nullish(),
-  tags: z.array(z.string()).nullish(),
-});
+const InternalEnvironmentName = z
+  .string()
+  .max(40, "Maximum length is 40 characters")
+  .regex(/^[a-z0-9-_]+$/, INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
+  .default("default");
 
-export const OptionalObservationBody = z.object({
-  traceId: idSchema.nullish(),
-  environment: EnvironmentName,
-  name: z.string().nullish(),
-  startTime: stringDateTime,
-  metadata: jsonSchema.nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  level: ObservationLevel.nullish(),
-  statusMessage: z.string().nullish(),
-  parentObservationId: z.string().nullish(),
-  version: z.string().nullish(),
-});
+/** @deprecated Use PublicEnvironmentName or InternalEnvironmentName instead */
+export const EnvironmentName = PublicEnvironmentName;
 
-export const CreateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const UpdateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const CreateSpanBody = CreateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const UpdateSpanBody = UpdateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const CreateGenerationBody = CreateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-export const UpdateGenerationBody = UpdateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-const BaseScoreBody = z.object({
-  id: idSchema.nullish(),
-  name: NonEmptyString,
-  traceId: z.string().nullish(),
-  sessionId: z.string().nullish(),
-  datasetRunId: z.string().nullish(),
-  environment: EnvironmentName,
-  observationId: z.string().nullish(),
-  comment: z.string().nullish(),
-  metadata: jsonSchema.nullish(),
-  source: z
-    .enum(["API", "EVAL", "ANNOTATION"])
-    .default("API" as ScoreSourceType),
-});
-
-/**
- * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
- */
-export const ScoreBody = applyScoreValidation(
-  z.discriminatedUnion("dataType", [
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number(),
-        dataType: z.literal("NUMERIC"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.string(),
-        dataType: z.literal("CATEGORICAL"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number().refine((value) => value === 0 || value === 1, {
-          message:
-            "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
-        }),
-        dataType: z.literal("BOOLEAN"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.union([z.string(), z.number()]),
-        dataType: z.undefined(),
-        configId: z.string().nullish(),
-      }),
-    ),
-  ]),
-);
+export const eventTypes = {
+  TRACE_CREATE: "trace-create",
+  SCORE_CREATE: "score-create",
+  EVENT_CREATE: "event-create",
+  SPAN_CREATE: "span-create",
+  SPAN_UPDATE: "span-update",
+  GENERATION_CREATE: "generation-create",
+  GENERATION_UPDATE: "generation-update",
+  SDK_LOG: "sdk-log",
+  DATASET_RUN_ITEM_CREATE: "dataset-run-item-create",
+  // LEGACY, only required for backwards compatibility
+  OBSERVATION_CREATE: "observation-create",
+  OBSERVATION_UPDATE: "observation-update",
+} as const;
 
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
@@ -475,83 +339,366 @@ export const SdkLogEvent = z.object({
   id: z.string().nullish(), // Not used, but makes downstream processing easier.
 });
 
-export const eventTypes = {
-  TRACE_CREATE: "trace-create",
-  SCORE_CREATE: "score-create",
-  EVENT_CREATE: "event-create",
-  SPAN_CREATE: "span-create",
-  SPAN_UPDATE: "span-update",
-  GENERATION_CREATE: "generation-create",
-  GENERATION_UPDATE: "generation-update",
-  SDK_LOG: "sdk-log",
-  // LEGACY, only required for backwards compatibility
-  OBSERVATION_CREATE: "observation-create",
-  OBSERVATION_UPDATE: "observation-update",
-} as const;
+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 
-const base = z.object({
-  id: idSchema,
-  timestamp: z.string().datetime({ offset: true }),
-  metadata: jsonSchema.nullish(),
-});
-export const traceEvent = base.extend({
-  type: z.literal(eventTypes.TRACE_CREATE),
-  body: TraceBody,
-});
-export type TraceEventType = z.infer<typeof traceEvent>;
+// Complete schema factory - single source of truth for ALL schemas
+const createAllIngestionSchemas = ({
+  isPublic = true,
+}: {
+  isPublic: boolean;
+}) => {
+  const environmentSchema = isPublic
+    ? PublicEnvironmentName
+    : InternalEnvironmentName;
 
-export const eventCreateEvent = base.extend({
-  type: z.literal(eventTypes.EVENT_CREATE),
-  body: CreateEventEvent,
-});
-export const spanCreateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_CREATE),
-  body: CreateSpanBody,
-});
-export const spanUpdateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_UPDATE),
-  body: UpdateSpanBody,
-});
-export const generationCreateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_CREATE),
-  body: CreateGenerationBody,
-});
-export const generationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_UPDATE),
-  body: UpdateGenerationBody,
-});
-export const scoreEvent = base.extend({
-  type: z.literal(eventTypes.SCORE_CREATE),
-  body: ScoreBody,
-});
-export type ScoreEventType = z.infer<typeof scoreEvent>;
-export const sdkLogEvent = base.extend({
-  type: z.literal(eventTypes.SDK_LOG),
-  body: SdkLogEvent,
-});
-export const legacyObservationCreateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_CREATE),
-  body: LegacyObservationBody,
-});
-export const legacyObservationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_UPDATE),
-  body: LegacyObservationBody,
-});
+  // Base schemas with environment
+  const TraceBody = z.object({
+    id: idSchema.nullish(),
+    timestamp: stringDateTime,
+    name: z.string().max(1000).nullish(),
+    externalId: z.string().nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    sessionId: z.string().nullish(),
+    userId: z.string().nullish(),
+    environment: environmentSchema,
+    metadata: jsonSchema.nullish(),
+    release: z.string().nullish(),
+    version: z.string().nullish(),
+    public: z.boolean().nullish(),
+    tags: z.array(z.string()).nullish(),
+  });
 
-export const ingestionEvent = z.discriminatedUnion("type", [
-  traceEvent,
-  scoreEvent,
-  eventCreateEvent,
-  spanCreateEvent,
-  spanUpdateEvent,
-  generationCreateEvent,
-  generationUpdateEvent,
-  sdkLogEvent,
-  // LEGACY, only required for backwards compatibility
-  legacyObservationCreateEvent,
-  legacyObservationUpdateEvent,
-]);
+  const OptionalObservationBody = z.object({
+    traceId: idSchema.nullish(),
+    environment: environmentSchema,
+    name: z.string().nullish(),
+    startTime: stringDateTime,
+    metadata: jsonSchema.nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    level: ObservationLevel.nullish(),
+    statusMessage: z.string().nullish(),
+    parentObservationId: z.string().nullish(),
+    version: z.string().nullish(),
+  });
+
+  // Derivative schemas
+  const CreateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const UpdateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const CreateSpanBody = CreateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const UpdateSpanBody = UpdateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const CreateGenerationBody = CreateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const UpdateGenerationBody = UpdateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const BaseScoreBody = z.object({
+    id: idSchema.nullish(),
+    name: NonEmptyString,
+    traceId: z.string().nullish(),
+    sessionId: z.string().nullish(),
+    datasetRunId: z.string().nullish(),
+    environment: environmentSchema,
+    observationId: z.string().nullish(),
+    comment: z.string().nullish(),
+    metadata: jsonSchema.nullish(),
+    source: z
+      .enum(["API", "EVAL", "ANNOTATION"])
+      .default("API" as ScoreSourceType),
+  });
+
+  const ScoreBody = applyScoreValidation(
+    z.discriminatedUnion("dataType", [
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number(),
+          dataType: z.literal("NUMERIC"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.string(),
+          dataType: z.literal("CATEGORICAL"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number().refine((value) => value === 0 || value === 1, {
+            message:
+              "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
+          }),
+          dataType: z.literal("BOOLEAN"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.union([z.string(), z.number()]),
+          dataType: z.undefined(),
+          configId: z.string().nullish(),
+        }),
+      ),
+    ]),
+  );
+
+  const DatasetRunItemBody = z.object({
+    // Core identifiers
+    id: idSchema.nullish(),
+    traceId: z.string(),
+    observationId: z.string().nullish(),
+    error: z.string().nullish(),
+    // Metadata (optional)
+    createdAt: stringDateTime.nullish(),
+    // Dataset identification
+    datasetId: z.string(),
+    // Run identification
+    runId: z.string(),
+    // Dataset item identification
+    datasetItemId: z.string(),
+  });
+
+  // Event schemas
+  const base = z.object({
+    id: idSchema,
+    timestamp: z.string().datetime({ offset: true }),
+    metadata: jsonSchema.nullish(),
+  });
+
+  const traceEvent = base.extend({
+    type: z.literal(eventTypes.TRACE_CREATE),
+    body: TraceBody,
+  });
+
+  const eventCreateEvent = base.extend({
+    type: z.literal(eventTypes.EVENT_CREATE),
+    body: CreateEventEvent,
+  });
+
+  const spanCreateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_CREATE),
+    body: CreateSpanBody,
+  });
+
+  const spanUpdateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_UPDATE),
+    body: UpdateSpanBody,
+  });
+
+  const generationCreateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_CREATE),
+    body: CreateGenerationBody,
+  });
+
+  const generationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_UPDATE),
+    body: UpdateGenerationBody,
+  });
+
+  const scoreEvent = base.extend({
+    type: z.literal(eventTypes.SCORE_CREATE),
+    body: ScoreBody,
+  });
+
+  const baseDatasetRunItemCreateEvent = base.extend({
+    type: z.literal(eventTypes.DATASET_RUN_ITEM_CREATE),
+    body: DatasetRunItemBody,
+  });
+
+  const datasetRunItemCreateEvent = isPublic
+    ? baseDatasetRunItemCreateEvent.refine(() => false, {
+        message: "Dataset run item creation is only allowed for internal usage",
+      })
+    : baseDatasetRunItemCreateEvent;
+
+  const sdkLogEvent = base.extend({
+    type: z.literal(eventTypes.SDK_LOG),
+    body: SdkLogEvent,
+  });
+
+  const legacyObservationCreateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_CREATE),
+    body: LegacyObservationBody,
+  });
+
+  const legacyObservationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_UPDATE),
+    body: LegacyObservationBody,
+  });
+
+  const ingestionEvent = z.discriminatedUnion("type", [
+    traceEvent,
+    scoreEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    sdkLogEvent,
+    datasetRunItemCreateEvent,
+    // LEGACY, only required for backwards compatibility
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+  ]);
+
+  return {
+    // Body schemas
+    TraceBody,
+    OptionalObservationBody,
+    CreateEventEvent,
+    UpdateEventEvent,
+    CreateSpanBody,
+    UpdateSpanBody,
+    CreateGenerationBody,
+    UpdateGenerationBody,
+    BaseScoreBody,
+    ScoreBody,
+    // Event schemas
+    traceEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    scoreEvent,
+    datasetRunItemCreateEvent,
+    sdkLogEvent,
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+    // Complete schema
+    ingestionEvent,
+  };
+};
+
+// Create both public and internal schema instances
+const publicSchemas = createAllIngestionSchemas({ isPublic: true });
+const internalSchemas = createAllIngestionSchemas({ isPublic: false });
+
+// Export individual schemas for backwards compatibility
+export const TraceBody = publicSchemas.TraceBody;
+export const OptionalObservationBody = publicSchemas.OptionalObservationBody;
+export const CreateEventEvent = publicSchemas.CreateEventEvent;
+export const UpdateEventEvent = publicSchemas.UpdateEventEvent;
+export const CreateSpanBody = publicSchemas.CreateSpanBody;
+export const UpdateSpanBody = publicSchemas.UpdateSpanBody;
+export const CreateGenerationBody = publicSchemas.CreateGenerationBody;
+export const UpdateGenerationBody = publicSchemas.UpdateGenerationBody;
+export const BaseScoreBody = publicSchemas.BaseScoreBody;
+
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = publicSchemas.ScoreBody;
+
+// Export individual event schemas for backwards compatibility
+export const traceEvent = publicSchemas.traceEvent;
+export const eventCreateEvent = publicSchemas.eventCreateEvent;
+export const spanCreateEvent = publicSchemas.spanCreateEvent;
+export const spanUpdateEvent = publicSchemas.spanUpdateEvent;
+export const generationCreateEvent = publicSchemas.generationCreateEvent;
+export const generationUpdateEvent = publicSchemas.generationUpdateEvent;
+export const scoreEvent = publicSchemas.scoreEvent;
+export const sdkLogEvent = publicSchemas.sdkLogEvent;
+export const datasetRunItemCreateEvent =
+  publicSchemas.datasetRunItemCreateEvent;
+export const legacyObservationCreateEvent =
+  publicSchemas.legacyObservationCreateEvent;
+export const legacyObservationUpdateEvent =
+  publicSchemas.legacyObservationUpdateEvent;
+
+/** @deprecated Use createIngestionEventSchema() instead */
+export const ingestionEvent = publicSchemas.ingestionEvent;
+
+/**
+ * Type definitions for both schema variants (public and internal).
+ * These types are equivalent to the return types of createIngestionEventSchema() and all exported schemas,
+ * since the factory patterns only differ in environment validation rules, not in the actual TypeScript types.
+ * The environment field remains `string` in all cases - only the validation logic differs.
+ */
 export type IngestionEventType = z.infer<typeof ingestionEvent>;
+export type TraceEventType = z.infer<typeof traceEvent>;
+export type ScoreEventType = z.infer<typeof scoreEvent>;
+export type DatasetRunItemEventType = z.infer<typeof datasetRunItemCreateEvent>;
+
+/**
+ * Creates an ingestion event schema with appropriate environment validation.
+ * @param isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ * @returns The ingestion event schema.
+ */
+export const createIngestionEventSchema = (isLangfuseInternal = false) => {
+  return isLangfuseInternal
+    ? internalSchemas.ingestionEvent
+    : publicSchemas.ingestionEvent;
+};
 
 export type ObservationEvent =
   | z.infer<typeof legacyObservationCreateEvent>

packages/shared/src/server/llm/compileChatMessages.ts

@@ -7,10 +7,9 @@ import {
   type PromptChatMessageSchema,
   type ChatMessageWithId,
   type ChatMessageWithIdNoPlaceholders,
-  ChatMessageSchema,
 } from "./types";
 
-export type MessagePlaceholderValues = Record<string, ChatMessage[]>;
+export type MessagePlaceholderValues = Record<string, unknown[]>;
 export type PromptMessage = z.infer<typeof PromptChatMessageSchema>;
 
 export function isPlaceholder(
@@ -19,10 +18,6 @@ export function isPlaceholder(
   return "type" in message && message.type === ChatMessageType.Placeholder;
 }
 
-function validateMessage(message: unknown): message is ChatMessage {
-  return ChatMessageSchema.safeParse(message).success;
-}
-
 function replaceTextVariables(
   content: string,
   textVariables: Record<string, string>,
@@ -54,14 +49,17 @@ function expandPlaceholder(
     );
   }
 
-  for (const replacementMsg of replacementMessages) {
-    if (!validateMessage(replacementMsg)) {
-      throw new Error(
-        `Invalid message format in placeholder '${placeholder.name}': messages must have 'role' and 'content' properties`,
-      );
+  // Allow arbitrary objects - just pass them through as ChatMessage
+  // Users might want to use ChatML with placeholders for any message structure
+  return replacementMessages.map((replacementMsg) => {
+    if (typeof replacementMsg === "object" && replacementMsg !== null) {
+      return replacementMsg as ChatMessage;
     }
-  }
-  return replacementMessages;
+
+    throw new Error(
+      `Invalid message in placeholder '${placeholder.name}': expected object but got ${typeof replacementMsg}`,
+    );
+  });
 }
 
 export function compileChatMessages(
@@ -81,7 +79,7 @@ export function compileChatMessages(
   }
 
   return expandedMessages.map((message) => {
-    if (!message.content) {
+    if (!message.content || typeof message.content !== "string") {
       return message;
     }
 
@@ -94,7 +92,7 @@ export function compileChatMessages(
 
 export function compileChatMessagesWithIds(
   messages: ChatMessageWithId[],
-  placeholderValues: Record<string, ChatMessage[]>,
+  placeholderValues: MessagePlaceholderValues,
   textVariables?: Record<string, string>,
 ): ChatMessageWithIdNoPlaceholders[] {
   // TODO: check, is it even important to retain the IDs?
@@ -114,7 +112,7 @@ export function compileChatMessagesWithIds(
   }
 
   return expandedMessages.map((message) => {
-    if (!message.content) {
+    if (!message.content || typeof message.content !== "string") {
       return message;
     }
 

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -19,10 +19,12 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
+import { env } from "../../env";
 import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
   VertexAIConfigSchema,
+  BEDROCK_USE_DEFAULT_CREDENTIALS,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
@@ -41,6 +43,8 @@ import {
 import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
+const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
+
 type ProcessTracedEvents = () => Promise<void>;
 
 type LLMCompletionParams = {
@@ -137,7 +141,7 @@ export async function fetchLLMCompletion(
     const handler = new CallbackHandler({
       _projectId: traceParams.projectId,
       _isLocalEventExportEnabled: true,
-      tags: traceParams.tags,
+      environment: traceParams.environment,
     });
     finalCallbacks.push(handler);
 
@@ -149,6 +153,7 @@ export async function fetchLLMCompletion(
         await processEventBatch(
           JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
           traceParams.authCheck,
+          { isLangfuseInternal: true },
         );
       } catch (e) {
         logger.error("Failed to process traced events", { error: e });
@@ -158,28 +163,48 @@ export async function fetchLLMCompletion(
 
   finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
 
+  // Helper function to safely stringify content
+  const safeStringify = (content: any): string => {
+    try {
+      return JSON.stringify(content);
+    } catch {
+      return "[Unserializable content]";
+    }
+  };
+
   let finalMessages: BaseMessage[];
   // VertexAI requires at least 1 user message
   if (modelParams.adapter === LLMAdapter.VertexAI && messages.length === 1) {
-    finalMessages = [new HumanMessage(messages[0].content)];
+    const safeContent =
+      typeof messages[0].content === "string"
+        ? messages[0].content
+        : JSON.stringify(messages[0].content);
+    finalMessages = [new HumanMessage(safeContent)];
   } else {
     finalMessages = messages.map((message) => {
+      // For arbitrary content types, convert to string safely
+      const safeContent =
+        typeof message.content === "string"
+          ? message.content
+          : safeStringify(message.content);
+
       if (message.role === ChatMessageRole.User)
-        return new HumanMessage(message.content);
+        return new HumanMessage(safeContent);
       if (
         message.role === ChatMessageRole.System ||
         message.role === ChatMessageRole.Developer
       )
-        return new SystemMessage(message.content);
+        return new SystemMessage(safeContent);
 
-      if (message.type === ChatMessageType.ToolResult)
+      if (message.type === ChatMessageType.ToolResult) {
         return new ToolMessage({
-          content: message.content,
+          content: safeContent,
           tool_call_id: message.toolCallId,
         });
+      }
 
       return new AIMessage({
-        content: message.content,
+        content: safeContent,
         tool_calls:
           message.type === ChatMessageType.AssistantToolCall
             ? (message.toolCalls as any)
@@ -243,7 +268,11 @@ export async function fetchLLMCompletion(
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
-    const credentials = BedrockCredentialSchema.parse(JSON.parse(apiKey));
+    // Handle both explicit credentials and default provider chain
+    const credentials =
+      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS && !isLangfuseCloud
+        ? undefined // undefined = use AWS SDK default credential provider chain
+        : BedrockCredentialSchema.parse(JSON.parse(apiKey));
 
     chatModel = new ChatBedrockConverse({
       model: modelParams.model,

packages/shared/src/server/llm/types.ts

@@ -196,7 +196,7 @@ export const ChatMessageSchema = z.union([
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
-      content: z.string(),
+      content: z.union([z.string(), z.array(z.any()), z.any()]), // Support arbitrary content types for message placeholders
     })
     .transform((msg) => {
       return {
@@ -233,11 +233,6 @@ export enum LLMAdapter {
   GoogleAIStudio = "google-ai-studio",
 }
 
-export const SYSTEM_ROLES: string[] = [
-  ChatMessageRole.System,
-  ChatMessageRole.Developer,
-];
-
 export const TextPromptContentSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
@@ -419,11 +414,17 @@ export type LLMApiKey =
     ? z.infer<typeof LLMApiKeySchema>
     : never;
 
+// NOTE: This string is whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
+export const PROMPT_EXPERIMENT_ENVIRONMENT =
+  "langfuse-prompt-experiment" as const;
+
+type PromptExperimentEnvironment = typeof PROMPT_EXPERIMENT_ENVIRONMENT;
+
 export type TraceParams = {
   traceName: string;
   traceId: string;
   projectId: string;
-  tags: string[];
+  environment: PromptExperimentEnvironment;
   tokenCountDelegate: TokenCountDelegate;
   authCheck: AuthHeaderValidVerificationResult;
 };

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -6,6 +6,7 @@ export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
   tablePrefix?: string,
+  useTracesAmtCompatMode: boolean = false,
 ) => {
   const prefix = tablePrefix ? `${tablePrefix}.` : "";
 
@@ -13,9 +14,11 @@ export const clickhouseSearchCondition = (
     !searchType || searchType.includes("id")
       ? `${prefix}id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
-    searchType && searchType.includes("content")
+    searchType && searchType.includes("content") && !useTracesAmtCompatMode
       ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
-      : null,
+      : searchType && searchType.includes("content") && useTracesAmtCompatMode
+        ? `finalizeAggregation(${prefix}input) ILIKE {searchString: String} OR finalizeAggregation(${prefix}output) ILIKE {searchString: String}`
+        : null,
   ].filter(Boolean);
 
   return {

packages/shared/src/server/queues.ts

@@ -40,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -101,6 +116,15 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -165,6 +189,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -192,6 +217,13 @@ export type DeadLetterRetryQueueEventType = z.infer<
 
 export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
 
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -214,6 +246,7 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
   WebhookQueue = "webhook-queue",
   EntityChangeQueue = "entity-change-queue",
@@ -241,6 +274,7 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
   WebhookJob = "webhook-job",
   EntityChangeJob = "entity-change-job",
@@ -265,6 +299,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -282,6 +322,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -306,6 +347,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/getQueue.ts

@@ -23,6 +23,7 @@ import { ScoreDeleteQueue } from "./scoreDelete";
 import { DeadLetterRetryQueue } from "./dlqRetryQueue";
 import { WebhookQueue } from "./webhookQueue";
 import { EntityChangeQueue } from "./entityChangeQueue";
+import { DatasetDeleteQueue } from "./datasetDelete";
 
 // IngestionQueue is sharded and requires a sharding key
 // Use IngestionQueue.getInstance({ shardName: queueName }) directly instead
@@ -36,6 +37,8 @@ export function getQueue(
       return CloudUsageMeteringQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.DatasetDelete:
+      return DatasetDeleteQueue.getInstance();
     case QueueName.EvaluationExecution:
       return EvalExecutionQueue.getInstance();
     case QueueName.ExperimentCreate:

packages/shared/src/server/redis/redis.ts

@@ -6,6 +6,7 @@ import { logger } from "../logger";
 const defaultRedisOptions: Partial<RedisOptions> = {
   maxRetriesPerRequest: null,
   enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
+  keyPrefix: env.REDIS_KEY_PREFIX ?? undefined,
 };
 
 export const redisQueueRetryOptions: Partial<RedisOptions> = {
@@ -156,6 +157,24 @@ export const getQueuePrefix = (queueName: string): string | undefined => {
   return undefined;
 };
 
+/**
+ * Execute multiple Redis DEL operations safely in cluster mode
+ */
+export const safeMultiDel = async (
+  redis: Redis | Cluster | null,
+  keys: string[],
+): Promise<void> => {
+  if (!redis || keys.length === 0) return;
+
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // In cluster mode, delete keys in separate commands to avoid CROSSSLOT errors
+    await Promise.all(keys.map(async (key: string) => redis.del(key)));
+  } else {
+    // In single-node mode, can delete all keys at once
+    await redis.del(keys);
+  }
+};
+
 const createRedisClient = () => {
   try {
     return createNewRedisInstance();

packages/shared/src/server/redis/webhookQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
 import { logger } from "../logger";
 
 export class WebhookQueue {
@@ -23,6 +27,7 @@ export class WebhookQueue {
           QueueName.WebhookQueue,
           {
             connection: newRedis,
+            prefix: getQueuePrefix(QueueName.WebhookQueue),
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,

packages/shared/src/server/repositories/automation-repository.ts

@@ -2,6 +2,7 @@ import {
   Action,
   ActionExecutionStatus,
   JobConfigState,
+  Prisma,
   prisma,
   Trigger,
 } from "../../db";
@@ -15,6 +16,7 @@ import {
   SafeWebhookActionConfig,
 } from "../../domain/automations";
 import { FilterState } from "../../types";
+import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";
 
 export const getActionByIdWithSecrets = async ({
   projectId,
@@ -35,15 +37,31 @@ export const getActionByIdWithSecrets = async ({
   }
 
   const config = actionConfig.config as WebhookActionConfigWithSecrets;
+
+  // Decrypt secret headers for webhook execution using new structure
+  const decryptedHeaders = config.requestHeaders
+    ? decryptSecretHeaders(mergeHeaders(config.headers, config.requestHeaders))
+    : config.headers
+      ? Object.entries(config.headers).reduce(
+          (acc, [key, value]) => {
+            acc[key] = { secret: false, value };
+            return acc;
+          },
+          {} as Record<string, { secret: boolean; value: string }>,
+        )
+      : {};
+
   return {
     ...actionConfig,
     config: {
       type: config.type,
       url: config.url,
-      headers: config.headers,
+      requestHeaders: decryptedHeaders,
+      displayHeaders: getDisplayHeaders(config),
       apiVersion: config.apiVersion,
       displaySecretKey: config.displaySecretKey,
       secretKey: config.secretKey,
+      lastFailingExecutionId: config.lastFailingExecutionId,
     },
   };
 };
@@ -114,16 +132,33 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {
   };
 };
 
+const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {
+  let displayHeaders = config.displayHeaders;
+  if (!displayHeaders && config.headers) {
+    // Convert legacy headers to displayHeaders format
+    displayHeaders = Object.entries(config.headers).reduce(
+      (acc, [key, value]) => {
+        acc[key] = { secret: false, value };
+        return acc;
+      },
+      {} as Record<string, { secret: boolean; value: string }>,
+    );
+  }
+  return displayHeaders;
+};
+
 const convertActionToDomain = (action: Action): ActionDomain => {
   const config = action.config as WebhookActionConfigWithSecrets;
+
   return {
     ...action,
     config: {
       type: config.type,
       url: config.url,
-      headers: config.headers,
+      displayHeaders: getDisplayHeaders(config),
       apiVersion: config.apiVersion,
       displaySecretKey: config.displaySecretKey,
+      lastFailingExecutionId: config.lastFailingExecutionId,
     } as SafeWebhookActionConfig,
   };
 };
@@ -197,28 +232,46 @@ export const getConsecutiveAutomationFailures = async ({
   automationId: string;
   projectId: string;
 }): Promise<number> => {
-  // First get the automation to extract triggerId and actionId
-  const automation = await prisma.automation.findFirst({
-    where: {
-      id: automationId,
-      projectId,
-    },
+  const automation = await getAutomationById({
+    automationId,
+    projectId,
   });
 
   if (!automation) {
     return 0;
   }
 
-  const { triggerId, actionId } = automation;
-  const executions = await prisma.automationExecution.findMany({
-    where: {
-      triggerId,
-      actionId,
-      projectId,
-      status: {
-        in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
-      },
+  // Build where clause - if lastFailingExecutionId is set, only consider executions newer than it
+  const whereClause: Prisma.AutomationExecutionWhereInput = {
+    triggerId: automation.trigger.id,
+    actionId: automation.action.id,
+    projectId,
+    status: {
+      in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
     },
+  };
+
+  // If there's a lastFailingExecutionId, we need to get executions that are newer than that execution
+  if (automation.action.config.lastFailingExecutionId) {
+    // First get the timestamp of the last failing execution
+    const lastFailingExecution = await prisma.automationExecution.findUnique({
+      where: {
+        id: automation.action.config.lastFailingExecutionId,
+      },
+      select: {
+        createdAt: true,
+      },
+    });
+
+    if (lastFailingExecution) {
+      whereClause.createdAt = {
+        gt: lastFailingExecution.createdAt,
+      };
+    }
+  }
+
+  const executions = await prisma.automationExecution.findMany({
+    where: whereClause,
     orderBy: {
       createdAt: "desc",
     },

packages/shared/src/server/repositories/dataset-run-items-converters.ts

@@ -0,0 +1,86 @@
+import { DatasetRunItemDomain } from "../../domain/dataset-run-items";
+import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { parseMetadataCHRecordToDomain } from "../utils/metadata_conversion";
+import { parseClickhouseUTCDateTimeFormat } from "./clickhouse";
+import { DatasetRunItemRecordReadType } from "./definitions";
+
+export const convertToDatasetRunMetrics = (row: any) => {
+  return {
+    id: row.dataset_run_id,
+    projectId: row.project_id,
+    createdAt: new Date(row.created_at),
+    updatedAt: new Date(row.updated_at),
+    name: row.datasetRunName,
+    description: row.datasetRunDescription ?? "",
+    metadata: row.datasetRunMetadata,
+    countRunItems: row.count_run_items,
+    avgTotalCost: undefined,
+    avgLatency: undefined,
+    scores: undefined,
+    datasetId: row.dataset_id,
+  };
+};
+
+export const convertDatasetRunItemDomainToClickhouse = (
+  datasetRunItem: DatasetRunItemDomain,
+): DatasetRunItemRecordReadType => {
+  return {
+    id: datasetRunItem.id,
+    project_id: datasetRunItem.projectId,
+    trace_id: datasetRunItem.traceId,
+    observation_id: datasetRunItem.observationId,
+    dataset_id: datasetRunItem.datasetId,
+    dataset_run_id: datasetRunItem.datasetRunId,
+    dataset_run_name: datasetRunItem.datasetRunName,
+    dataset_run_description: datasetRunItem.datasetRunDescription,
+    dataset_run_metadata: datasetRunItem.datasetRunMetadata as Record<
+      string,
+      string
+    >,
+    dataset_item_id: datasetRunItem.datasetItemId,
+    dataset_item_input: datasetRunItem.datasetItemInput as string,
+    dataset_item_expected_output:
+      datasetRunItem.datasetItemExpectedOutput as string,
+    dataset_item_metadata: datasetRunItem.datasetItemMetadata as Record<
+      string,
+      string
+    >,
+    created_at: convertDateToClickhouseDateTime(datasetRunItem.createdAt),
+    updated_at: convertDateToClickhouseDateTime(datasetRunItem.updatedAt),
+    event_ts: convertDateToClickhouseDateTime(new Date()),
+    is_deleted: 0,
+    dataset_run_created_at: convertDateToClickhouseDateTime(
+      datasetRunItem.datasetRunCreatedAt,
+    ),
+    error: datasetRunItem.error,
+  };
+};
+
+export const convertDatasetRunItemClickhouseToDomain = (
+  row: DatasetRunItemRecordReadType,
+): DatasetRunItemDomain => {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    traceId: row.trace_id,
+    observationId: row.observation_id ?? null,
+    datasetRunId: row.dataset_run_id,
+    datasetRunName: row.dataset_run_name,
+    datasetRunDescription: row.dataset_run_description ?? null,
+    datasetRunCreatedAt: parseClickhouseUTCDateTimeFormat(
+      row.dataset_run_created_at,
+    ),
+    datasetRunMetadata:
+      parseMetadataCHRecordToDomain(row.dataset_run_metadata) ?? null,
+    datasetItemId: row.dataset_item_id,
+    datasetItemInput: row.dataset_item_input,
+    datasetItemExpectedOutput: row.dataset_item_expected_output,
+    datasetItemMetadata: parseMetadataCHRecordToDomain(
+      row.dataset_item_metadata,
+    ),
+    createdAt: parseClickhouseUTCDateTimeFormat(row.created_at),
+    updatedAt: parseClickhouseUTCDateTimeFormat(row.updated_at),
+    datasetId: row.dataset_id,
+    error: row.error ?? null,
+  };
+};

packages/shared/src/server/repositories/dataset-run-items.ts

@@ -0,0 +1,94 @@
+import { env } from "../../env";
+import { commandClickhouse } from "./clickhouse";
+
+export const deleteDatasetRunItemsByProjectId = async ({
+  projectId,
+}: {
+  projectId: string;
+}) => {
+  const query = `
+      DELETE FROM dataset_run_items
+      WHERE project_id = {projectId: String};
+    `;
+  await commandClickhouse({
+    query: query,
+    params: {
+      projectId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetId = async ({
+  projectId,
+  datasetId,
+}: {
+  projectId: string;
+  datasetId: string;
+}) => {
+  const query = `
+  DELETE FROM dataset_run_items
+  WHERE project_id = {projectId: String}
+  AND dataset_id = {datasetId: String}
+`;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};
+
+export const deleteDatasetRunItemsByDatasetRunIds = async ({
+  projectId,
+  datasetRunIds,
+  datasetId,
+}: {
+  projectId: string;
+  datasetRunIds: string[];
+  datasetId: string;
+}) => {
+  const query = `
+    DELETE FROM dataset_run_items
+    WHERE project_id = {projectId: String}
+    AND dataset_id = {datasetId: String}
+    AND dataset_run_id IN ({datasetRunIds: Array(String)})
+  `;
+
+  await commandClickhouse({
+    query,
+    params: {
+      projectId,
+      datasetRunIds,
+      datasetId,
+    },
+    clickhouseConfigs: {
+      request_timeout: env.LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS,
+    },
+    tags: {
+      feature: "datasets",
+      type: "dataset-run-items",
+      kind: "delete",
+      projectId,
+    },
+  });
+};

packages/shared/src/server/repositories/definitions.ts

@@ -196,6 +196,45 @@ export const scoreRecordInsertSchema = scoreRecordBaseSchema.extend({
 });
 export type ScoreRecordInsertType = z.infer<typeof scoreRecordInsertSchema>;
 
+const datasetRunItemRecordBaseSchema = z.object({
+  id: z.string(),
+  project_id: z.string(),
+  trace_id: z.string(),
+  observation_id: z.string().nullish(),
+  dataset_id: z.string(),
+  dataset_run_id: z.string(),
+  dataset_item_id: z.string(),
+  dataset_run_name: z.string(),
+  dataset_run_description: z.string().nullish(),
+  dataset_run_metadata: z.record(z.string(), z.string()),
+  dataset_item_input: z.string(),
+  dataset_item_expected_output: z.string(),
+  dataset_item_metadata: z.record(z.string(), z.string()),
+  is_deleted: z.number(),
+  error: z.string().nullish(),
+});
+
+const datasetRunItemRecordReadSchema = datasetRunItemRecordBaseSchema.extend({
+  dataset_run_created_at: clickhouseStringDateSchema,
+  created_at: clickhouseStringDateSchema,
+  updated_at: clickhouseStringDateSchema,
+  event_ts: clickhouseStringDateSchema,
+});
+export type DatasetRunItemRecordReadType = z.infer<
+  typeof datasetRunItemRecordReadSchema
+>;
+
+export const datasetRunItemRecordInsertSchema =
+  datasetRunItemRecordBaseSchema.extend({
+    created_at: z.number(),
+    updated_at: z.number(),
+    event_ts: z.number(),
+    dataset_run_created_at: z.number(),
+  });
+export type DatasetRunItemRecordInsertType = z.infer<
+  typeof datasetRunItemRecordInsertSchema
+>;
+
 export const blobStorageFileLogRecordBaseSchema = z.object({
   id: z.string(),
   project_id: z.string(),
@@ -308,6 +347,50 @@ export const convertPostgresTraceToInsert = (
   };
 };
 
+export const convertPostgresDatasetRunItemToInsert = (
+  datasetRunItem: Record<string, any>,
+): DatasetRunItemRecordInsertType => {
+  return {
+    id: datasetRunItem.id,
+    project_id: datasetRunItem.project_id,
+    dataset_run_id: datasetRunItem.dataset_run_id,
+    dataset_item_id: datasetRunItem.dataset_item_id,
+    dataset_id: datasetRunItem.dataset_id,
+    trace_id: datasetRunItem.trace_id,
+    observation_id: datasetRunItem.observation_id,
+    error: datasetRunItem.error,
+    created_at: datasetRunItem.created_at?.getTime(),
+    updated_at: datasetRunItem.updated_at?.getTime(),
+    // denormalized run data
+    dataset_run_name: datasetRunItem.dataset_run_name,
+    dataset_run_description: datasetRunItem.dataset_run_description,
+    dataset_run_metadata:
+      typeof datasetRunItem.dataset_run_metadata === "string" ||
+      typeof datasetRunItem.dataset_run_metadata === "number" ||
+      typeof datasetRunItem.dataset_run_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_run_metadata }
+        : Array.isArray(datasetRunItem.dataset_run_metadata)
+          ? { metadata: datasetRunItem.dataset_run_metadata }
+          : (datasetRunItem.dataset_run_metadata ?? {}),
+    dataset_run_created_at: datasetRunItem.dataset_run_created_at?.getTime(),
+    // denormalized item data
+    dataset_item_input: JSON.stringify(datasetRunItem.dataset_item_input),
+    dataset_item_expected_output: JSON.stringify(
+      datasetRunItem.dataset_item_expected_output,
+    ),
+    dataset_item_metadata:
+      typeof datasetRunItem.dataset_item_metadata === "string" ||
+      typeof datasetRunItem.dataset_item_metadata === "number" ||
+      typeof datasetRunItem.dataset_item_metadata === "boolean"
+        ? { metadata: datasetRunItem.dataset_item_metadata }
+        : Array.isArray(datasetRunItem.dataset_item_metadata)
+          ? { metadata: datasetRunItem.dataset_item_metadata }
+          : (datasetRunItem.dataset_item_metadata ?? {}),
+    event_ts: datasetRunItem.created_at?.getTime(),
+    is_deleted: 0,
+  };
+};
+
 /**
  * Expects a single record from a
  * `select o.*,

packages/shared/src/server/repositories/index.ts

@@ -13,3 +13,5 @@ export * from "./scores-utils";
 export * from "./blobStorageLog";
 export * from "./environments";
 export * from "./automation-repository";
+export * from "./dataset-run-items-converters";
+export * from "./dataset-run-items";

packages/shared/src/server/repositories/scores.ts

@@ -1335,6 +1335,8 @@ export const getScoresForPostHog = async function* (
       s.timestamp as timestamp,
       s.name as name,
       s.value as value,
+      s.string_value as string_value,
+      s.data_type as data_type,
       s.comment as comment,
       s.environment as environment,
       t.name as trace_name,
@@ -1342,6 +1344,7 @@ export const getScoresForPostHog = async function* (
       t.user_id as trace_user_id,
       t.release as trace_release,
       t.tags as trace_tags,
+      s.metadata as metadata,
       t.metadata['$posthog_session_id'] as posthog_session_id
     FROM scores s FINAL
     LEFT JOIN traces t FINAL ON s.trace_id = t.id AND s.project_id = t.project_id
@@ -1382,6 +1385,9 @@ export const getScoresForPostHog = async function* (
       langfuse_score_name: record.name,
       langfuse_score_value: record.value,
       langfuse_score_comment: record.comment,
+      langfuse_score_metadata: record.metadata,
+      langfuse_score_string_value: record.string_value,
+      langfuse_score_data_type: record.data_type,
       langfuse_trace_name: record.trace_name,
       langfuse_id: record.id,
       langfuse_session_id: record.trace_session_id,

packages/shared/src/server/services/PromptService/index.ts

@@ -3,6 +3,7 @@ import { Redis, Cluster } from "ioredis";
 import { env } from "../../../env";
 import { logger } from "../../logger";
 import { escapeRegex } from "./utils";
+import { safeMultiDel } from "../../redis/redis";
 import {
   PromptGraph,
   PromptParams,
@@ -30,9 +31,12 @@ export class PromptService {
     (name: string, value?: number) => void,
     cacheEnabled?: boolean, // used for testing
   ) {
-    this.cacheEnabled =
-      Boolean(redis) &&
-      (cacheEnabled || env.LANGFUSE_CACHE_PROMPT_ENABLED === "true");
+    if (cacheEnabled !== undefined) {
+      this.cacheEnabled = cacheEnabled;
+    } else {
+      this.cacheEnabled =
+        Boolean(redis) && env.LANGFUSE_CACHE_PROMPT_ENABLED === "true";
+    }
 
     this.ttlSeconds = env.LANGFUSE_CACHE_PROMPT_TTL_SECONDS;
   }
@@ -237,13 +241,14 @@ export class PromptService {
     const legacyKeyIndexKey = `${keyIndexKey}:${params.promptName}`;
     const legacyKeys = await this.redis?.smembers(legacyKeyIndexKey);
 
-    // Delete all keys for the prefix and the key index
-    await this.redis?.del([
+    // Delete all keys for the prefix and the key index using safe multi-delete
+    const keysToDelete = [
       ...(keys ?? []),
       keyIndexKey,
       ...(legacyKeys ?? []),
       legacyKeyIndexKey,
-    ]);
+    ];
+    await safeMultiDel(this.redis, keysToDelete);
   }
 
   private getCacheKey(params: PromptParams): string {

packages/shared/src/server/services/datasets-ui-table-service.ts

@@ -9,7 +9,7 @@ type FetchDatasetItemsTableProps = {
   filter: FilterState;
 };
 
-const getDatasetRunItemsTableGeneric = async <T>(
+const getDatasetRunItemsTableGenericPg = async <T>(
   props: FetchDatasetItemsTableProps,
 ) => {
   const { select, projectId, filter } = props;
@@ -45,11 +45,11 @@ const getDatasetRunItemsTableGeneric = async <T>(
   return res;
 };
 
-export const getDatasetRunItemsTableCount = async (props: {
+export const getDatasetRunItemsTableCountPg = async (props: {
   projectId: string;
   filter: FilterState;
 }) => {
-  const res = await getDatasetRunItemsTableGeneric<Array<{ count: bigint }>>({
+  const res = await getDatasetRunItemsTableGenericPg<Array<{ count: bigint }>>({
     select: "count",
     projectId: props.projectId,
     filter: props.filter,

packages/shared/src/server/services/sessions-ui-table-service.ts

@@ -3,6 +3,7 @@ import { OrderByState } from "../../interfaces/orderBy";
 import { sessionCols } from "../../tableDefinitions/mapSessionTable";
 import { FilterState } from "../../types";
 import { convertDateToClickhouseDateTime } from "../clickhouse/client";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { DateTimeFilter, FilterList, orderByToClickhouseSql } from "../queries";
 import {
   getProjectIdDefaultFilter,
@@ -11,6 +12,7 @@ import {
 import {
   TRACE_TO_OBSERVATIONS_INTERVAL,
   queryClickhouse,
+  getTimeframesTracesAMT,
 } from "../repositories";
 
 export type SessionDataReturnType = {
@@ -235,113 +237,133 @@ const getSessionsTableGeneric = async <T>(props: FetchSessionsTableProps) => {
 
   // We use deduplicated traces and observations CTEs instead of final to be able to use Skip indices in Clickhouse.
   const query = `
-    WITH deduplicated_traces AS (
-      SELECT * EXCEPT input, output, metadata
-      FROM traces t
-      WHERE t.session_id IS NOT NULL 
-        AND t.project_id = {projectId: String}
-        ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    deduplicated_observations AS (
-        SELECT * 
-        FROM observations o
-        WHERE o.project_id = {projectId: String}
-        ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-        AND o.trace_id IN (
-          SELECT id
-          FROM deduplicated_traces
-        )
-        ORDER BY event_ts DESC
-        LIMIT 1 BY id, project_id
-    ),
-    observations_agg AS (
-      SELECT o.trace_id,
-            count(*) as obs_count,
-            min(o.start_time) as min_start_time,
-            max(o.end_time) as max_end_time,
-            sumMap(usage_details) as sum_usage_details,
-            sumMap(cost_details) as sum_cost_details,
-            anyLast(project_id) as project_id
-      FROM deduplicated_observations o
-      WHERE o.project_id = {projectId: String}
-      ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
-      GROUP BY o.trace_id
-    ),
-    session_data AS (
-        SELECT
-            t.session_id,
-            anyLast(t.project_id) as project_id,
-            max(t.timestamp) as max_timestamp,
-            min(t.timestamp) as min_timestamp,
-            groupArray(t.id) AS trace_ids,
-            groupUniqArray(t.user_id) AS user_ids,
-            count(*) as trace_count,
-            groupUniqArrayArray(t.tags) as trace_tags,
-            anyLast(t.environment) as trace_environment
-            -- Aggregate observations data at session level
-            ${
-              selectMetrics
-                ? `
-            ,
-            sum(o.obs_count) as total_observations,
-            -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
-            -- LLM session started on that date so this behaviour should yield better results.
-            date_diff('millisecond', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
-            sumMap(o.sum_usage_details) as session_usage_details,
-            sumMap(o.sum_cost_details) as session_cost_details,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
-            sumMap(o.sum_cost_details)['total'] as session_total_cost,          
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
-            arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
-            sumMap(o.sum_usage_details)['total'] as session_total_usage`
-                : ""
-            }
-        FROM deduplicated_traces t
-        ${
-          selectMetrics
-            ? `LEFT JOIN observations_agg o
-        ON t.id = o.trace_id AND t.project_id = o.project_id`
-            : ""
-        }
-        WHERE t.session_id IS NOT NULL
+        WITH deduplicated_traces AS (
+          SELECT * EXCEPT input, output, metadata
+          FROM __TRACE_TABLE__ t FINAL
+          WHERE t.session_id IS NOT NULL 
             AND t.project_id = {projectId: String}
             ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
-        GROUP BY t.session_id
-    )
-    SELECT ${sqlSelect}
-    FROM session_data s
-    WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
-    ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
-    `;
+        ),
+        deduplicated_observations AS (
+            SELECT * 
+            FROM observations o
+            WHERE o.project_id = {projectId: String}
+            ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+            AND o.trace_id IN (
+              SELECT id
+              FROM deduplicated_traces
+            )
+            ORDER BY event_ts DESC
+            LIMIT 1 BY id, project_id
+        ),
+        observations_agg AS (
+          SELECT o.trace_id,
+                count(*) as obs_count,
+                min(o.start_time) as min_start_time,
+                max(o.end_time) as max_end_time,
+                sumMap(usage_details) as sum_usage_details,
+                sumMap(cost_details) as sum_cost_details,
+                anyLast(project_id) as project_id
+          FROM deduplicated_observations o
+          WHERE o.project_id = {projectId: String}
+          ${traceTimestampFilter ? `AND o.start_time >= {observationsStartTime: DateTime64(3)} - ${TRACE_TO_OBSERVATIONS_INTERVAL}` : ""}
+          GROUP BY o.trace_id
+        ),
+        session_data AS (
+            SELECT
+                t.session_id,
+                anyLast(t.project_id) as project_id,
+                max(t.timestamp) as max_timestamp,
+                min(t.timestamp) as min_timestamp,
+                groupArray(t.id) AS trace_ids,
+                groupUniqArray(t.user_id) AS user_ids,
+                count(*) as trace_count,
+                groupUniqArrayArray(t.tags) as trace_tags,
+                anyLast(t.environment) as trace_environment
+                -- Aggregate observations data at session level
+                ${
+                  selectMetrics
+                    ? `,
+                      sum(o.obs_count) as total_observations,
+                      -- Use minIf, because ClickHouse fills 1970-01-01 on left joins. We assume that no
+                      -- LLM session started on that date so this behaviour should yield better results.
+                      date_diff('second', minIf(min_start_time, min_start_time > '1970-01-01'), max(max_end_time)) as duration,
+                      sumMap(o.sum_usage_details) as session_usage_details,
+                      sumMap(o.sum_cost_details) as session_cost_details,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_cost_details)))) as session_input_cost,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_cost_details)))) as session_output_cost,
+                      sumMap(o.sum_cost_details)['total'] as session_total_cost,          
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'input') > 0, sumMap(o.sum_usage_details)))) as session_input_usage,
+                      arraySum(mapValues(mapFilter(x -> positionCaseInsensitive(x.1, 'output') > 0, sumMap(o.sum_usage_details)))) as session_output_usage,
+                      sumMap(o.sum_usage_details)['total'] as session_total_usage`
+                    : ""
+                }
+            FROM deduplicated_traces t
+            ${
+              selectMetrics
+                ? `LEFT JOIN observations_agg o
+                   ON t.id = o.trace_id AND t.project_id = o.project_id`
+                : ""
+            }
+            WHERE t.session_id IS NOT NULL
+                AND t.project_id = {projectId: String}
+                ${singleTraceFilter?.query ? ` AND ${singleTraceFilter.query}` : ""}
+            GROUP BY t.session_id
+        )
+        SELECT ${sqlSelect}
+        FROM session_data s
+        WHERE ${tracesFilterRes.query ? tracesFilterRes.query : ""}
+        ${orderByToClickhouseSql(orderBy ?? null, sessionCols)}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+        `;
 
-  const obsStartTimeValue = traceTimestampFilter
-    ? convertDateToClickhouseDateTime(traceTimestampFilter.value)
-    : null;
-
-  const res = await queryClickhouse<T>({
-    query: query,
-    params: {
-      projectId,
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      ...tracesFilterRes.params,
-      ...singleTraceFilter?.params,
-      ...(obsStartTimeValue
-        ? { observationsStartTime: obsStartTimeValue }
-        : {}),
+  return measureAndReturn({
+    operationName: "getSessionsTableGeneric",
+    projectId,
+    input: {
+      params: {
+        projectId,
+        limit: limit,
+        offset: limit && page ? limit * page : 0,
+        ...tracesFilterRes.params,
+        ...singleTraceFilter?.params,
+        ...(traceTimestampFilter
+          ? {
+              observationsStartTime: convertDateToClickhouseDateTime(
+                traceTimestampFilter.value,
+              ),
+            }
+          : {}),
+      },
+      tags: {
+        ...(props.tags ?? {}),
+        feature: "tracing",
+        type: "sessions-table",
+        projectId,
+      },
     },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "sessions-table",
-      projectId,
+    existingExecution: async (input) => {
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", "traces"),
+        params: input.params,
+        tags: input.tags,
+        clickhouseConfigs,
+      });
+    },
+    newExecution: async (input) => {
+      // Extract the timestamp from filter for AMT table selection
+      const fromTimestamp = filter?.find(
+        (f) =>
+          f.column === "min_timestamp" &&
+          (f.operator === ">=" || f.operator === ">"),
+      )?.value as Date | undefined;
+      const traceAmt = getTimeframesTracesAMT(fromTimestamp);
+      return queryClickhouse<T>({
+        query: query.replace("__TRACE_TABLE__", traceAmt),
+        params: input.params,
+        tags: input.tags,
+        clickhouseConfigs,
+      });
     },
-    clickhouseConfigs,
   });
-
-  return res;
 };

packages/shared/src/server/services/traces-ui-table-service.ts

@@ -21,7 +21,9 @@ import {
   parseClickhouseUTCDateTimeFormat,
   queryClickhouse,
   reduceUsageOrCostDetails,
+  getTimeframesTracesAMT,
 } from "../repositories";
+import { measureAndReturn } from "../clickhouse/measureAndReturn";
 import { TracingSearchType } from "../../interfaces/search";
 import { ObservationLevelType, TraceDomain } from "../../domain";
 import { ClickHouseClientConfigOptions } from "@clickhouse/client";
@@ -217,54 +219,6 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
     clickhouseConfigs,
   } = props;
 
-  let sqlSelect: string;
-  switch (select) {
-    case "count":
-      sqlSelect = "count(*) as count";
-      break;
-    case "metrics":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as project_id,
-        t.timestamp as timestamp,
-        os.latency_milliseconds / 1000 as latency,
-        os.cost_details as cost_details,
-        os.usage_details as usage_details,
-        os.aggregated_level as level,
-        os.error_count as error_count,
-        os.warning_count as warning_count,
-        os.default_count as default_count,
-        os.debug_count as debug_count,
-        os.observation_count as observation_count,
-        s.scores_avg as scores_avg,
-        s.score_categories as score_categories,
-        t.public as public`;
-      break;
-    case "rows":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as project_id,
-        t.timestamp as timestamp,
-        t.tags as tags,
-        t.bookmarked as bookmarked,
-        t.name as name,
-        t.release as release,
-        t.version as version,
-        t.user_id as user_id,
-        t.environment as environment,
-        t.session_id as session_id,
-        t.public as public`;
-      break;
-    case "identifiers":
-      sqlSelect = `
-        t.id as id,
-        t.project_id as projectId,
-        t.timestamp as timestamp`;
-      break;
-    default:
-      throw new Error(`Unknown select type: ${select}`);
-  }
-
   const { tracesFilter, scoresFilter, observationsFilter } =
     getProjectIdDefaultFilter(projectId, { tracesPrefix: "t" });
 
@@ -329,149 +283,318 @@ async function getTracesTableGeneric(props: FetchTracesTableProps) {
   const scoresFilterRes = scoresFilter.apply();
   const observationFilterRes = observationsFilter.apply();
 
-  const search = clickhouseSearchCondition(searchQuery, searchType, "t");
-
-  const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
-  const orderByCols = [
-    ...tracesTableUiColumnDefinitions,
-    {
-      clickhouseSelect: "toDate(t.timestamp)",
-      uiTableName: "timestamp_to_date",
-      uiTableId: "timestamp_to_date",
-      clickhouseTableName: "traces",
-    },
-    {
-      clickhouseSelect: "t.event_ts",
-      uiTableName: "event_ts",
-      uiTableId: "event_ts",
-      clickhouseTableName: "traces",
-    },
-  ];
-  const chOrderBy = orderByToClickhouseSql(
-    [
-      defaultOrder
-        ? [
-            {
-              column: "timestamp_to_date",
-              order: orderBy.order,
-            },
-            { column: "timestamp", order: orderBy.order },
-            { column: "event_ts", order: "DESC" as "DESC" },
-          ]
-        : null,
-      orderBy ?? null,
-    ].flat(),
-    orderByCols,
-  );
-
-  // complex query ahead:
-  // - we only join scores and observations if we really need them to speed up default views
-  // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
-  // - we filter the observations and scores as much as possible before joining them to traces.
-  // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
-  //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
-
-  const query = `
+  const observationsAndScoresCTE = `
     WITH observations_stats AS (
       SELECT
         COUNT(*) AS observation_count,
-          sumMap(usage_details) as usage_details,
-          SUM(total_cost) AS total_cost,
-          date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
-          countIf(level = 'ERROR') as error_count,
-          countIf(level = 'WARNING') as warning_count,
-          countIf(level = 'DEFAULT') as default_count,
-          countIf(level = 'DEBUG') as debug_count,
-          multiIf(
-            arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
-            arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
-            arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
-            'DEBUG'
-          ) AS aggregated_level,
-          sumMap(cost_details) as cost_details,
-          trace_id,
-          project_id
-      FROM observations o FINAL 
+        sumMap(usage_details) as usage_details,
+        SUM(total_cost) AS total_cost,
+        date_diff('millisecond', least(min(start_time), min(end_time)), greatest(max(start_time), max(end_time))) as latency_milliseconds,
+        countIf(level = 'ERROR') as error_count,
+        countIf(level = 'WARNING') as warning_count,
+        countIf(level = 'DEFAULT') as default_count,
+        countIf(level = 'DEBUG') as debug_count,
+        multiIf(
+          arrayExists(x -> x = 'ERROR', groupArray(level)), 'ERROR',
+          arrayExists(x -> x = 'WARNING', groupArray(level)), 'WARNING',
+          arrayExists(x -> x = 'DEFAULT', groupArray(level)), 'DEFAULT',
+          'DEBUG'
+        ) AS aggregated_level,
+        sumMap(cost_details) as cost_details,
+        trace_id,
+        project_id
+      FROM observations o FINAL
       WHERE o.project_id = {projectId: String}
-      ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
-      ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
+        ${timeStampFilter ? `AND o.start_time >= {traceTimestamp: DateTime64(3)} - ${OBSERVATIONS_TO_TRACE_INTERVAL}` : ""}
+        ${observationsFilter ? `AND ${observationFilterRes.query}` : ""}
       GROUP BY trace_id, project_id
     ),
-    scores_avg AS (
-      SELECT
-        project_id,
-        trace_id,
-        -- For numeric scores, use tuples of (name, avg_value)
-        groupArrayIf(
-          tuple(name, avg_value),
-          data_type IN ('NUMERIC', 'BOOLEAN')
-        ) AS scores_avg,
-        -- For categorical scores, use name:value format for improved query performance
-        groupArrayIf(
-          concat(name, ':', string_value),
-          data_type = 'CATEGORICAL' AND notEmpty(string_value)
-        ) AS score_categories
-      FROM (
-        SELECT 
-          project_id,
-          trace_id,
-          name,
-          data_type,
-          string_value,
-          avg(value) as avg_value
-        FROM scores s FINAL 
-        WHERE 
-          project_id = {projectId: String}
-          ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
-          ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
-        GROUP BY 
-          project_id,
-          trace_id,
-          name,
-          data_type,
-          string_value
-      ) tmp
-      GROUP BY project_id, trace_id
-    )
-    SELECT ${sqlSelect}
-    -- FINAL is used for non default ordering and count.
-    FROM traces t  ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "" : "FINAL"}
-    ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
-    ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
-    WHERE t.project_id = {projectId: String}
-    ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
-    ${search.query}
-    ${chOrderBy}
-    -- This is used for metrics and row queries. Count has only one result.
-    -- This is only used for default ordering. Otherwise, we use final.
-    ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
-    ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+         scores_avg AS (
+           SELECT
+             project_id,
+             trace_id,
+             -- For numeric scores, use tuples of (name, avg_value)
+             groupArrayIf(
+               tuple(name, avg_value),
+               data_type IN ('NUMERIC', 'BOOLEAN')
+             ) AS scores_avg,
+             -- For categorical scores, use name:value format for improved query performance
+             groupArrayIf(
+               concat(name, ':', string_value),
+               data_type = 'CATEGORICAL' AND notEmpty(string_value)
+             ) AS score_categories
+           FROM (
+                  SELECT
+                    project_id,
+                    trace_id,
+                    name,
+                    data_type,
+                    string_value,
+                    avg(value) as avg_value
+                  FROM scores s FINAL
+                  WHERE
+                    project_id = {projectId: String}
+                    ${timeStampFilter ? `AND s.timestamp >= {traceTimestamp: DateTime64(3)} - ${SCORE_TO_TRACE_OBSERVATIONS_INTERVAL}` : ""}
+                    ${scoresFilterRes ? `AND ${scoresFilterRes.query}` : ""}
+                  GROUP BY
+                    project_id,
+                    trace_id,
+                    name,
+                    data_type,
+                    string_value
+                ) tmp
+           GROUP BY project_id, trace_id
+         )
   `;
 
-  const res = await queryClickhouse<
-    SelectReturnTypeMap[keyof SelectReturnTypeMap]
-  >({
-    query: query,
-    params: {
-      limit: limit,
-      offset: limit && page ? limit * page : 0,
-      traceTimestamp: timeStampFilter?.value.getTime(),
-      projectId: projectId,
-      ...tracesFilterRes.params,
-      ...observationFilterRes.params,
-      ...scoresFilterRes.params,
-      ...search.params,
-    },
-    tags: {
-      ...(props.tags ?? {}),
-      feature: "tracing",
-      type: "traces-table",
-      projectId,
-    },
-    clickhouseConfigs,
-  });
+  return measureAndReturn({
+    operationName: "getTracesTableGeneric",
+    projectId: props.projectId,
+    input: props,
+    existingExecution: async (props) => {
+      let sqlSelect: string;
+      switch (select) {
+        case "count":
+          sqlSelect = "count(*) as count";
+          break;
+        case "metrics":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            os.latency_milliseconds / 1000 as latency,
+            os.cost_details as cost_details,
+            os.usage_details as usage_details,
+            os.aggregated_level as level,
+            os.error_count as error_count,
+            os.warning_count as warning_count,
+            os.default_count as default_count,
+            os.debug_count as debug_count,
+            os.observation_count as observation_count,
+            s.scores_avg as scores_avg,
+            s.score_categories as score_categories,
+            t.public as public`;
+          break;
+        case "rows":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            t.tags as tags,
+            t.bookmarked as bookmarked,
+            t.name as name,
+            t.release as release,
+            t.version as version,
+            t.user_id as user_id,
+            t.environment as environment,
+            t.session_id as session_id,
+            t.public as public`;
+          break;
+        case "identifiers":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as projectId,
+            t.timestamp as timestamp`;
+          break;
+        default:
+          throw new Error(`Unknown select type: ${select}`);
+      }
 
-  return res;
+      const search = clickhouseSearchCondition(searchQuery, searchType, "t");
+
+      const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
+      const orderByCols = [
+        ...tracesTableUiColumnDefinitions,
+        {
+          clickhouseSelect: "toDate(t.timestamp)",
+          uiTableName: "timestamp_to_date",
+          uiTableId: "timestamp_to_date",
+          clickhouseTableName: "traces",
+        },
+        {
+          clickhouseSelect: "t.event_ts",
+          uiTableName: "event_ts",
+          uiTableId: "event_ts",
+          clickhouseTableName: "traces",
+        },
+      ];
+      const chOrderBy = orderByToClickhouseSql(
+        [
+          defaultOrder
+            ? [
+                {
+                  column: "timestamp_to_date",
+                  order: orderBy.order,
+                },
+                { column: "timestamp", order: orderBy.order },
+                { column: "event_ts", order: "DESC" as "DESC" },
+              ]
+            : null,
+          orderBy ?? null,
+        ].flat(),
+        orderByCols,
+      );
+
+      // complex query ahead:
+      // - we only join scores and observations if we really need them to speed up default views
+      // - we use FINAL on traces only in case we not need to order by something different than time. Otherwise we cannot guarantee correct reads.
+      // - we filter the observations and scores as much as possible before joining them to traces.
+      // - we order by todate(timestamp), event_ts desc per default and do not use FINAL.
+      //   In this case, CH is able to read the data only from the latest date from disk and filtering them in memory. No need to read all data e.g. for 1 month from disk.
+
+      const query = `
+        ${observationsAndScoresCTE}        
+
+        SELECT ${sqlSelect}
+        -- FINAL is used for non default ordering and count.
+        FROM traces t  ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "" : "FINAL"}
+        ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+        ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+        WHERE t.project_id = {projectId: String}
+        ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
+        ${search.query}
+        ${chOrderBy}
+        -- This is used for metrics and row queries. Count has only one result.
+        -- This is only used for default ordering. Otherwise, we use final.
+        ${["metrics", "rows", "identifiers"].includes(select) && defaultOrder ? "LIMIT 1 BY id, project_id" : ""}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+      `;
+
+      const res = await queryClickhouse<
+        SelectReturnTypeMap[keyof SelectReturnTypeMap]
+      >({
+        query: query,
+        params: {
+          limit: limit,
+          offset: limit && page ? limit * page : 0,
+          traceTimestamp: timeStampFilter?.value.getTime(),
+          projectId: projectId,
+          ...tracesFilterRes.params,
+          ...observationFilterRes.params,
+          ...scoresFilterRes.params,
+          ...search.params,
+        },
+        tags: {
+          ...(props.tags ?? {}),
+          feature: "tracing",
+          type: "traces-table",
+          projectId,
+        },
+        clickhouseConfigs,
+      });
+
+      return res;
+    },
+    newExecution: async () => {
+      let sqlSelect: string;
+      switch (select) {
+        case "count":
+          sqlSelect = "count(*) as count";
+          break;
+        case "metrics":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            os.latency_milliseconds / 1000 as latency,
+            os.cost_details as cost_details,
+            os.usage_details as usage_details,
+            os.aggregated_level as level,
+            os.error_count as error_count,
+            os.warning_count as warning_count,
+            os.default_count as default_count,
+            os.debug_count as debug_count,
+            os.observation_count as observation_count,
+            s.scores_avg as scores_avg,
+            s.score_categories as score_categories,
+            t.public as public`;
+          break;
+        case "rows":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as project_id,
+            t.timestamp as timestamp,
+            t.tags as tags,
+            finalizeAggregation(t.bookmarked) as bookmarked,
+            t.name as name,
+            t.release as release,
+            t.version as version,
+            t.user_id as user_id,
+            t.environment as environment,
+            t.session_id as session_id,
+            finalizeAggregation(t.public) as public`;
+          break;
+        case "identifiers":
+          sqlSelect = `
+            t.id as id,
+            t.project_id as projectId,
+            t.timestamp as timestamp`;
+          break;
+        default:
+          throw new Error(`Unknown select type: ${select}`);
+      }
+
+      const search = clickhouseSearchCondition(
+        searchQuery,
+        searchType,
+        "t",
+        true,
+      );
+
+      const defaultOrder = orderBy?.order && orderBy?.column === "timestamp";
+      const chOrderBy = orderByToClickhouseSql(
+        [
+          defaultOrder ? [{ column: "timestamp", order: orderBy.order }] : null,
+          orderBy ?? null,
+        ].flat(),
+        tracesTableUiColumnDefinitions,
+      );
+
+      const tracesAmt =
+        select === "metrics"
+          ? "traces_all_amt"
+          : getTimeframesTracesAMT(timeStampFilter?.value);
+
+      const query = `
+          ${observationsAndScoresCTE}        
+
+        SELECT ${sqlSelect}
+        FROM ${tracesAmt} t FINAL
+        ${select === "metrics" || requiresObservationsJoin ? `LEFT JOIN observations_stats os on os.project_id = t.project_id and os.trace_id = t.id` : ""}
+        ${select === "metrics" || requiresScoresJoin ? `LEFT JOIN scores_avg s on s.project_id = t.project_id and s.trace_id = t.id` : ""}
+        WHERE t.project_id = {projectId: String}
+        ${tracesFilterRes ? `AND ${tracesFilterRes.query}` : ""}
+        ${search.query}
+        ${chOrderBy}
+        ${limit !== undefined && page !== undefined ? `LIMIT {limit: Int32} OFFSET {offset: Int32}` : ""}
+      `;
+
+      const res = await queryClickhouse<
+        SelectReturnTypeMap[keyof SelectReturnTypeMap]
+      >({
+        query: query,
+        params: {
+          limit: limit,
+          offset: limit && page ? limit * page : 0,
+          traceTimestamp: timeStampFilter?.value.getTime(),
+          projectId: projectId,
+          ...tracesFilterRes.params,
+          ...observationFilterRes.params,
+          ...scoresFilterRes.params,
+          ...search.params,
+        },
+        tags: {
+          ...(props.tags ?? {}),
+          feature: "tracing",
+          type: "traces-table",
+          projectId,
+        },
+        clickhouseConfigs,
+      });
+
+      return res;
+    },
+  });
 }
 
 export const getTracesTableCount = async (props: {

packages/shared/src/server/test-utils/clickhouse-helpers.ts

@@ -3,6 +3,7 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
   convertTraceToTraceMt,
 } from "../repositories/definitions";
 import { env } from "../../env";
@@ -41,3 +42,13 @@ export const createScoresCh = async (scores: ScoreRecordInsertType[]) => {
     values: scores,
   });
 };
+
+export const createDatasetRunItemsCh = async (
+  datasetRunItems: DatasetRunItemRecordInsertType[],
+) => {
+  return await clickhouseClient().insert({
+    table: "dataset_run_items",
+    format: "JSONEachRow",
+    values: datasetRunItems,
+  });
+};

packages/shared/src/server/test-utils/tracing-factory.ts

@@ -3,6 +3,7 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
 } from "../repositories/definitions";
 
 export const createTrace = (
@@ -33,6 +34,32 @@ export const createTrace = (
   };
 };
 
+export const createDatasetRunItem = (
+  datasetRunItem: Partial<DatasetRunItemRecordInsertType>,
+): DatasetRunItemRecordInsertType => {
+  return {
+    id: v4(),
+    project_id: v4(),
+    trace_id: v4(),
+    observation_id: null,
+    dataset_run_id: v4(),
+    dataset_item_id: v4(),
+    dataset_id: v4(),
+    dataset_run_name: "test-run-name" + v4(),
+    dataset_run_metadata: { key: "value" },
+    dataset_item_input: "{}",
+    dataset_item_expected_output: "{}",
+    dataset_item_metadata: { key: "value" },
+    dataset_run_created_at: Date.now(),
+    created_at: Date.now(),
+    updated_at: Date.now(),
+    event_ts: Date.now(),
+    is_deleted: 0,
+    error: datasetRunItem.error ?? null,
+    ...datasetRunItem,
+  };
+};
+
 export const createObservation = (
   observation: Partial<ObservationRecordInsertType>,
 ): ObservationRecordInsertType => {

packages/shared/src/server/utils/headerUtils.ts

@@ -0,0 +1,90 @@
+import { encrypt, decrypt } from "../../encryption";
+
+export function mergeHeaders(
+  legacyHeaders: Record<string, string> = {},
+  requestHeaders: Record<string, { secret: boolean; value: string }> = {},
+): Record<string, { secret: boolean; value: string }> {
+  const merged: Record<string, { secret: boolean; value: string }> = {};
+
+  // Add legacy headers as non-secret
+  for (const [key, value] of Object.entries(legacyHeaders)) {
+    merged[key] = { secret: false, value };
+  }
+
+  // requestHeaders takes precedence
+  for (const [key, headerObj] of Object.entries(requestHeaders)) {
+    merged[key] = headerObj;
+  }
+
+  return merged;
+}
+
+export function createDisplayHeaders(
+  mergedHeaders: Record<string, { secret: boolean; value: string }>,
+): Record<string, { secret: boolean; value: string }> {
+  const displayHeaders: Record<string, { secret: boolean; value: string }> = {};
+
+  for (const [key, headerObj] of Object.entries(mergedHeaders)) {
+    displayHeaders[key] = {
+      secret: headerObj.secret,
+      value: headerObj.secret
+        ? maskSecretValue(headerObj.value)
+        : headerObj.value,
+    };
+  }
+
+  return displayHeaders;
+}
+
+export function encryptSecretHeaders(
+  headers: Record<string, { secret: boolean; value: string }>,
+): Record<string, { secret: boolean; value: string }> {
+  const processedRequestHeaders: Record<
+    string,
+    { secret: boolean; value: string }
+  > = {};
+
+  for (const [key, headerObj] of Object.entries(headers)) {
+    if (headerObj.secret) {
+      processedRequestHeaders[key] = {
+        secret: true,
+        value: encrypt(headerObj.value),
+      };
+    } else {
+      processedRequestHeaders[key] = { secret: false, value: headerObj.value };
+    }
+  }
+
+  return processedRequestHeaders;
+}
+
+export function decryptSecretHeaders(
+  requestHeaders: Record<string, { secret: boolean; value: string }>,
+): Record<string, { secret: boolean; value: string }> {
+  const decryptedRequestHeaders: Record<
+    string,
+    { secret: boolean; value: string }
+  > = {};
+
+  for (const [key, headerObj] of Object.entries(requestHeaders)) {
+    if (headerObj.secret) {
+      try {
+        decryptedRequestHeaders[key] = {
+          secret: true,
+          value: decrypt(headerObj.value),
+        };
+      } catch (error) {
+        console.error(`Failed to decrypt header ${key}:`, error);
+      }
+    } else {
+      decryptedRequestHeaders[key] = { secret: false, value: headerObj.value };
+    }
+  }
+
+  return decryptedRequestHeaders;
+}
+
+function maskSecretValue(value: string): string {
+  if (value.length <= 8) return "****";
+  return `${value.substring(0, 4)}...${value.substring(value.length - 4)}`;
+}

packages/shared/src/utils/zod.ts

@@ -80,14 +80,24 @@ export const paginationMetaResponseZod = z.object({
   totalPages: z.number().int().nonnegative(),
 });
 
-export const htmlRegex = /<[^>]*>/;
-export const noHtmlCheck = (value: string) => !htmlRegex.test(value);
-
 const urlRegex = /https?:\/\/[^\s/$.?#].[^\s]*/i;
 export const noUrlCheck = (value: string) => !urlRegex.test(value);
 
 export const NonEmptyString = z.string().min(1);
 
+export const htmlRegex = /<[^>]*>/;
+
+export const StringNoHTML = z.string().refine((val) => !htmlRegex.test(val), {
+  message: "Text cannot contain HTML tags",
+});
+
+export const StringNoHTMLNonEmpty = z
+  .string()
+  .min(1, "Text cannot be empty")
+  .refine((val) => !htmlRegex.test(val), {
+    message: "Text cannot contain HTML tags",
+  });
+
 /**
  * Validates an object against a Zod schema and helps with IDE type warnings.
  *

turbo.json

@@ -21,17 +21,17 @@
     "dev": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": ["db:generate"]
     },
     "dev:worker": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": ["db:generate"]
     },
     "dev:web": {
       "cache": false,
       "persistent": true,
-      "dependsOn": ["db:generate", "@langfuse/shared#build"]
+      "dependsOn": ["db:generate"]
     },
     "db:generate": {
       "cache": false,

web/.eslintrc.js

@@ -5,4 +5,33 @@ module.exports = {
   parserOptions: {
     project: true,
   },
+  rules: {
+    "no-restricted-imports": [
+      "error",
+      {
+        patterns: [
+          {
+            group: [
+              "react-icons",
+              "react-icons/!(si|tb)",
+              "react-icons/!(si|tb)/*",
+            ],
+            message:
+              "Only react-icons/si and react-icons/tb are allowed. Please use lucide-react for other icons.",
+          },
+        ],
+      },
+    ],
+  },
+  overrides: [
+    {
+      files: [
+        "src/components/nav/support-menu-dropdown.tsx",
+        "src/pages/auth/sign-in.tsx",
+      ],
+      rules: {
+        "no-restricted-imports": "off",
+      },
+    },
+  ],
 };

web/Dockerfile

@@ -1,11 +1,11 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine3.20 AS alpine
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:20-alpine AS alpine
 
 # It's important to update the index before installing packages to ensure you're getting the latest versions.
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client
 
 FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS base
-RUN npm install turbo@^2.5.4 --global
+RUN npm install turbo@^2.5.5 --global
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
 RUN corepack enable

web/jest.config.mjs

@@ -68,8 +68,4 @@ const config = {
   ],
 };
 
-process.env = Object.assign(process.env, {
-  LANGFUSE_CACHE_API_KEY_ENABLED: "true",
-});
-
 export default config;

web/next.config.mjs

@@ -5,6 +5,7 @@
 await import("./src/env.mjs");
 import { withSentryConfig } from "@sentry/nextjs";
 import { env } from "./src/env.mjs";
+import bundleAnalyzer from "@next/bundle-analyzer";
 
 /**
  * CSP headers
@@ -191,7 +192,7 @@ const nextConfig = {
   },
 };
 
-export default withSentryConfig(nextConfig, {
+const sentryConfig = withSentryConfig(nextConfig, {
   // For all available options, see:
   // https://github.com/getsentry/sentry-webpack-plugin#options
 
@@ -232,3 +233,11 @@ export default withSentryConfig(nextConfig, {
   // https://vercel.com/docs/cron-jobs
   automaticVercelMonitors: false,
 });
+
+// Enable bundle analyzer in analyze mode, otherwise use standard config
+const withBundleAnalyzer = bundleAnalyzer({
+  enabled: process.env.ANALYZE === "true",
+  openAnalyzer: false, // Open analyzer in browser
+});
+
+export default withBundleAnalyzer(sentryConfig);

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "3.79.0",
+  "version": "3.87.0",
   "private": true,
   "license": "MIT",
   "engines": {
@@ -24,9 +24,9 @@
     "@anthropic-ai/tokenizer": "^0.0.4",
     "@appsignal/opentelemetry-instrumentation-bullmq": "^0.7.3",
     "@baselime/trpc-opentelemetry-middleware": "^0.1.2",
-    "@codemirror/lang-json": "^6.0.1",
-    "@codemirror/language": "^6.10.1",
-    "@codemirror/lint": "^6.4.2",
+    "@codemirror/lang-json": "^6.0.2",
+    "@codemirror/language": "^6.11.2",
+    "@codemirror/lint": "^6.8.5",
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
@@ -58,32 +58,30 @@
     "@opentelemetry/sdk-trace-node": "^1.26.0",
     "@paralleldrive/cuid2": "^2.2.2",
     "@prisma/instrumentation": "^6.10.1",
-    "@radix-ui/react-accordion": "^1.2.1",
-    "@radix-ui/react-alert-dialog": "^1.1.2",
-    "@radix-ui/react-avatar": "^1.1.1",
-    "@radix-ui/react-checkbox": "^1.1.2",
-    "@radix-ui/react-collapsible": "^1.1.1",
-    "@radix-ui/react-dialog": "^1.1.2",
-    "@radix-ui/react-dropdown-menu": "^2.1.2",
-    "@radix-ui/react-hover-card": "^1.1.2",
-    "@radix-ui/react-icons": "^1.3.0",
-    "@radix-ui/react-label": "^2.1.0",
-    "@radix-ui/react-popover": "^1.1.2",
-    "@radix-ui/react-progress": "^1.1.1",
-    "@radix-ui/react-radio-group": "^1.2.1",
-    "@radix-ui/react-scroll-area": "^1.2.0",
-    "@radix-ui/react-select": "^2.1.2",
-    "@radix-ui/react-separator": "^1.1.0",
-    "@radix-ui/react-slider": "^1.2.1",
-    "@radix-ui/react-slot": "^1.1.0",
-    "@radix-ui/react-switch": "^1.1.1",
-    "@radix-ui/react-tabs": "^1.1.1",
-    "@radix-ui/react-toggle": "^1.1.0",
-    "@radix-ui/react-toggle-group": "^1.1.0",
-    "@radix-ui/react-tooltip": "^1.1.3",
+    "@radix-ui/react-accordion": "^1.2.11",
+    "@radix-ui/react-alert-dialog": "^1.1.14",
+    "@radix-ui/react-avatar": "^1.1.10",
+    "@radix-ui/react-checkbox": "^1.3.2",
+    "@radix-ui/react-collapsible": "^1.1.11",
+    "@radix-ui/react-dialog": "^1.1.14",
+    "@radix-ui/react-dropdown-menu": "^2.1.15",
+    "@radix-ui/react-hover-card": "^1.1.14",
+    "@radix-ui/react-icons": "^1.3.2",
+    "@radix-ui/react-label": "^2.1.7",
+    "@radix-ui/react-popover": "^1.1.14",
+    "@radix-ui/react-progress": "^1.1.7",
+    "@radix-ui/react-radio-group": "^1.3.7",
+    "@radix-ui/react-scroll-area": "^1.2.9",
+    "@radix-ui/react-select": "^2.2.5",
+    "@radix-ui/react-separator": "^1.1.7",
+    "@radix-ui/react-slider": "^1.3.5",
+    "@radix-ui/react-slot": "^1.2.3",
+    "@radix-ui/react-switch": "^1.2.5",
+    "@radix-ui/react-tabs": "^1.1.12",
+    "@radix-ui/react-toggle": "^1.1.9",
+    "@radix-ui/react-toggle-group": "^1.1.10",
+    "@radix-ui/react-tooltip": "^1.2.7",
     "@remixicon/react": "^4.2.0",
-    "@repo/eslint-config": "workspace:*",
-    "@repo/typescript-config": "workspace:*",
     "@sentry/nextjs": "^8.52.0",
     "@t3-oss/env-nextjs": "^0.11.1",
     "@tailwindcss/container-queries": "^0.1.1",
@@ -96,7 +94,7 @@
     "@trpc/react-query": "^10.45.0",
     "@trpc/server": "^10.45.0",
     "@uiw/codemirror-themes": "^4.23.7",
-    "@uiw/react-codemirror": "^4.21.24",
+    "@uiw/react-codemirror": "^4.24.1",
     "ai": "^3.4.9",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.34.10",
@@ -127,6 +125,7 @@
     "posthog-js": "^1.176.0",
     "posthog-node": "^4.3.1",
     "prexit": "^2.2.0",
+    "prism-react-renderer": "^2.4.1",
     "prisma": "^6.10.1",
     "protobufjs": "^7.4.0",
     "rate-limiter-flexible": "^5.0.3",
@@ -135,11 +134,10 @@
     "react-dom": "18.2.0",
     "react-grid-layout": "^1.5.1",
     "react-hook-form": "^7.57.0",
-    "react-icons": "^5.2.1",
+    "react-icons": "^5.5.0",
     "react-markdown": "^9.0.1",
     "react-resizable-panels": "^2.1.1",
     "react-responsive": "^10.0.0",
-    "react-syntax-highlighter": "^15.6.1",
     "react18-json-view": "^0.2.8-canary.6",
     "recharts": "^2.15.2",
     "remark-gfm": "^4.0.0",
@@ -162,7 +160,10 @@
   "devDependencies": {
     "@jedmao/location": "^3.0.0",
     "@mermaid-js/mermaid-cli": "^11.2.0",
+    "@next/bundle-analyzer": "^15.4.1",
     "@playwright/test": "^1.47.2",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
     "@tailwindcss/forms": "^0.5.7",
     "@testing-library/jest-dom": "^6.4.6",
     "@testing-library/react": "^15.0.7",
@@ -173,11 +174,10 @@
     "@types/eslint": "^8.56.7",
     "@types/jest": "^29.5.12",
     "@types/lodash": "^4.17.10",
-    "@types/node": "20.10.5",
+    "@types/node": "20.11.29",
     "@types/react": "~18.2.79",
     "@types/react-dom": "~18.2.25",
     "@types/react-grid-layout": "^1.3.5",
-    "@types/react-syntax-highlighter": "^15.5.13",
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^7.12.0",
@@ -194,17 +194,10 @@
     "tailwindcss": "^3.4.17",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "tsx": "^4.19.1",
     "typescript": "^5.4.5",
     "wait-for-expect": "^3.0.2"
   },
   "ct3aMetadata": {
     "initVersion": "7.13.0"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8",
-      "katex": "0.16.21"
-    }
   }
 }

web/public/generated/api/openapi.yml

@@ -1942,6 +1942,15 @@ paths:
           schema:
             type: string
             nullable: true
+        - name: level
+          in: query
+          description: >-
+            Optional filter for observations with a specific level (e.g.
+            "DEBUG", "DEFAULT", "WARNING", "ERROR").
+          required: false
+          schema:
+            $ref: '#/components/schemas/ObservationLevel'
+            nullable: true
         - name: parentObservationId
           in: query
           required: false

web/public/generated/postman/collection.json

@@ -1462,7 +1462,7 @@
           "request": {
             "description": "Get a list of observations",
             "url": {
-              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&parentObservationId=&environment=&fromStartTime=&toStartTime=&version=",
+              "raw": "{{baseUrl}}/api/public/observations?page=&limit=&name=&userId=&type=&traceId=&level=&parentObservationId=&environment=&fromStartTime=&toStartTime=&version=",
               "host": [
                 "{{baseUrl}}"
               ],
@@ -1502,6 +1502,11 @@
                   "value": "",
                   "description": null
                 },
+                {
+                  "key": "level",
+                  "value": "",
+                  "description": "Optional filter for observations with a specific level (e.g. \"DEBUG\", \"DEFAULT\", \"WARNING\", \"ERROR\")."
+                },
                 {
                   "key": "parentObservationId",
                   "value": "",

web/src/__e2e__/create-project.spec.ts

@@ -108,12 +108,14 @@ test.describe("Create project", () => {
   });
 
   [
-    { title: "Traces", url: "/traces" },
+    { title: "Tracing", url: "/traces", subTitle: "Traces" },
     { title: "Sessions", url: "/sessions" },
-    { title: "Observations", url: "/observations" },
+    { title: "Tracing", url: "/observations", subTitle: "Observations" },
     { title: "Scores", url: "/scores" },
-  ].forEach(({ title, url }) => {
-    test(`Check ${title} page`, async ({ page }) => {
+  ].forEach(({ title, url, subTitle }) => {
+    test(`Check ${title} ${subTitle ? `- ${subTitle}` : ""} page`, async ({
+      page,
+    }) => {
       // const errors = await checkConsoleErrors(page);
       await signin(page);
 

web/src/__tests__/async/datasets-ui-table.servertest.ts

@@ -1,6 +1,6 @@
 import {
   createOrgProjectAndApiKey,
-  getDatasetRunItemsTableCount,
+  getDatasetRunItemsTableCountPg,
 } from "@langfuse/shared/src/server";
 import { v4 as uuidv4 } from "uuid";
 import { prisma } from "@langfuse/shared/src/db";
@@ -65,7 +65,7 @@ describe("trpc.datasets", () => {
   });
   describe("GET datasetItems.countAll", () => {
     it("should GET all dataset run items with no filter", async () => {
-      const { totalCount } = await getDatasetRunItemsTableCount({
+      const { totalCount } = await getDatasetRunItemsTableCountPg({
         projectId: projectId,
         filter: [],
       });
@@ -74,7 +74,7 @@ describe("trpc.datasets", () => {
     });
 
     it("should GET all dataset run items with filter", async () => {
-      const { totalCount } = await getDatasetRunItemsTableCount({
+      const { totalCount } = await getDatasetRunItemsTableCountPg({
         projectId: projectId,
         filter: generateFilter([datasetIds[0]]),
       });

web/src/__tests__/async/observations-api.servertest.ts

@@ -0,0 +1,282 @@
+import {
+  createObservation,
+  createTrace,
+  createTracesCh,
+} from "@langfuse/shared/src/server";
+import { createObservationsCh } from "@langfuse/shared/src/server";
+import { makeZodVerifiedAPICall } from "@/src/__tests__/test-utils";
+import { GetObservationsV1Response } from "@/src/features/public-api/types/observations";
+import { randomUUID } from "crypto";
+
+const projectId = "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a";
+
+describe("/api/public/observations API Endpoint", () => {
+  describe("GET /api/public/observations", () => {
+    it("should fetch all observations with basic data structure", async () => {
+      const traceId = randomUUID();
+      const timestamp = new Date();
+
+      // Create a trace first
+      const createdTrace = createTrace({
+        id: traceId,
+        name: "test-trace",
+        user_id: "user-1",
+        timestamp: timestamp.getTime(),
+        project_id: projectId,
+        metadata: { testKey: "testValue" },
+        release: "1.0.0",
+        version: "2.0.0",
+      });
+
+      // Create multiple observations with different types and levels
+      const observations = [
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "generation-observation",
+          type: "GENERATION",
+          level: "DEFAULT",
+          start_time: timestamp.getTime(),
+          end_time: timestamp.getTime() + 1000,
+          input: "What is the capital of France?",
+          output: "The capital of France is Paris.",
+          provided_model_name: "gpt-4",
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "span-observation",
+          type: "SPAN",
+          level: "DEBUG",
+          start_time: timestamp.getTime() + 500,
+          end_time: timestamp.getTime() + 2000,
+          input: "Processing request",
+          output: "Request processed successfully",
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "event-observation",
+          type: "EVENT",
+          level: "WARNING",
+          start_time: timestamp.getTime() + 1500,
+          input: "User action recorded",
+          metadata: { eventType: "click", target: "submit-button" },
+        }),
+      ];
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh(observations);
+
+      const response = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        "/api/public/observations",
+      );
+
+      expect(response.status).toBe(200);
+      expect(response.body.data).toBeDefined();
+      expect(response.body.meta).toBeDefined();
+      expect(response.body.meta.totalItems).toBeGreaterThanOrEqual(3);
+      expect(response.body.data.length).toBeGreaterThanOrEqual(3);
+
+      // Find our created observations in the response
+      const createdObservations = response.body.data.filter(
+        (obs) => obs.traceId === traceId,
+      );
+      expect(createdObservations.length).toBe(3);
+
+      // Verify data structure and content
+      const generationObs = createdObservations.find(
+        (obs) => obs.type === "GENERATION",
+      );
+      expect(generationObs).toBeDefined();
+      expect(generationObs?.name).toBe("generation-observation");
+      expect(generationObs?.level).toBe("DEFAULT");
+      expect(generationObs?.input).toBe("What is the capital of France?");
+      expect(generationObs?.output).toBe("The capital of France is Paris.");
+      expect(generationObs?.model).toBe("gpt-4");
+      expect(generationObs?.projectId).toBe(projectId);
+      expect(generationObs?.traceId).toBe(traceId);
+
+      const spanObs = createdObservations.find((obs) => obs.type === "SPAN");
+      expect(spanObs).toBeDefined();
+      expect(spanObs?.name).toBe("span-observation");
+      expect(spanObs?.level).toBe("DEBUG");
+      expect(spanObs?.input).toBe("Processing request");
+      expect(spanObs?.output).toBe("Request processed successfully");
+
+      const eventObs = createdObservations.find((obs) => obs.type === "EVENT");
+      expect(eventObs).toBeDefined();
+      expect(eventObs?.name).toBe("event-observation");
+      expect(eventObs?.level).toBe("WARNING");
+      expect(eventObs?.input).toBe("User action recorded");
+      expect(eventObs?.metadata).toEqual({
+        eventType: "click",
+        target: "submit-button",
+      });
+    }, 15_000);
+
+    it("should filter observations by level parameter", async () => {
+      const traceId = randomUUID();
+      const timestamp = new Date();
+
+      // Create a trace
+      const createdTrace = createTrace({
+        id: traceId,
+        project_id: projectId,
+        timestamp: timestamp.getTime(),
+      });
+
+      // Create observations with different levels
+      const observations = [
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "debug-observation",
+          type: "EVENT",
+          level: "DEBUG",
+          start_time: timestamp.getTime(),
+          input: "Debug information",
+          output: "Debug output",
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "default-observation",
+          type: "GENERATION",
+          level: "DEFAULT",
+          start_time: timestamp.getTime() + 1000,
+          input: "Regular operation",
+          output: "Operation completed",
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "warning-observation",
+          type: "SPAN",
+          level: "WARNING",
+          start_time: timestamp.getTime() + 2000,
+          input: "Warning condition detected",
+          output: "Warning handled",
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "error-observation",
+          type: "EVENT",
+          level: "ERROR",
+          start_time: timestamp.getTime() + 3000,
+          input: "Error occurred",
+          output: "Error logged",
+        }),
+      ];
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh(observations);
+
+      // Test filtering by DEBUG level
+      const debugResponse = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?traceId=${traceId}&level=DEBUG`,
+      );
+
+      expect(debugResponse.body.data.length).toBe(1);
+      expect(debugResponse.body.data[0]?.level).toBe("DEBUG");
+      expect(debugResponse.body.data[0]?.name).toBe("debug-observation");
+      expect(debugResponse.body.data[0]?.input).toBe("Debug information");
+
+      // Test filtering by DEFAULT level
+      const defaultResponse = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?traceId=${traceId}&level=DEFAULT`,
+      );
+
+      expect(defaultResponse.body.data.length).toBe(1);
+      expect(defaultResponse.body.data[0]?.level).toBe("DEFAULT");
+      expect(defaultResponse.body.data[0]?.name).toBe("default-observation");
+      expect(defaultResponse.body.data[0]?.type).toBe("GENERATION");
+
+      // Test filtering by WARNING level
+      const warningResponse = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?traceId=${traceId}&level=WARNING`,
+      );
+
+      expect(warningResponse.body.data.length).toBe(1);
+      expect(warningResponse.body.data[0]?.level).toBe("WARNING");
+      expect(warningResponse.body.data[0]?.name).toBe("warning-observation");
+      expect(warningResponse.body.data[0]?.type).toBe("SPAN");
+
+      // Test filtering by ERROR level
+      const errorResponse = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?traceId=${traceId}&level=ERROR`,
+      );
+
+      expect(errorResponse.body.data.length).toBe(1);
+      expect(errorResponse.body.data[0]?.level).toBe("ERROR");
+      expect(errorResponse.body.data[0]?.name).toBe("error-observation");
+      expect(errorResponse.body.data[0]?.input).toBe("Error occurred");
+    });
+
+    it("should return empty results when filtering by level with no matches", async () => {
+      const traceId = randomUUID();
+      const timestamp = new Date();
+
+      // Create a trace
+      const createdTrace = createTrace({
+        id: traceId,
+        project_id: projectId,
+        timestamp: timestamp.getTime(),
+      });
+
+      // Create observations with only DEFAULT level
+      const observations = [
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "default-observation-1",
+          type: "EVENT",
+          level: "DEFAULT",
+          start_time: timestamp.getTime(),
+        }),
+        createObservation({
+          id: randomUUID(),
+          trace_id: traceId,
+          project_id: projectId,
+          name: "default-observation-2",
+          type: "GENERATION",
+          level: "DEFAULT",
+          start_time: timestamp.getTime() + 1000,
+        }),
+      ];
+
+      await createTracesCh([createdTrace]);
+      await createObservationsCh(observations);
+
+      // Test filtering by ERROR level (should return no results)
+      const errorResponse = await makeZodVerifiedAPICall(
+        GetObservationsV1Response,
+        "GET",
+        `/api/public/observations?traceId=${traceId}&level=ERROR`,
+      );
+
+      expect(errorResponse.body.data.length).toBe(0);
+      expect(errorResponse.body.meta.totalItems).toBe(0);
+      expect(errorResponse.body.meta.totalPages).toBe(0);
+    });
+  });
+});

web/src/__tests__/async/prompts-trpc.servertest.ts

@@ -617,4 +617,400 @@ describe("prompts trpc", () => {
       });
     });
   });
+
+  describe("prompts.all with search", () => {
+    it("should find prompts by searching in content, name, tags, and labels", async () => {
+      const { project, caller } = await prepare();
+
+      // Create test prompts with different searchable content
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "customer-service-prompt",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "You are a helpful customer support agent. Answer questions about billing and account issues.",
+          },
+          createdBy: "test-user",
+          tags: ["support", "customer"],
+          labels: ["production"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "marketing-prompt",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "Create engaging marketing content that drives sales and conversions.",
+          },
+          createdBy: "test-user",
+          tags: ["marketing", "content"],
+          labels: ["staging"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "technical-docs",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "Generate comprehensive technical documentation for APIs and software systems.",
+          },
+          createdBy: "test-user",
+          tags: ["technical", "documentation"],
+          labels: ["latest"],
+        },
+      });
+
+      // Test 1: Search by prompt content
+      const contentSearchResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "customer support agent",
+        searchType: ["content"],
+      });
+
+      expect(contentSearchResults.prompts).toHaveLength(1);
+      expect(contentSearchResults.prompts[0].name).toBe(
+        "customer-service-prompt",
+      );
+
+      // Test 2: Search by prompt name
+      const nameSearchResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "marketing",
+        searchType: ["id"],
+      });
+
+      expect(nameSearchResults.prompts).toHaveLength(1);
+      expect(nameSearchResults.prompts[0].name).toBe("marketing-prompt");
+
+      // Test 3: Search by tags
+      const tagSearchResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "documentation",
+        searchType: ["id"],
+      });
+
+      expect(tagSearchResults.prompts).toHaveLength(1);
+      expect(tagSearchResults.prompts[0].name).toBe("technical-docs");
+
+      // Test 4: Search with no matches
+      const noMatchResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "nonexistent content",
+        searchType: ["id", "content"],
+      });
+
+      expect(noMatchResults.prompts).toHaveLength(0);
+
+      // Test 5: Case insensitive search
+      const caseInsensitiveResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "TECHNICAL",
+        searchType: ["id"],
+      });
+
+      expect(caseInsensitiveResults.prompts).toHaveLength(1);
+      expect(caseInsensitiveResults.prompts[0].name).toBe("technical-docs");
+    });
+
+    it("should find prompts with multiple versions when searching content", async () => {
+      const { project, caller } = await prepare();
+
+      // Create multiple versions of the same prompt with different content
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "evolving-prompt",
+          version: 1,
+          type: "text",
+          prompt: { text: "You are a basic chatbot. Answer simple questions." },
+          createdBy: "test-user",
+          tags: ["basic"],
+          labels: [],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "evolving-prompt",
+          version: 2,
+          type: "text",
+          prompt: {
+            text: "You are an advanced AI assistant with expertise in machine learning.",
+          },
+          createdBy: "test-user",
+          tags: ["advanced"],
+          labels: ["latest"],
+        },
+      });
+
+      // Search for content that exists in version 2 but not version 1
+      const searchResults = await caller.prompts.all({
+        projectId: project.id,
+        page: 0,
+        limit: 10,
+        filter: [],
+        orderBy: { column: "createdAt", order: "DESC" },
+        searchQuery: "machine learning",
+        searchType: ["content"],
+      });
+
+      // Should find the prompt because one of its versions contains the search term
+      expect(searchResults.prompts).toHaveLength(1);
+      expect(searchResults.prompts[0].name).toBe("evolving-prompt");
+      // The returned prompt should be the latest version
+      expect(searchResults.prompts[0].version).toBe(2);
+    });
+  });
+
+  describe("prompts.count with search and searchType", () => {
+    it("should count prompts correctly with different search types", async () => {
+      const { project, caller } = await prepare();
+
+      // Create test prompts with different searchable content
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "customer-service-prompt",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "You are a helpful customer support agent. Answer questions about billing and account issues.",
+          },
+          createdBy: "test-user",
+          tags: ["support", "customer"],
+          labels: ["production"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "marketing-prompt",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "Create engaging marketing content that drives sales and conversions.",
+          },
+          createdBy: "test-user",
+          tags: ["marketing", "content"],
+          labels: ["staging"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "technical-docs",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "Generate comprehensive technical documentation for APIs and software systems.",
+          },
+          createdBy: "test-user",
+          tags: ["technical", "documentation"],
+          labels: ["latest"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "sales-support-prompt",
+          version: 1,
+          type: "text",
+          prompt: {
+            text: "Assist sales teams with lead qualification and customer outreach.",
+          },
+          createdBy: "test-user",
+          tags: ["sales", "support"],
+          labels: ["production"],
+        },
+      });
+
+      // Test 1: Count all prompts (no search)
+      const totalCountResult = await caller.prompts.count({
+        projectId: project.id,
+      });
+      expect(totalCountResult.totalCount).toBe(BigInt(4));
+
+      // Test 2: Count with ID search type (default) - search in names and tags
+      const idSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "support",
+        searchType: ["id"],
+      });
+      // Should find "customer-service-prompt" and "sales-support-prompt" (by name and tags)
+      expect(idSearchCountResult.totalCount).toBe(BigInt(2));
+
+      // Test 3: Count with content search type - search in prompt content
+      const contentSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "customer",
+        searchType: ["content"],
+      });
+      // Should find "customer-service-prompt" and "sales-support-prompt" (by content: "customer" appears in both)
+      expect(contentSearchCountResult.totalCount).toBe(BigInt(2));
+
+      // Test 4: Count with both ID and content search types
+      const combinedSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "marketing",
+        searchType: ["id", "content"],
+      });
+      // Should find "marketing-prompt" (by both name and content)
+      expect(combinedSearchCountResult.totalCount).toBe(BigInt(1));
+
+      // Test 5: Count with tag-specific search (ID search type)
+      const tagSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "documentation",
+        searchType: ["id"],
+      });
+      // Should find "technical-docs" (by tag)
+      expect(tagSearchCountResult.totalCount).toBe(BigInt(1));
+
+      // Test 6: Count with content that doesn't exist in names/tags but exists in prompt text
+      const contentOnlySearchResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "billing",
+        searchType: ["content"],
+      });
+      // Should find "customer-service-prompt" (by content only)
+      expect(contentOnlySearchResult.totalCount).toBe(BigInt(1));
+
+      // Test 7: Count with no matches
+      const noMatchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "nonexistent",
+        searchType: ["id", "content"],
+      });
+      expect(noMatchCountResult.totalCount).toBe(BigInt(0));
+
+      // Test 8: Count with case insensitive search
+      const caseInsensitiveCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "TECHNICAL",
+        searchType: ["id"],
+      });
+      // Should find "technical-docs" (case insensitive)
+      expect(caseInsensitiveCountResult.totalCount).toBe(BigInt(1));
+
+      // Test 9: Count with filters and search combined
+      const filteredSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        searchQuery: "support",
+        searchType: ["id"],
+        filter: [
+          {
+            column: "labels",
+            type: "arrayOptions",
+            operator: "any of",
+            value: ["production"],
+          },
+        ],
+      });
+      // Should find prompts with "support" AND "production" label
+      expect(filteredSearchCountResult.totalCount).toBe(BigInt(2));
+    });
+
+    it("should count prompts with folder path prefix and search", async () => {
+      const { project, caller } = await prepare();
+
+      // Create prompts in different folder structures
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "folder1/customer-prompt",
+          version: 1,
+          type: "text",
+          prompt: { text: "Customer service prompt" },
+          createdBy: "test-user",
+          tags: ["customer"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "folder1/support-prompt",
+          version: 1,
+          type: "text",
+          prompt: { text: "Technical support prompt" },
+          createdBy: "test-user",
+          tags: ["support"],
+        },
+      });
+
+      await prisma.prompt.create({
+        data: {
+          id: v4(),
+          projectId: project.id,
+          name: "folder2/customer-prompt",
+          version: 1,
+          type: "text",
+          prompt: { text: "Another customer prompt" },
+          createdBy: "test-user",
+          tags: ["customer"],
+        },
+      });
+
+      // Test: Count with path prefix and search
+      const folderSearchCountResult = await caller.prompts.count({
+        projectId: project.id,
+        pathPrefix: "folder1",
+        searchQuery: "customer",
+        searchType: ["id"],
+      });
+      // Should only find prompts in folder1 that match "customer"
+      expect(folderSearchCountResult.totalCount).toBe(BigInt(1));
+
+      // Test: Count with path prefix but no search
+      const folderCountResult = await caller.prompts.count({
+        projectId: project.id,
+        pathPrefix: "folder1",
+      });
+      // Should find all prompts in folder1
+      expect(folderCountResult.totalCount).toBe(BigInt(2));
+    });
+  });
 });

web/src/__tests__/async/prompts.v2.servertest.ts

@@ -922,6 +922,9 @@ describe("/api/public/v2/prompts API Endpoint", () => {
         );
         expect(response.status).toBe(400);
         expect(response.body.message).toBe("Invalid request data");
+        expect(JSON.stringify(response.body.error)).toContain(
+          `"message":"${expectedError}"`,
+        );
         const hasExpectedMessage = JSON.stringify(response.body.error).includes(
           `"message":"${expectedError}"`,
         );
@@ -969,7 +972,12 @@ describe("/api/public/v2/prompts API Endpoint", () => {
           auth,
         );
         await testInvalidName("new", "Prompt name cannot be 'new'", auth);
-        await testInvalidName("", "Enter a name", auth);
+        await testInvalidName("", "Text cannot be empty", auth);
+        await testInvalidName(
+          "Test <div>",
+          "Text cannot contain HTML tags",
+          auth,
+        );
       });
 
       it("should accept valid prompt names", async () => {

web/src/__tests__/async/sessions-trpc.servertest.ts

@@ -52,7 +52,7 @@ describe("traces trpc", () => {
   const ctx = createInnerTRPCContext({ session });
   const caller = appRouter.createCaller({ ...ctx, prisma });
 
-  describe("sessions.byId", () => {
+  describe("sessions.byIdWithScores", () => {
     it("access private session", async () => {
       const sessionId = randomUUID();
 
@@ -92,7 +92,7 @@ describe("traces trpc", () => {
 
       await createObservationsCh([observation, observation2, observation3]);
 
-      const sessionRes = await caller.sessions.byId({
+      const sessionRes = await caller.sessions.byIdWithScores({
         projectId,
         sessionId,
       });
@@ -105,6 +105,7 @@ describe("traces trpc", () => {
         environment: "default",
         bookmarked: false,
         public: false,
+        scores: [],
         traces: expect.arrayContaining([
           expect.objectContaining({
             id: trace.id,

web/src/__tests__/async/sessions-ui-table.servertest.ts

@@ -383,7 +383,6 @@ describe("trpc.sessions", () => {
 
     expect(sessions[0]).toBeDefined();
     expect(sessions[0]?.trace_count).toBe(2);
-    expect(parseInt(sessions[0]?.duration as any)).toBeGreaterThan(995);
-    expect(parseInt(sessions[0]?.duration as any)).toBeLessThan(1005);
+    expect(parseInt(sessions[0]?.duration as any)).toBe(1);
   });
 });

web/src/__tests__/compileChatMessages.servertest.ts

@@ -65,35 +65,48 @@ describe("compileChatMessages", () => {
     }).toThrow("Missing value for message placeholder: missing_placeholder");
   });
 
-  it("should throw error when placeholder messages lack required properties", () => {
+  it("should allow arbitrary placeholder fill-in values", () => {
     const promptTemplate = [
+      { role: "system", content: "You are a helpful assistant." },
       {
         type: ChatMessageType.Placeholder,
-        name: "invalid_messages",
+        name: "arbitrary_messages",
       },
     ];
 
-    // Test missing role property
-    const placeholderValuesNoRole = {
-      invalid_messages: [{ content: "Hello" }],
+    // Test with arbitrary objects (non-role/content structure)
+    const placeholderValues = {
+      arbitrary_messages: [
+        { type: "custom", data: ["some data", "and some more data!"], id: 123 },
+        { action: "click", target: "button", value: "submit" },
+        { role: "user", content: "This still works" }, // Mixed with standard format
+      ],
     };
 
-    expect(() => {
-      compileChatMessages(promptTemplate, placeholderValuesNoRole);
-    }).toThrow(
-      "Invalid message format in placeholder 'invalid_messages': messages must have 'role' and 'content' properties",
+    const compiledMessages = compileChatMessages(
+      promptTemplate,
+      placeholderValues,
     );
 
-    // Test missing content property
-    const placeholderValuesNoContent = {
-      invalid_messages: [{ role: "user" }],
-    };
-
-    expect(() => {
-      compileChatMessages(promptTemplate, placeholderValuesNoContent);
-    }).toThrow(
-      "Invalid message format in placeholder 'invalid_messages': messages must have 'role' and 'content' properties",
-    );
+    expect(compiledMessages).toHaveLength(4);
+    expect(compiledMessages[0]).toEqual({
+      role: "system",
+      content: "You are a helpful assistant.",
+    });
+    expect(compiledMessages[1]).toEqual({
+      type: "custom",
+      data: ["some data", "and some more data!"],
+      id: 123,
+    });
+    expect(compiledMessages[2]).toEqual({
+      action: "click",
+      target: "button",
+      value: "submit",
+    });
+    expect(compiledMessages[3]).toEqual({
+      role: "user",
+      content: "This still works",
+    });
   });
 
   it("should compile placeholders without applying text substitutions when no variables provided", () => {

web/src/__tests__/model-definitions.servertest.ts

@@ -40,9 +40,19 @@ describe("/models API Endpoints", () => {
     });
     await prisma.price.createMany({
       data: [
-        { modelId: "model-1", usageType: "input", price: 0.001 },
-        { modelId: "model-1", usageType: "output", price: 0.002 },
-        { modelId: "model-1", usageType: "total", price: 0.1 },
+        {
+          modelId: "model-1",
+          projectId: null,
+          usageType: "input",
+          price: 0.001,
+        },
+        {
+          modelId: "model-1",
+          projectId: null,
+          usageType: "output",
+          price: 0.002,
+        },
+        { modelId: "model-1", projectId: null, usageType: "total", price: 0.1 },
       ],
     });
     await prisma.model.create({
@@ -60,8 +70,18 @@ describe("/models API Endpoints", () => {
     });
     await prisma.price.createMany({
       data: [
-        { modelId: "model-2", usageType: "input", price: 0.002 },
-        { modelId: "model-2", usageType: "output", price: 0.004 },
+        {
+          modelId: "model-2",
+          projectId: null,
+          usageType: "input",
+          price: 0.02,
+        },
+        {
+          modelId: "model-2",
+          projectId: null,
+          usageType: "output",
+          price: 0.04,
+        },
       ],
     });
   });

web/src/__tests__/promptCache.servertest.ts

@@ -97,7 +97,7 @@ describe("PromptService", () => {
         "prompt:project1:testPrompt:1",
         JSON.stringify(mockPrompt),
         "EX",
-        3600,
+        300,
       );
 
       expect(mockRedis.sadd).toHaveBeenCalledWith(
@@ -200,7 +200,6 @@ describe("PromptService", () => {
         mockPrisma,
         null,
         mockMetricIncrementer,
-        true,
       );
     });