chore: release v3.97.2

* fix: fail silently when /api/latest-releases returns invalid schema

* push

.claude/settings.json

@@ -4,8 +4,11 @@
       "Bash(find:*)",
       "Bash(rg:*)",
       "Bash(grep:*)",
-      "Bash(pnpm run test:*)"
+      "Bash(ls:*)",
+      "Bash(cat:*)",
+      "Bash(head:*)",
+      "Bash(tail:*)"
     ],
     "deny": []
   }
-}
+}

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx,notIn
+ignore-words-list = afterall,vertx,notIn,alue
 

.devcontainer.json

@@ -1,6 +0,0 @@
-{
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
-}

.devcontainer/Dockerfile

@@ -0,0 +1,13 @@
+# Dev container Dockerfile
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:20-bookworm
+
+# Install golang-migrate for database migrations
+RUN curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && \
+    chmod +x migrate && \
+    mv migrate /usr/local/bin/migrate
+
+# Install pnpm globally
+RUN npm install -g pnpm@9.5.0
+
+# Install Claude Code CLI
+RUN npm install -g @anthropic-ai/claude-code

.devcontainer/devcontainer.json

@@ -0,0 +1,8 @@
+{
+  "name": "langfuse-development",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "postCreateCommand": "cp .env.dev.example .env && pnpm i"
+}

.env.dev-azure.example

@@ -68,6 +68,7 @@ LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
 LANGFUSE_S3_EVENT_UPLOAD_PREFIX=events/
 
 LANGFUSE_USE_AZURE_BLOB=true
+LANGFUSE_AZURE_SKIP_CONTAINER_CHECK=false
 
 # Set during docker build of application
 # Used to disable environment verification at build time

.env.dev-redis-cluster.example

@@ -76,9 +76,18 @@ REDIS_AUTH="bitnami"
 REDIS_CLUSTER_ENABLED="true"
 REDIS_CLUSTER_NODES="127.0.0.1:6370,127.0.0.1:6371,127.0.0.1:6372,127.0.0.1:6373,127.0.0.1:6374,127.0.0.1:6375"
 LANGFUSE_INGESTION_QUEUE_SHARD_COUNT=8
+LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT=4
 
 # openssl rand -hex 32 used only here
 ENCRYPTION_KEY=0000000000000000000000000000000000000000000000000000000000000000 
 
 # speeds up local development by not executing init scripts on server startup
 NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
+
+# Use the following settings to enforce running the new AMTs during the tests
+LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES="true"
+LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a"
+LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT="true"
+LANGFUSE_EXPERIMENT_SAMPLING_RATE=1

.env.dev.example

@@ -83,3 +83,8 @@ NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT="false"
 # For SDK integration tests to pass, decrease the ingestion queue delay by uncommenting the env vars:
 # LANGFUSE_INGESTION_QUEUE_DELAY_MS=10
 # LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10
+
+# Slack credentials for development
+SLACK_CLIENT_ID=your_slack_client_id
+SLACK_CLIENT_SECRET=your_slack_client_secret
+SLACK_STATE_SECRET=your_slack_state_secret

.env.prod.example

@@ -172,6 +172,7 @@ OTEL_SERVICE_NAME="langfuse"
 # REDIS_HOST=
 # REDIS_PORT=
 # REDIS_AUTH=
+# REDIS_USERNAME=default
 # REDIS_CONNECTION_STRING=
 # REDIS_ENABLE_AUTO_PIPELINING=
 

.github/dependabot.yml

@@ -36,6 +36,7 @@ updates:
         patterns:
           - "express"
           - "@types/express"
+          - "@types/express-serve-static-core"
       observability:
         patterns:
           - "dd-trace"

.github/workflows/_deploy_ecs_service.yml

@@ -15,10 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - name: Get app name
         uses: winterjung/split@v2
         id: split

.github/workflows/pipeline.yml

@@ -114,7 +114,9 @@ jobs:
         run: echo "NEXT_PUBLIC_BUILD_ID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Build and run both images from compose
         run: |
-          docker compose -f docker-compose.build.yml up -d
+          docker compose --progress plain --verbose -f docker-compose.build.yml build --print > /tmp/bake.json
+          docker buildx bake -f /tmp/bake.json
+          docker compose --progress plain -f docker-compose.build.yml up -d
           sleep 5 # Wait for PostgreSQL to accept connections
       - name: Ensure no unhealthy status
         run: |
@@ -172,6 +174,7 @@ jobs:
           cp .env.dev.example .env
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=false" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run dev containers
         run: |
@@ -434,8 +437,6 @@ jobs:
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
-          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -484,13 +485,21 @@ jobs:
     if: always()
     steps:
       - name: Successful deploy
-        if: ${{ !(contains(needs.*.result, 'failure')) }}
+        if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
         run: exit 0
         working-directory: .
       - name: Failing deploy
-        if: ${{ contains(needs.*.result, 'failure') }}
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
+      - name: Notify Slack
+        uses: ravsamhq/notify-slack-action@v2
+        if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
   push-docker-image:
     needs: all-ci-passed

.github/workflows/snyk.yml

@@ -1,65 +1,28 @@
 name: Snyk Container
 on:
-  pull_request:
-    branches:
-      - "**"
   push:
-    branches:
-      - main
-  # Snyk cannot upload results in merge group. Hence, we only run on PRs and when pushingon the main branch https://github.com/github/codeql-action/issues/1572
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    branches: ["main"]
 
 jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Build a Docker image
-        run: docker compose -f docker-compose.build.yml up -d
+      - uses: actions/checkout@v3
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
-        continue-on-error: true
+      - name: Scan web image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-web
-          args: --file=web/Dockerfile
+          image: langfuse/langfuse # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)
 
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: web
-
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
-        continue-on-error: true
+      - name: Scan worker image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-worker
-          args: --file=worker/Dockerfile
-
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: worker
+          image: langfuse/langfuse-worker # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)

.gitignore

@@ -50,15 +50,13 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 
-/generated/typescript-server
+/generated
 
 # openapi spec that is copied during build
 /public/openapi*.yml
 
 
 # vscode
-.devcontainer
-
 node_modules
 **/node_modules
 **/dist
@@ -67,4 +65,7 @@ node_modules
 .yarn
 .turbo
 
-web/test-results/*
+web/test-results/*
+
+# local config files
+*.local.*

.nvmrc

@@ -1 +1 @@
-v20
+v20.19.2

CLAUDE.md

@@ -192,3 +192,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## General Coding Guidelines
 - For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so
+
+## Development Tips
+- Before trying to build the package, try running the linter once first

CONTRIBUTING.md

@@ -137,7 +137,7 @@ Requirements
     cp .env.dev.example .env
    ```
 
-4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it.
+4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it. Also, this will fail on the very first run. Please run it again.
 
    ```bash
    pnpm run dx # first run only (resets db, docker containers, etc...)

README.cn.md

@@ -94,7 +94,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
   
 - [提示管理](https://langfuse.com/docs/prompts/get-started) 帮助你集中管理、版本控制并协作迭代提示。得益于服务器和客户端的高效缓存，你可以在不增加延迟的情况下反复迭代提示。
 
-- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为“裁判”、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
+- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为"裁判"、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
 
 - [数据集](https://langfuse.com/docs/datasets/overview) 为评估你的 LLM 应用提供测试集和基准。它们支持持续改进、部署前测试、结构化实验、灵活评估，并能与 LangChain、LlamaIndex 等框架无缝整合。
 
@@ -135,7 +135,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 
 - [虚拟机](https://langfuse.com/self-hosting/docker-compose)：使用 Docker Compose 在单台虚拟机上部署 Langfuse。
 
-- 【计划中】：针对各云平台的部署指南，欢迎在以下讨论中投票和评论：[AWS](https://github.com/orgs/langfuse/discussions/4645)、[Google Cloud](https://github.com/langfuse/discussions/4646)、[Azure](https://github.com/orgs/langfuse/discussions/4647)。
+- Terraform 模板: [AWS](https://langfuse.com/self-hosting/aws)、[Azure](https://langfuse.com/self-hosting/azure)、[GCP](https://langfuse.com/self-hosting/gcp)
 
 请参阅 [自托管文档](https://langfuse.com/self-hosting) 了解更多关于架构和配置选项的信息。
 
@@ -155,6 +155,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | [LiteLLM](https://langfuse.com/docs/integrations/litellm)               | Python, JS/TS (仅代理)     | 允许使用任何 LLM 替代 GPT。支持 Azure、OpenAI、Cohere、Anthropic、Ollama、VLLM、Sagemaker、HuggingFace、Replicate（100+ LLMs）。         |
 | [Vercel AI SDK](https://langfuse.com/docs/integrations/vercel-ai-sdk)     | JS/TS                    | 基于 TypeScript 的工具包，帮助开发者使用 React、Next.js、Vue、Svelte 和 Node.js 构建 AI 驱动的应用。                                           |
 | [API](https://langfuse.com/docs/api)                                    |                          | 直接调用公共 API。提供 OpenAPI 规格。                                                                                                       |
+| [Google VertexAI 和 Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 模型                | 在 Google 上运行基础模型和微调模型。                                                                         |
 
 ### 与 Langfuse 集成的软件包：
 
@@ -162,9 +163,9 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | ---------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------- |
 | [Instructor](https://langfuse.com/docs/integrations/instructor)         | 库                  | 用于获取结构化 LLM 输出（JSON、Pydantic）的库。                                                         |
 | [DSPy](https://langfuse.com/docs/integrations/dspy)                     | 库                  | 一个系统性优化语言模型提示和权重的框架。                                                                 |
+| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 库                  | 构建 LLM 应用的 Python 工具包。                                                                           |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 模型（本地）         | 在你的机器上轻松运行开源 LLM。                                                                            |
-| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 代理框架             | 用于构建分布式代理的开源 LLM 平台。                                                                       |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 聊天/代理界面         | 基于 JS/TS 的无代码构建器，用于定制化 LLM 流程。                                                          |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 聊天/代理界面         | 基于 Python 的 LangChain 用户界面，采用 react-flow 设计，提供便捷的实验与原型构建体验。                   |
@@ -253,7 +254,7 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 
 - 我们的 [文档](https://langfuse.com/docs) 是查找答案的最佳起点。内容全面，我们投入大量时间进行维护。你也可以通过 GitHub 提出文档修改建议。
 - [Langfuse 常见问题](https://langfuse.com/faq) 解答了最常见的问题。
-- 使用 “[Ask AI](https://langfuse.com/docs/ask-ai)” 立即获取问题答案。
+- 使用 "Ask AI" 立即获取问题答案。
 
 支持渠道：
 
@@ -351,4 +352,4 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 所有数据均不会与第三方共享，也不包含任何敏感信息。我们对这一过程保持高度透明，你可以在 [此处](/web/src/features/telemetry/index.ts) 查看我们收集的具体数据。
 
 你可以通过设置 `TELEMETRY_ENABLED=false` 来选择退出。
-````
+```

README.ja.md

@@ -141,9 +141,7 @@ Langfuseチームによるマネージドデプロイメント。充実した無
 - **[VM](https://langfuse.com/self-hosting/docker-compose):**  
   Docker Composeを使用して、単一の仮想マシン上でLangfuseを実行します。
 
-- **Planned:**  
-  クラウド固有のデプロイガイドは計画中です。以下のスレッドに対して投票やコメントをお願いします:  
-  [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform テンプレート: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 [セルフホスティングのドキュメント](https://langfuse.com/self-hosting)を参照し、アーキテクチャや設定オプションの詳細をご確認ください。
 

README.kr.md

@@ -128,7 +128,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 
 - [Kubernetes (Helm)](https://langfuse.com/self-hosting/kubernetes-helm): Helm을 사용해 Kubernetes 클러스터에서 Langfuse를 실행합니다. 이는 권장되는 프로덕션 배포 방식입니다.
 - [VM](https://langfuse.com/self-hosting/docker-compose): Docker Compose를 사용해 단일 가상 머신에서 Langfuse를 실행합니다.
-- 예정: 클라우드별 배포 가이드 – 아래 스레드에서 투표 및 댓글을 남겨주세요: [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform 템플릿: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 자세한 내용은 [자체 호스팅 문서](https://langfuse.com/self-hosting)를 참조하세요.
 
@@ -158,6 +158,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 라이브러리         | LLM 애플리케이션 구축을 위한 Python 툴킷입니다.                                                                    |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 모델 (로컬)        | 자신의 컴퓨터에서 오픈 소스 LLM을 손쉽게 실행할 수 있습니다.                                                      |
 | [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 모델               | AWS에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
+| [Google VertexAI and Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 모델               | Google에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 에이전트 프레임워크 | 분산 에이전트 구축을 위한 오픈 소스 LLM 플랫폼입니다.                                                              |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 채팅/에이전트 UI    | 맞춤형 LLM 플로우를 위한 JS/TS 코드 없는(no-code) 빌더입니다.                                                       |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 채팅/에이전트 UI    | react-flow를 활용하여 실험 및 프로토타이핑을 손쉽게 할 수 있도록 디자인된 LangChain용 Python 기반 UI입니다.          |

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     ports:
       - 127.0.0.1:3030:3030
     environment: &langfuse-worker-env
+      NEXTAUTH_URL: http://localhost:3000
       DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres # CHANGEME
       SALT: "mysalt" # CHANGEME
       ENCRYPTION_KEY: "0000000000000000000000000000000000000000000000000000000000000000" # CHANGEME: generate via `openssl rand -hex 32`
@@ -62,6 +63,8 @@ services:
       REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
       REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
       REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
+      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-}
+      SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-}
 
   langfuse-web:
     image: docker.io/langfuse/langfuse:3
@@ -71,7 +74,6 @@ services:
       - 3000:3000
     environment:
       <<: *langfuse-worker-env
-      NEXTAUTH_URL: http://localhost:3000
       NEXTAUTH_SECRET: mysecret # CHANGEME
       LANGFUSE_INIT_ORG_ID: ${LANGFUSE_INIT_ORG_ID:-}
       LANGFUSE_INIT_ORG_NAME: ${LANGFUSE_INIT_ORG_NAME:-}

ee/package.json

@@ -26,7 +26,6 @@
   "dependencies": {
     "@langfuse/shared": "workspace:*",
     "@opentelemetry/api": ">=1.0.0 <1.10.0",
-    "axios": "^1.8.2",
     "https-proxy-agent": "^7.0.6",
     "next": "^14.2.30",
     "next-auth": "^4.24.11",
@@ -45,10 +44,5 @@
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

fern/apis/server/definition/annotation-queues.yml

@@ -105,6 +105,28 @@ service:
           docs: The unique identifier of the annotation queue item
       response: DeleteAnnotationQueueItemResponse
 
+    createQueueAssignment:
+      docs: Create an assignment for a user to an annotation queue
+      method: POST
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: CreateAnnotationQueueAssignmentResponse
+
+    deleteQueueAssignment:
+      docs: Delete an assignment for a user to an annotation queue
+      method: DELETE
+      path: /annotation-queues/{queueId}/assignments
+      path-parameters:
+        queueId:
+          type: string
+          docs: The unique identifier of the annotation queue
+      request: AnnotationQueueAssignmentRequest
+      response: DeleteAnnotationQueueAssignmentResponse
+
 types:
   AnnotationQueueStatus:
     enum:
@@ -163,3 +185,17 @@ types:
     properties:
       success: boolean
       message: string
+
+  AnnotationQueueAssignmentRequest:
+    properties:
+      userId: string
+
+  DeleteAnnotationQueueAssignmentResponse:
+    properties:
+      success: boolean
+
+  CreateAnnotationQueueAssignmentResponse:
+    properties:
+      userId: string
+      queueId: string
+      projectId: string

fern/apis/server/definition/llm-connections.yml

@@ -0,0 +1,102 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fern-api/fern/main/fern.schema.json
+imports:
+  commons: ./commons.yml
+  pagination: ./utils/pagination.yml
+service:
+  auth: true
+  base-path: /api/public
+  endpoints:
+    list:
+      method: GET
+      docs: Get all LLM connections in a project
+      path: /llm-connections
+      request:
+        name: GetLlmConnectionsRequest
+        query-parameters:
+          page:
+            type: optional<integer>
+            docs: page number, starts at 1
+          limit:
+            type: optional<integer>
+            docs: limit of items per page
+      response: PaginatedLlmConnections
+    upsert:
+      method: PUT
+      docs: Create or update an LLM connection. The connection is upserted on provider.
+      path: /llm-connections
+      request: UpsertLlmConnectionRequest
+      response: LlmConnection
+
+types:
+  LlmConnection:
+    docs: LLM API connection configuration (secrets excluded)
+    properties:
+      id: string
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: string
+        docs: The adapter used to interface with the LLM
+      displaySecretKey:
+        type: string
+        docs: Masked version of the secret key for display purposes
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: list<string>
+        docs: List of custom model names available for this connection
+      withDefaultModels:
+        type: boolean
+        docs: Whether to include default models for this adapter
+      extraHeaderKeys:
+        type: list<string>
+        docs: Keys of extra headers sent with requests (values excluded for security)
+      createdAt: datetime
+      updatedAt: datetime
+
+  PaginatedLlmConnections:
+    properties:
+      data: list<LlmConnection>
+      meta: pagination.MetaResponse
+
+  UpsertLlmConnectionRequest:
+    docs: Request to create or update an LLM connection (upsert)
+    properties:
+      provider:
+        type: string
+        docs: Provider name (e.g., 'openai', 'my-gateway'). Must be unique in project, used for upserting.
+      adapter:
+        type: LlmAdapter
+        docs: The adapter used to interface with the LLM
+      secretKey:
+        type: string
+        docs: Secret key for the LLM API.
+      baseURL:
+        type: optional<string>
+        docs: Custom base URL for the LLM API
+      customModels:
+        type: optional<list<string>>
+        docs: List of custom model names
+      withDefaultModels:
+        type: optional<boolean>
+        docs: Whether to include default models. Default is true.
+      extraHeaders:
+        type: optional<map<string, string>>
+        docs: Extra headers to send with requests
+
+  LlmAdapter:
+    enum:
+      - value: anthropic
+        name: Anthropic
+      - value: openai
+        name: OpenAI
+      - value: azure
+        name: Azure
+      - value: bedrock
+        name: Bedrock
+      - value: google-vertex-ai
+        name: GoogleVertexAI
+      - value: google-ai-studio
+        name: GoogleAIStudio

fern/apis/server/definition/observations.yml

@@ -32,6 +32,9 @@ service:
           userId: optional<string>
           type: optional<string>
           traceId: optional<string>
+          level:
+            type: optional<commons.ObservationLevel>
+            docs: Optional filter for observations with a specific level (e.g. "DEBUG", "DEFAULT", "WARNING", "ERROR").
           parentObservationId: optional<string>
           environment:
             type: optional<string>

fern/apis/server/generators.yml

@@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://schema.buildwithfern.dev/generators-yml.json
 default-group: local
 groups:
   local:
@@ -7,6 +8,7 @@ groups:
         output:
           location: local-file-system
           path: ../../../web/public/generated/api
+
       - name: fernapi/fern-python-sdk
         version: 2.16.0
         output:
@@ -19,35 +21,32 @@ groups:
           pydantic_config:
             require_optional_fields: false
             use_str_enums: false
-#      - name: fernapi/fern-java-sdk
-#        version: 2.20.1
-#        output:
-#          location: local-file-system
-#          path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
-#        config:
-#          client-class-name: LangfuseClient
+
+      # - name: fernapi/fern-java-sdk
+      #   version: 2.20.1
+      #   output:
+      #     location: local-file-system
+      #     path: ../../../../langfuse-java/src/main/java/com/langfuse/client/
+      #   config:
+      #     client-class-name: LangfuseClient
+
+      - name: fernapi/fern-typescript-node-sdk
+        version: 2.6.1
+        output:
+          location: local-file-system
+          path: ../../../generated/typescript
+        config:
+          namespaceExport: LangfuseAPI
+          outputSourceFiles: true
+          skipResponseValidation: true
+          fetchSupport: native
+          formDataSupport: Node18
+          fileResponseType: binary-response
+          streamType: web
+          omitFernHeaders: true
+
       - name: fernapi/fern-postman
         version: 0.0.45
         output:
           location: local-file-system
           path: ../../../web/public/generated/postman
-  # published:
-  #   generators:
-  #     - name: fernapi/fern-python-sdk
-  #       version: 0.3.7
-  #       output:
-  #         location: pypi
-  #         url: pypi.buildwithfern.com
-  #         package-name: finto-fern-langfuse
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true
-  #     - name: fernapi/fern-typescript-node-sdk
-  #       version: 0.7.1
-  #       output:
-  #         location: npm
-  #         url: npm.buildwithfern.com
-  #         package-name: "@finto-fern/langfuse-node"
-  #       config:
-  #         namespaceExport: Langfuse
-  #         allowCustomFetcher: true

generated/.gitignore

@@ -1 +0,0 @@
-python/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.80.0",
+  "version": "3.97.2",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -40,7 +40,7 @@
     "husky": "^9.0.11",
     "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {
@@ -91,7 +91,8 @@
       "nanoid": "^3.3.8",
       "katex": "^0.16.21",
       "tar-fs": "^2.1.2",
-      "rollup@^4.0.0": "^4.22.4"
+      "rollup@^4.0.0": "^4.22.4",
+      "@types/node-fetch": "^2.6.13"
     },
     "patchedDependencies": {
       "next-auth@4.24.11": "patches/next-auth@4.24.11.patch"

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items ON CLUSTER default (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_7d_amt_mv ON CLUSTER default;
+DROP VIEW IF EXISTS traces_all_amt_mv ON CLUSTER default;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_7d_amt ON CLUSTER default;
+DROP TABLE IF EXISTS traces_all_amt ON CLUSTER default;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null ON CLUSTER default;

packages/shared/clickhouse/migrations/clustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt ON CLUSTER default
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv ON CLUSTER default TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv ON CLUSTER default TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt ON CLUSTER default
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations -- DO NOT USE
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv ON CLUSTER default TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations -- DO NOT USE
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.down.sql

@@ -0,0 +1 @@
+DROP TABLE dataset_run_items;

packages/shared/clickhouse/migrations/unclustered/0022_dataset_run_items.up.sql

@@ -0,0 +1,36 @@
+CREATE TABLE dataset_run_items (
+    -- primary identifiers
+    `id` String,
+    `project_id` String,
+    `dataset_run_id` String,
+    `dataset_item_id` String,
+    `dataset_id` String,
+    `trace_id` String,
+    `observation_id` Nullable(String),
+
+    -- error field
+    `error` Nullable(String),
+
+     -- timestamps
+    `created_at` DateTime64(3) DEFAULT now(),
+    `updated_at` DateTime64(3) DEFAULT now(),
+
+    -- denormalized immutable dataset run fields
+    `dataset_run_name` String,
+    `dataset_run_description` Nullable(String),
+    `dataset_run_metadata` Map(LowCardinality(String), String),
+    `dataset_run_created_at` DateTime64(3),
+
+    -- denormalized dataset item fields (mutable, but snapshots are relevant)
+    `dataset_item_input` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_expected_output` Nullable(String) CODEC(ZSTD(3)), -- json
+    `dataset_item_metadata` Map(LowCardinality(String), String),
+
+    -- clickhouse engine fields
+    `event_ts` DateTime64(3),
+    `is_deleted` UInt8,
+
+    -- For dataset item lookups
+    INDEX idx_dataset_item dataset_item_id TYPE bloom_filter(0.001) GRANULARITY 1,
+) ENGINE = ReplacingMergeTree(event_ts, is_deleted)
+ORDER BY (project_id, dataset_id, dataset_run_id, id);

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.down.sql

@@ -0,0 +1,12 @@
+-- Drop materialized views first
+DROP VIEW IF EXISTS traces_30d_amt_mv;
+DROP VIEW IF EXISTS traces_7d_amt_mv;
+DROP VIEW IF EXISTS traces_all_amt_mv;
+
+-- Drop AMT tables
+DROP TABLE IF EXISTS traces_30d_amt;
+DROP TABLE IF EXISTS traces_7d_amt;
+DROP TABLE IF EXISTS traces_all_amt;
+
+-- Drop the Null table
+DROP TABLE IF EXISTS traces_null;

packages/shared/clickhouse/migrations/unclustered/0023_traces_aggregating_merge_trees.up.sql

@@ -0,0 +1,300 @@
+-- Create a Null table that serves as a trigger for all materialized views.
+-- We use a Null engine here to avoid storing intermediate results and save on storage.
+CREATE TABLE traces_null
+(
+    -- Identifiers
+    `project_id`      String,
+    `id`              String,
+    `start_time`      DateTime64(3),
+    `end_time`        Nullable(DateTime64(3)),
+    `name`            Nullable(String),
+
+    -- Metadata properties
+    `metadata`        Map(LowCardinality(String), String),
+    `user_id`         Nullable(String),
+    `session_id`      Nullable(String),
+    `environment`     String,
+    `tags`            Array(String),
+    `version`         Nullable(String),
+    `release`         Nullable(String),
+
+    -- UI properties - We make them nullable to prevent absent values being interpreted as overwrites.
+    `bookmarked`      Nullable(Bool),
+    `public`          Nullable(Bool),
+
+    -- Aggregations
+    `observation_ids` Array(String),
+    `score_ids`       Array(String),
+    `cost_details`    Map(String, Decimal64(12)),
+    `usage_details`   Map(String, UInt64),
+    -- TODO: Do we want to aggregate/collect `levels` seen within the trace?
+
+    -- Input/Output
+    `input`           String,
+    `output`          String,
+
+    `created_at`      DateTime64(3),
+    `updated_at`      DateTime64(3),
+    `event_ts`        DateTime64(3)
+) Engine = Null();
+
+-- Create the all AMT
+CREATE TABLE traces_all_amt
+(    
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_trace_id id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+      ORDER BY (project_id, id);
+
+-- Create materialized view for all_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_all_amt_mv TO traces_all_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 7-day TTL AMT
+CREATE TABLE traces_7d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 7 DAY;
+
+-- Create materialized view for 7d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_7d_amt_mv TO traces_7d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;
+
+-- Create the 30-day TTL AMT
+CREATE TABLE traces_30d_amt
+(
+    -- Identifiers
+    `project_id`         String,
+    `id`                 String,
+    `timestamp`          SimpleAggregateFunction(min, DateTime64(3)),  -- Backward compatibility: redundant with start_time
+    `start_time`         SimpleAggregateFunction(min, DateTime64(3)),
+    `end_time`           SimpleAggregateFunction(max, Nullable(DateTime64(3))),
+    `name`               SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- Metadata properties
+    `metadata`           SimpleAggregateFunction(maxMap, Map(String, String)),
+    `user_id`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `session_id`         SimpleAggregateFunction(anyLast, Nullable(String)),
+    `environment`        SimpleAggregateFunction(anyLast, String),
+    `tags`               SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `version`            SimpleAggregateFunction(anyLast, Nullable(String)),
+    `release`            SimpleAggregateFunction(anyLast, Nullable(String)),
+
+    -- UI properties
+    `bookmarked`         AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+    `public`             AggregateFunction(argMax, Nullable(Bool), DateTime64(3)),
+
+    -- Aggregations
+    `observation_ids`    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `score_ids`          SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    `cost_details`       SimpleAggregateFunction(sumMap, Map(String, Decimal(38, 12))),
+    `usage_details`      SimpleAggregateFunction(sumMap, Map(String, UInt64)),
+
+    -- Input/Output -> prefer correctness via argMax
+    `input`       AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+    `output`      AggregateFunction(argMax, String, DateTime64(3)) CODEC (ZSTD(3)),
+
+    `created_at`         SimpleAggregateFunction(min, DateTime64(3)),
+    `updated_at`         SimpleAggregateFunction(max, DateTime64(3)),
+
+    -- Indexes
+    INDEX idx_user_id user_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_session_id session_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_name name TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_version version TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_release release TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_tags tags TYPE bloom_filter(0.001) GRANULARITY 1
+) Engine = AggregatingMergeTree()
+    ORDER BY (project_id, id)
+    TTL toDate(start_time) + INTERVAL 30 DAY;
+
+-- Create materialized view for 30d_amt
+CREATE MATERIALIZED VIEW IF NOT EXISTS traces_30d_amt_mv TO traces_30d_amt AS
+SELECT
+    -- Identifiers
+    tn.project_id                                                                              as project_id,
+    tn.id                                                                                      as id,
+    min(tn.start_time)                                                                         as timestamp,  -- Backward compatibility: redundant with start_time
+    min(tn.start_time)                                                                         as start_time,
+    max(coalesce(tn.end_time, tn.start_time))                                                  as end_time,
+    anyLast(tn.name)                                                                           as name,
+
+    -- Metadata properties
+    maxMap(tn.metadata)                                                                        as metadata,
+    anyLast(tn.user_id)                                                                        as user_id,
+    anyLast(tn.session_id)                                                                     as session_id,
+    anyLast(tn.environment)                                                                    as environment,
+    groupUniqArrayArray(tn.tags)                                                               as tags,
+    anyLast(tn.version)                                                                        as version,
+    anyLast(tn.release)                                                                        as release,
+
+    -- UI properties
+    argMaxState(tn.bookmarked, if(tn.bookmarked is not null, tn.event_ts, toDateTime64(0, 3))) as bookmarked,
+    argMaxState(tn.public, if(tn.public is not null, tn.event_ts, toDateTime64(0, 3)))         as public,
+
+    -- Aggregations
+    groupUniqArrayArray(tn.observation_ids)                                                    as observation_ids,
+    groupUniqArrayArray(tn.score_ids)                                                          as score_ids,
+    sumMap(tn.cost_details)                                                                    as cost_details,
+    sumMap(tn.usage_details)                                                                   as usage_details,
+
+    -- Input/Output
+    argMaxState(tn.input, if(tn.input <> '', tn.event_ts, toDateTime64(0, 3)))                 as input,
+    argMaxState(tn.output, if(tn.output <> '', tn.event_ts, toDateTime64(0, 3)))               as output,
+
+    min(tn.created_at)                                                                         as created_at,
+    max(tn.updated_at)                                                                         as updated_at
+FROM traces_null tn
+GROUP BY project_id, id;

packages/shared/clickhouse/scripts/up.sh

@@ -5,8 +5,30 @@
 
 # Check if CLICKHOUSE_URL is configured
 if [ -z "${CLICKHOUSE_URL}" ]; then
-  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
-  exit 0
+  echo "Error: CLICKHOUSE_URL is not configured."
+  echo "Please set CLICKHOUSE_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_MIGRATION_URL is configured
+if [ -z "${CLICKHOUSE_MIGRATION_URL}" ]; then
+  echo "Error: CLICKHOUSE_MIGRATION_URL is not configured."
+  echo "Please set CLICKHOUSE_MIGRATION_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_USER is set
+if [ -z "${CLICKHOUSE_USER}" ]; then
+  echo "Error: CLICKHOUSE_USER is not set."
+  echo "Please set CLICKHOUSE_USER in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_PASSWORD is set
+if [ -z "${CLICKHOUSE_PASSWORD}" ]; then
+  echo "Error: CLICKHOUSE_PASSWORD is not set."
+  echo "Please set CLICKHOUSE_PASSWORD in your environment variables."
+  exit 1
 fi
 
 # Check if golang-migrate is installed

packages/shared/package.json

@@ -61,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -73,13 +73,14 @@
     "@prisma/client": "^6.10.1",
     "@react-email/components": "^0.1.0",
     "@react-email/render": "^1.1.2",
+    "@slack/oauth": "^3.0.3",
+    "@slack/web-api": "^7.9.3",
     "@types/bcryptjs": "^2.4.6",
-    "axios": "^1.8.2",
     "bcryptjs": "^2.4.3",
     "bullmq": "^5.34.10",
     "dd-trace": "^5.36.0",
     "decimal.js": "^10.4.3",
-    "exponential-backoff": "^3.1.1",
+    "exponential-backoff": "^3.1.2",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "jsonpath-plus": "10.3.0",
@@ -118,17 +119,10 @@
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
-    "tsx": "^4.19.1",
-    "typescript": "^5.4.5",
-    "vitest": "^2.1.2"
+    "typescript": "^5.4.5"
   },
   "peerDependencies": {
     "@types/react": "~18.2.79",
     "react": "~18.2.0"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

packages/shared/prisma/generated/types.ts

@@ -55,6 +55,7 @@ export type AnnotationQueueStatus =
 export const AnnotationQueueObjectType = {
   TRACE: "TRACE",
   OBSERVATION: "OBSERVATION",
+  SESSION: "SESSION",
 } as const;
 export type AnnotationQueueObjectType =
   (typeof AnnotationQueueObjectType)[keyof typeof AnnotationQueueObjectType];
@@ -138,6 +139,7 @@ export type DashboardWidgetChartType =
   (typeof DashboardWidgetChartType)[keyof typeof DashboardWidgetChartType];
 export const ActionType = {
   WEBHOOK: "WEBHOOK",
+  SLACK: "SLACK",
 } as const;
 export type ActionType = (typeof ActionType)[keyof typeof ActionType];
 export const ActionExecutionStatus = {
@@ -148,6 +150,11 @@ export const ActionExecutionStatus = {
 } as const;
 export type ActionExecutionStatus =
   (typeof ActionExecutionStatus)[keyof typeof ActionExecutionStatus];
+export const SurveyName = {
+  ORG_ONBOARDING: "org_onboarding",
+  USER_ONBOARDING: "user_onboarding",
+} as const;
+export type SurveyName = (typeof SurveyName)[keyof typeof SurveyName];
 export type Account = {
   id: string;
   user_id: string;
@@ -183,6 +190,14 @@ export type AnnotationQueue = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type AnnotationQueueAssignment = {
+  id: string;
+  project_id: string;
+  user_id: string;
+  queue_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type AnnotationQueueItem = {
   id: string;
   queue_id: string;
@@ -359,6 +374,8 @@ export type Dataset = {
   name: string;
   description: string | null;
   metadata: unknown | null;
+  remote_experiment_url: string | null;
+  remote_experiment_payload: unknown | null;
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
@@ -624,6 +641,15 @@ export type OrganizationMembership = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
 };
+export type PendingDeletion = {
+  id: string;
+  project_id: string;
+  object: string;
+  object_id: string;
+  is_deleted: Generated<boolean>;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type PosthogIntegration = {
   project_id: string;
   encrypted_posthog_api_key: string;
@@ -637,6 +663,7 @@ export type Price = {
   created_at: Generated<Timestamp>;
   updated_at: Generated<Timestamp>;
   model_id: string;
+  project_id: string | null;
   usage_type: string;
   price: string;
 };
@@ -710,6 +737,16 @@ export type Session = {
   user_id: string;
   expires: Timestamp;
 };
+export type SlackIntegration = {
+  id: string;
+  project_id: string;
+  team_id: string;
+  team_name: string;
+  bot_token: string;
+  bot_user_id: string;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+};
 export type SsoConfig = {
   domain: string;
   created_at: Generated<Timestamp>;
@@ -717,6 +754,15 @@ export type SsoConfig = {
   auth_provider: string;
   auth_config: unknown | null;
 };
+export type Survey = {
+  id: string;
+  created_at: Generated<Timestamp>;
+  survey_name: SurveyName;
+  response: unknown;
+  user_id: string | null;
+  user_email: string | null;
+  org_id: string | null;
+};
 export type TableViewPreset = {
   id: string;
   created_at: Generated<Timestamp>;
@@ -780,6 +826,7 @@ export type VerificationToken = {
 export type DB = {
   Account: Account;
   actions: Action;
+  annotation_queue_assignments: AnnotationQueueAssignment;
   annotation_queue_items: AnnotationQueueItem;
   annotation_queues: AnnotationQueue;
   api_keys: ApiKey;
@@ -812,6 +859,7 @@ export type DB = {
   observations: LegacyPrismaObservation;
   organization_memberships: OrganizationMembership;
   organizations: Organization;
+  pending_deletions: PendingDeletion;
   posthog_integrations: PosthogIntegration;
   prices: Price;
   project_memberships: ProjectMembership;
@@ -822,7 +870,9 @@ export type DB = {
   score_configs: ScoreConfig;
   scores: LegacyPrismaScore;
   Session: Session;
+  slack_integrations: SlackIntegration;
   sso_configs: SsoConfig;
+  surveys: Survey;
   table_view_presets: TableViewPreset;
   trace_media: TraceMedia;
   trace_sessions: TraceSession;

packages/shared/prisma/migrations/20250711105322_prices_add_project_id/migration.sql

@@ -0,0 +1,13 @@
+-- AlterTable
+ALTER TABLE "prices"
+    ADD COLUMN "project_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "prices"
+    ADD CONSTRAINT "prices_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- BackfillData
+UPDATE "prices"
+SET "project_id" = (SELECT "models"."project_id"
+                    FROM "models"
+                    WHERE "models"."id" = "prices"."model_id");

packages/shared/prisma/migrations/20250711134738_add_patch_llm_tool_schema_audit_logs_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('3445cac4-d9d5-4750-8b65-351135c1b85e', '20250711_1347_patch_llm_tool_schema_audit_logs', 'patchLLMToolAndLLLMSchemaAuditLogs', '{}');

packages/shared/prisma/migrations/20250714151410_add_trace_session_combined_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "trace_sessions_project_id_created_at_idx" ON "trace_sessions"("project_id", "created_at" DESC);

packages/shared/prisma/migrations/20250714151410_remove_trace_session_created_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_created_at_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_project_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_project_id_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_updated_at_idx";

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/migrations/20250730100100_add_slack_integration/migration.sql

@@ -0,0 +1,30 @@
+-- Migration: Add Slack Integration Support
+-- This migration adds support for Slack automation actions by:
+-- 1. Adding SLACK to the ActionType enum
+-- 2. Creating slack_integrations table for centralized token storage
+
+-- AlterEnum
+ALTER TYPE "ActionType" ADD VALUE 'SLACK';
+
+-- CreateTable
+CREATE TABLE "slack_integrations" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "team_name" TEXT NOT NULL,
+    "bot_token" TEXT NOT NULL,
+    "bot_user_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "slack_integrations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "slack_integrations_project_id_key" ON "slack_integrations"("project_id");
+
+-- CreateIndex
+CREATE INDEX "slack_integrations_team_id_idx" ON "slack_integrations"("team_id");
+
+-- AddForeignKey
+ALTER TABLE "slack_integrations" ADD CONSTRAINT "slack_integrations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE; 

packages/shared/prisma/migrations/20250731100100_add_dataset_run_items_pg_to_ch_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('8d47f91b-3e5c-4a26-9f85-c12d6e4b9a3d', '20250731_1001_migrate_dataset_run_items_pg_to_ch', 'migrateDatasetRunItemsFromPostgresToClickhouse', '{}');

packages/shared/prisma/migrations/20250731202005_add_trace_deletion_table/migration.sql

@@ -0,0 +1,21 @@
+-- CreateTable
+CREATE TABLE "pending_deletions" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "object" TEXT NOT NULL,
+    "object_id" TEXT NOT NULL,
+    "is_deleted" BOOLEAN NOT NULL DEFAULT false,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "pending_deletions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_project_id_object_is_deleted_idx" ON "pending_deletions"("project_id", "object", "is_deleted");
+
+-- CreateIndex
+CREATE INDEX "pending_deletions_object_id_object_idx" ON "pending_deletions"("object_id", "object");
+
+-- AddForeignKey
+ALTER TABLE "pending_deletions" ADD CONSTRAINT "pending_deletions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250806100613_add_annotation_queue_assignment_table/migration.sql

@@ -0,0 +1,23 @@
+-- CreateTable
+CREATE TABLE "annotation_queue_assignments" (
+    "id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "user_id" TEXT NOT NULL,
+    "queue_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "annotation_queue_assignments_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "annotation_queue_assignments_project_id_queue_id_key" ON "annotation_queue_assignments"("project_id", "queue_id", "user_id");
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "annotation_queue_assignments" ADD CONSTRAINT "annotation_queue_assignments_queue_id_fkey" FOREIGN KEY ("queue_id") REFERENCES "annotation_queues"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250808081624_add_surveys_table/migration.sql

@@ -0,0 +1,24 @@
+-- CreateEnum
+CREATE TYPE "SurveyName" AS ENUM ('org_onboarding', 'user_onboarding');
+
+-- CreateTable
+CREATE TABLE "surveys" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "survey_name" "SurveyName" NOT NULL,
+    "response" JSONB NOT NULL,
+    "user_id" TEXT,
+    "user_email" TEXT,
+    "org_id" TEXT,
+
+    CONSTRAINT "surveys_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "surveys" ADD CONSTRAINT "surveys_org_id_fkey" FOREIGN KEY ("org_id") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "surveys" ADD CONSTRAINT "surveys_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- RenameIndex
+ALTER INDEX "annotation_queue_assignments_project_id_queue_id_key" RENAME TO "annotation_queue_assignments_project_id_queue_id_user_id_key";

packages/shared/prisma/schema.prisma

@@ -65,29 +65,31 @@ model Session {
 }
 
 model User {
-  id                      String                   @id @default(cuid())
-  name                    String?
-  email                   String?                  @unique
-  emailVerified           DateTime?                @map("email_verified")
-  password                String?
-  image                   String?
-  admin                   Boolean                  @default(false)
-  accounts                Account[]
-  sessions                Session[]
-  organizationMemberships OrganizationMembership[]
-  projectMemberships      ProjectMembership[]
-  invitations             MembershipInvitation[]
-  createdAt               DateTime                 @default(now()) @map("created_at")
-  updatedAt               DateTime                 @default(now()) @updatedAt @map("updated_at")
-  featureFlags            String[]                 @default([]) @map("feature_flags")
-  annotatedLockedItem     AnnotationQueueItem[]    @relation("LockedByUser")
-  annotatedCompletedItem  AnnotationQueueItem[]    @relation("AnnotatorUser")
-  dashboardWidgetsCreated DashboardWidget[]        @relation("CreatedByUser")
-  dashboardWidgetsUpdated DashboardWidget[]        @relation("UpdatedByUser")
-  dashboardCreated        Dashboard[]              @relation("CreatedByUser")
-  dashboardUpdated        Dashboard[]              @relation("UpdatedByUser")
-  tableViewPresetCreated  TableViewPreset[]        @relation("CreatedByUser")
-  tableViewPresetUpdated  TableViewPreset[]        @relation("UpdatedByUser")
+  id                        String                      @id @default(cuid())
+  name                      String?
+  email                     String?                     @unique
+  emailVerified             DateTime?                   @map("email_verified")
+  password                  String?
+  image                     String?
+  admin                     Boolean                     @default(false)
+  accounts                  Account[]
+  sessions                  Session[]
+  organizationMemberships   OrganizationMembership[]
+  projectMemberships        ProjectMembership[]
+  invitations               MembershipInvitation[]
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  featureFlags              String[]                    @default([]) @map("feature_flags")
+  annotatedLockedItem       AnnotationQueueItem[]       @relation("LockedByUser")
+  annotatedCompletedItem    AnnotationQueueItem[]       @relation("AnnotatorUser")
+  dashboardWidgetsCreated   DashboardWidget[]           @relation("CreatedByUser")
+  dashboardWidgetsUpdated   DashboardWidget[]           @relation("UpdatedByUser")
+  dashboardCreated          Dashboard[]                 @relation("CreatedByUser")
+  dashboardUpdated          Dashboard[]                 @relation("UpdatedByUser")
+  tableViewPresetCreated    TableViewPreset[]           @relation("CreatedByUser")
+  tableViewPresetUpdated    TableViewPreset[]           @relation("UpdatedByUser")
+  annotationQueueAssignment AnnotationQueueAssignment[]
+  surveys                   Survey[]
 
   @@map("users")
 }
@@ -112,56 +114,61 @@ model Organization {
   projects                Project[]
   MembershipInvitation    MembershipInvitation[]
   ApiKey                  ApiKey[]
+  surveys                 Survey[]
 
   @@map("organizations")
 }
 
 model Project {
-  id                     String                    @id @default(cuid())
-  orgId                  String                    @map("org_id")
-  createdAt              DateTime                  @default(now()) @map("created_at")
-  updatedAt              DateTime                  @default(now()) @updatedAt @map("updated_at")
-  deletedAt              DateTime?                 @map("deleted_at")
-  name                   String
-  retentionDays          Int?                      @map("retention_days")
-  metadata               Json?
-  projectMembers         ProjectMembership[]
-  organization           Organization              @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
-  apiKeys                ApiKey[]
-  dataset                Dataset[]
-  invitations            MembershipInvitation[]
-  sessions               TraceSession[]
-  Prompt                 Prompt[]
-  Model                  Model[]
-  EvalTemplate           EvalTemplate[]
-  JobConfiguration       JobConfiguration[]
-  JobExecution           JobExecution[]
-  LlmApiKeys             LlmApiKeys[]
-  PosthogIntegration     PosthogIntegration[]
-  BlobStorageIntegration BlobStorageIntegration[]
-  scoreConfig            ScoreConfig[]
-  BatchExport            BatchExport[]
-  comment                Comment[]
-  annotationQueue        AnnotationQueue[]
-  annotationQueueItem    AnnotationQueueItem[]
-  TraceMedia             TraceMedia[]
-  Media                  Media[]
-  ObservationMedia       ObservationMedia[]
-  LegacyTrace            LegacyPrismaTrace[]
-  LegacyObservation      LegacyPrismaObservation[]
-  LegacyScore            LegacyPrismaScore[]
-  PromptDependency       PromptDependency[]
-  LlmSchema              LlmSchema[]
-  LlmTool                LlmTool[]
-  PromptProtectedLabels  PromptProtectedLabels[]
-  Dashboard              Dashboard[]
-  DashboardWidget        DashboardWidget[]
-  TableViewPreset        TableViewPreset[]
-  actions                Action[]
-  triggers               Trigger[]
-  automationExecutions   AutomationExecution[]
-  Automation             Automation[]
-  DefaultLlmModel        DefaultLlmModel[]
+  id                        String                      @id @default(cuid())
+  orgId                     String                      @map("org_id")
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  deletedAt                 DateTime?                   @map("deleted_at")
+  name                      String
+  retentionDays             Int?                        @map("retention_days")
+  metadata                  Json?
+  projectMembers            ProjectMembership[]
+  organization              Organization                @relation(fields: [orgId], references: [id], onUpdate: Cascade, onDelete: Cascade)
+  apiKeys                   ApiKey[]
+  dataset                   Dataset[]
+  invitations               MembershipInvitation[]
+  sessions                  TraceSession[]
+  Prompt                    Prompt[]
+  Model                     Model[]
+  EvalTemplate              EvalTemplate[]
+  JobConfiguration          JobConfiguration[]
+  JobExecution              JobExecution[]
+  LlmApiKeys                LlmApiKeys[]
+  PosthogIntegration        PosthogIntegration[]
+  BlobStorageIntegration    BlobStorageIntegration[]
+  scoreConfig               ScoreConfig[]
+  BatchExport               BatchExport[]
+  comment                   Comment[]
+  annotationQueue           AnnotationQueue[]
+  annotationQueueItem       AnnotationQueueItem[]
+  TraceMedia                TraceMedia[]
+  Media                     Media[]
+  ObservationMedia          ObservationMedia[]
+  LegacyTrace               LegacyPrismaTrace[]
+  LegacyObservation         LegacyPrismaObservation[]
+  LegacyScore               LegacyPrismaScore[]
+  PromptDependency          PromptDependency[]
+  LlmSchema                 LlmSchema[]
+  LlmTool                   LlmTool[]
+  PromptProtectedLabels     PromptProtectedLabels[]
+  Dashboard                 Dashboard[]
+  DashboardWidget           DashboardWidget[]
+  TableViewPreset           TableViewPreset[]
+  actions                   Action[]
+  triggers                  Trigger[]
+  automationExecutions      AutomationExecution[]
+  Automation                Automation[]
+  DefaultLlmModel           DefaultLlmModel[]
+  Price                     Price[]
+  SlackIntegration          SlackIntegration?
+  PendingDeletion           PendingDeletion[]
+  AnnotationQueueAssignment AnnotationQueueAssignment[]
 
   @@index([orgId])
   @@map("projects")
@@ -310,9 +317,7 @@ model TraceSession {
   environment String   @default("default")
 
   @@id([id, projectId])
-  @@index([projectId])
-  @@index([createdAt])
-  @@index([updatedAt])
+  @@index([projectId, createdAt(sort: Desc)])
   @@map("trace_sessions")
 }
 
@@ -490,15 +495,16 @@ enum ScoreDataType {
 }
 
 model AnnotationQueue {
-  id                  String                @id @default(cuid())
-  name                String
-  description         String?
-  scoreConfigIds      String[]              @default([]) @map("score_config_ids")
-  projectId           String                @map("project_id")
-  project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt           DateTime              @default(now()) @map("created_at")
-  updatedAt           DateTime              @default(now()) @updatedAt @map("updated_at")
-  annotationQueueItem AnnotationQueueItem[]
+  id                        String                      @id @default(cuid())
+  name                      String
+  description               String?
+  scoreConfigIds            String[]                    @default([]) @map("score_config_ids")
+  projectId                 String                      @map("project_id")
+  project                   Project                     @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt                 DateTime                    @default(now()) @map("created_at")
+  updatedAt                 DateTime                    @default(now()) @updatedAt @map("updated_at")
+  annotationQueueItem       AnnotationQueueItem[]
+  annotationQueueAssignment AnnotationQueueAssignment[]
 
   @@unique([projectId, name])
   @@index([id, projectId])
@@ -540,6 +546,22 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
+}
+
+model AnnotationQueueAssignment {
+  id        String          @id @default(cuid())
+  projectId String          @map("project_id")
+  project   Project         @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  userId    String          @map("user_id")
+  user      User            @relation(fields: [userId], references: [id], onDelete: Cascade)
+  queueId   String          @map("queue_id")
+  queue     AnnotationQueue @relation(fields: [queueId], references: [id], onDelete: Cascade)
+  createdAt DateTime        @default(now()) @map("created_at")
+  updatedAt DateTime        @default(now()) @updatedAt @map("updated_at")
+
+  @@unique([projectId, queueId, userId])
+  @@map("annotation_queue_assignments")
 }
 
 model CronJobs {
@@ -552,16 +574,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -755,6 +779,8 @@ model Price {
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
   modelId   String   @map("model_id") // Model is already linked to project (or default), so we don't need projectId here
   Model     Model    @relation(fields: [modelId], references: [id], onDelete: Cascade)
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
   usageType String   @map("usage_type")
   price     Decimal
 
@@ -1294,6 +1320,7 @@ model Automation {
 
 enum ActionType {
   WEBHOOK
+  SLACK
   // More action types can be added as needed
 }
 
@@ -1334,3 +1361,62 @@ model AutomationExecution {
   @@index([projectId])
   @@map("automation_executions")
 }
+
+// Slack Integration: Stores centralized Slack workspace connection for each project
+// One project can connect to one Slack workspace, supporting multiple channel automations
+model SlackIntegration {
+  id        String  @id @default(cuid())
+  projectId String  @unique @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // Installation details (encrypted using shared encryption utilities)
+  teamId    String @map("team_id") // Slack workspace ID
+  teamName  String @map("team_name") // Human-readable workspace name
+  botToken  String @map("bot_token") // Encrypted bot token for API calls
+  botUserId String @map("bot_user_id") // Bot user ID for workspace
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([teamId])
+  @@map("slack_integrations")
+}
+
+// Pending Deletions: Tracks objects (like traces) that are scheduled for batch deletion
+model PendingDeletion {
+  id        String  @id @default(cuid())
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  object    String  @map("object") // e.g., "trace", "observation", etc.
+  objectId  String  @map("object_id") // The ID of the object to be deleted
+  isDeleted Boolean @default(false) @map("is_deleted")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  @@index([projectId, object, isDeleted])
+  @@index([objectId, object])
+  @@map("pending_deletions")
+}
+
+model Survey {
+  id         String        @id @default(cuid())
+  createdAt  DateTime      @default(now()) @map("created_at")
+  surveyName SurveyName    @map("survey_name")
+  response   Json
+  userId     String?       @map("user_id")
+  userEmail  String?       @map("user_email")
+  orgId      String?       @map("org_id")
+  org        Organization? @relation(fields: [orgId], references: [id], onDelete: Cascade)
+  user       User?         @relation(fields: [userId], references: [id], onDelete: Cascade)
+
+  @@map("surveys")
+}
+
+enum SurveyName {
+  ORG_ONBOARDING  @map("org_onboarding")
+  USER_ONBOARDING @map("user_onboarding")
+
+  @@map("SurveyName")
+}

packages/shared/scripts/seeder/seed-postgres.ts

@@ -23,6 +23,7 @@ import {
   SEED_TEXT_PROMPTS,
 } from "./utils/postgres-seed-constants";
 import {
+  generateDatasetItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -517,6 +518,7 @@ export async function createDatasets(
             description: data.description,
             projectId,
             metadata: data.metadata,
+            id: `${datasetName}-${projectId.slice(-8)}`,
           },
         }));
 
@@ -532,13 +534,23 @@ export async function createDatasets(
         const datasetItem = await prisma.datasetItem.upsert({
           where: {
             id_projectId: {
-              id: `${dataset.id}-${index}`,
+              id: generateDatasetItemId(
+                datasetName,
+                index,
+                projectId,
+                SEED_DATASETS.indexOf(data) || 0,
+              ),
               projectId,
             },
           },
           create: {
             projectId,
-            id: `${dataset.id}-${index}`,
+            id: generateDatasetItemId(
+              datasetName,
+              index,
+              projectId,
+              SEED_DATASETS.indexOf(data) || 0,
+            ),
             datasetId: dataset.id,
             sourceTraceId: sourceTraceId ?? null,
             sourceObservationId: null,
@@ -554,14 +566,14 @@ export async function createDatasets(
       for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.upsert({
           where: {
-            datasetId_projectId_name: {
-              datasetId: dataset.id,
+            id_projectId: {
+              id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
               projectId,
-              name: `demo-dataset-run-${datasetRunNumber}`,
             },
           },
           create: {
             projectId,
+            id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -2,6 +2,8 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItemsCh,
 } from "../../../src/server";
 import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
 import {
@@ -42,6 +44,16 @@ export class ClickHouseQueryBuilder {
     return await createObservationsCh(observations);
   }
 
+  /**
+   * Creates INSERT query for dataset run items data using VALUES syntax.
+   * Use for: Small datasets, dataset run items that link to postgres data (e.g. dataset runs)
+   */
+  async executeDatasetRunItemsInsert(
+    datasetRunItems: DatasetRunItemRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createDatasetRunItemsCh(datasetRunItems);
+  }
+
   /**
    * Creates INSERT query for score data using VALUES syntax.
    * Use for: Small datasets, scores with custom values and metadata.

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -6,6 +6,8 @@ import {
   REALISTIC_MODELS,
 } from "./clickhouse-seed-constants";
 import {
+  generateDatasetItemId,
+  generateDatasetRunItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -23,6 +25,8 @@ import {
   ObservationRecordInsertType,
   ScoreRecordInsertType,
   TraceRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItem,
 } from "../../../src/server";
 
 /**
@@ -60,6 +64,49 @@ export class DataGenerator {
     return Math.floor(Math.random() * (max - min + 1)) + min;
   }
 
+  /**
+   * Creates dataset run items for dataset runs.
+   * Use for: Dataset experiment scenarios.
+   */
+  generateDatasetRunItem(
+    input: DatasetItemInput & { runCreatedAt: number },
+    projectId: string,
+  ): DatasetRunItemRecordInsertType {
+    const datasetRunItemId = generateDatasetRunItemId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    // TODO: there are too many dataset run items in the postgres database?
+    return createDatasetRunItem({
+      id: datasetRunItemId,
+      project_id: projectId,
+      trace_id: generateDatasetRunTraceId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_id: `${input.datasetName}-${projectId.slice(-8)}`,
+      dataset_run_id: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_name: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_created_at: input.runCreatedAt,
+      dataset_run_description:
+        (input.runNumber || 0) % 2 === 0 ? "Dataset run description" : "",
+      dataset_run_metadata: { key: "value" },
+      dataset_item_id: generateDatasetItemId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_item_input: input.item.input,
+      dataset_item_expected_output: input.item.output,
+    });
+  }
+
   /**
    * Creates traces from dataset items for experiment runs.
    * Use for: Dataset experiments scenarios.

packages/shared/scripts/seeder/utils/markdown.txt

@@ -11,29 +11,29 @@
 ## 🎯 Getting Started
 
 ### Prerequisites
-\`\`\`bash
-npm install @langfuse/core
-pip install langfuse
+\`\`\`bash  
+npm install @langfuse/core  
+pip install langfuse  
 \`\`\`
 
 ### Quick Setup
 1. **Initialize your project**
-   \`\`\`typescript
-   import { Langfuse } from 'langfuse'
-   
-   const langfuse = new Langfuse({
-     secretKey: process.env.LANGFUSE_SECRET_KEY,
-     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-     baseUrl: 'https://cloud.langfuse.com'
-   })
+   \`\`\`typescript  
+   import { Langfuse } from 'langfuse'  
+
+   const langfuse = new Langfuse({  
+     secretKey: process.env.LANGFUSE_SECRET_KEY,  
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,  
+     baseUrl: 'https://cloud.langfuse.com'  
+   })  
    \`\`\`
 
 2. **Create your first trace**
-   \`\`\`python
-   from langfuse import Langfuse
-   
-   langfuse = Langfuse()
-   trace = langfuse.trace(name="chat-application")
+   \`\`\`python  
+   from langfuse import Langfuse  
+
+   langfuse = Langfuse()  
+   trace = langfuse.trace(name="chat-application")  
    \`\`\`
 
 ---
@@ -72,75 +72,75 @@ graph TD
 > **Note:** Traces are the foundation of observability in LLM applications.
 
 #### Creating Traces
-\`\`\`typescript
-// Basic trace creation
-const trace = langfuse.trace({
-  name: "user-query-processing",
-  userId: "user-123",
-  sessionId: "session-456",
-  metadata: {
-    environment: "production",
-    version: "2.1.0"
-  }
-})
+\`\`\`typescript  
+// Basic trace creation  
+const trace = langfuse.trace({  
+  name: "user-query-processing",  
+  userId: "user-123",  
+  sessionId: "session-456",  
+  metadata: {  
+    environment: "production",  
+    version: "2.1.0"  
+  }  
+})  
 
-// Nested observations
-const span = trace.span({
-  name: "document-retrieval",
-  input: { query: "What is machine learning?" },
-  metadata: { vectorStore: "pinecone" }
-})
+// Nested observations  
+const span = trace.span({  
+  name: "document-retrieval",  
+  input: { query: "What is machine learning?" },  
+  metadata: { vectorStore: "pinecone" }  
+})  
 
-const generation = span.generation({
-  name: "answer-generation",
-  model: "gpt-4",
-  input: retrievedDocs,
-  output: generatedAnswer,
-  usage: {
-    promptTokens: 1250,
-    completionTokens: 420,
-    totalTokens: 1670
-  }
-})
+const generation = span.generation({  
+  name: "answer-generation",  
+  model: "gpt-4",  
+  input: retrievedDocs,  
+  output: generatedAnswer,  
+  usage: {  
+    promptTokens: 1250,  
+    completionTokens: 420,  
+    totalTokens: 1670  
+  }  
+})  
 \`\`\`
 
 ### Advanced Features
 
 #### 🔄 Async Processing
-\`\`\`python
-import asyncio
-from langfuse import Langfuse
+\`\`\`python  
+import asyncio  
+from langfuse import Langfuse  
 
-async def process_batch():
-    langfuse = Langfuse()
-    
-    tasks = []
-    for item in batch_items:
-        task = asyncio.create_task(
-            process_item_with_tracing(langfuse, item)
-        )
-        tasks.append(task)
-    
-    results = await asyncio.gather(*tasks)
-    return results
+async def process_batch():  
+    langfuse = Langfuse()  
+
+    tasks = []  
+    for item in batch_items:  
+        task = asyncio.create_task(  
+            process_item_with_tracing(langfuse, item)  
+        )  
+        tasks.append(task)  
+
+    results = await asyncio.gather(*tasks)  
+    return results  
 \`\`\`
 
 #### 🎯 Custom Scoring
-\`\`\`typescript
-// Automated scoring
-trace.score({
-  name: "relevance",
-  value: 0.95,
-  comment: "Highly relevant response"
-})
+\`\`\`typescript  
+// Automated scoring  
+trace.score({  
+  name: "relevance",  
+  value: 0.95,  
+  comment: "Highly relevant response"  
+})  
 
-// Human feedback scoring
-trace.score({
-  name: "user-satisfaction", 
-  value: 1,
-  source: "user-feedback",
-  comment: "User rated 5/5 stars"
-})
+// Human feedback scoring  
+trace.score({  
+  name: "user-satisfaction",  
+  value: 1,  
+  source: "user-feedback",  
+  comment: "User rated 5/5 stars"  
+})  
 \`\`\`
 
 ---
@@ -156,20 +156,20 @@ trace.score({
 - **User Satisfaction**: Quality metrics
 
 #### Dashboard Setup
-\`\`\`yaml
-# monitoring-config.yml
-dashboards:
-  - name: "LLM Performance"
-    panels:
-      - type: "time-series"
-        title: "Response Latency"
-        query: "avg(response_time) by (model)"
-      - type: "stat"
-        title: "Daily Token Usage" 
-        query: "sum(tokens_used)"
-      - type: "table"
-        title: "Top Errors"
-        query: "topk(10, count by (error_type))"
+\`\`\`yaml  
+# monitoring-config.yml  
+dashboards:  
+  - name: "LLM Performance"  
+    panels:  
+      - type: "time-series"  
+        title: "Response Latency"  
+        query: "avg(response_time) by (model)"  
+      - type: "stat"  
+        title: "Daily Token Usage"  
+        query: "sum(tokens_used)"  
+      - type: "table"  
+        title: "Top Errors"  
+        query: "topk(10, count by (error_type))"  
 \`\`\`
 
 ### 🔐 Security Considerations
@@ -177,40 +177,40 @@ dashboards:
 > ⚠️ **Important**: Never log sensitive user data in traces
 
 #### Data Sanitization
-\`\`\`python
-def sanitize_input(data):
-    """Remove PII from trace data"""
-    sanitized = data.copy()
-    
-    # Remove email addresses
-    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
-                      '[EMAIL_REDACTED]', sanitized)
-    
-    # Remove phone numbers
-    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
-                      '[PHONE_REDACTED]', sanitized)
-    
-    return sanitized
+\`\`\`python  
+def sanitize_input(data):  
+    """Remove PII from trace data"""  
+    sanitized = data.copy()  
+
+    # Remove email addresses  
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b',  
+                      '[EMAIL_REDACTED]', sanitized)  
+
+    # Remove phone numbers  
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b',  
+                      '[PHONE_REDACTED]', sanitized)  
+
+    return sanitized  
 \`\`\`
 
 ### 🚀 Performance Optimization
 
 #### Batch Processing
-\`\`\`typescript
-// Efficient batch uploads
-const batchSize = 100
-const traces = []
+\`\`\`typescript  
+// Efficient batch uploads  
+const batchSize = 100  
+const traces = []  
 
-for (let i = 0; i < data.length; i += batchSize) {
-  const batch = data.slice(i, i + batchSize)
-  const processedBatch = await Promise.all(
-    batch.map(item => processWithLangfuse(item))
-  )
-  traces.push(...processedBatch)
-}
+for (let i = 0; i < data.length; i += batchSize) {  
+  const batch = data.slice(i, i + batchSize)  
+  const processedBatch = await Promise.all(  
+    batch.map(item => processWithLangfuse(item))  
+  )  
+  traces.push(...processedBatch)  
+}  
 
-// Flush all traces at once
-await langfuse.flushAsync()
+// Flush all traces at once  
+await langfuse.flushAsync()  
 \`\`\`
 
 ---
@@ -219,34 +219,34 @@ await langfuse.flushAsync()
 
 ### Multi-Agent System Tracing
 \`\`\`python
-class MultiAgentTracer:
-    def __init__(self):
-        self.langfuse = Langfuse()
-    
-    async def orchestrate_agents(self, task):
-        # Main orchestration trace
-        main_trace = self.langfuse.trace(
-            name="multi-agent-orchestration",
-            input={"task": task}
-        )
-        
-        # Agent 1: Research
-        research_span = main_trace.span(name="research-agent")
-        research_result = await self.research_agent.process(task)
-        research_span.end(output=research_result)
-        
+class MultiAgentTracer:  
+    def __init__(self):  
+        self.langfuse = Langfuse()  
+
+    async def orchestrate_agents(self, task):  
+        # Main orchestration trace  
+        main_trace = self.langfuse.trace(  
+            name="multi-agent-orchestration",  
+            input={"task": task}  
+        )  
+
+        # Agent 1: Research  
+        research_span = main_trace.span(name="research-agent")  
+        research_result = await self.research_agent.process(task)  
+        research_span.end(output=research_result)  
+
         # Agent 2: Analysis  
-        analysis_span = main_trace.span(name="analysis-agent")
-        analysis_result = await self.analysis_agent.process(research_result)
-        analysis_span.end(output=analysis_result)
-        
-        # Agent 3: Synthesis
-        synthesis_span = main_trace.span(name="synthesis-agent")
-        final_result = await self.synthesis_agent.process(analysis_result)
-        synthesis_span.end(output=final_result)
-        
-        main_trace.end(output=final_result)
-        return final_result
+        analysis_span = main_trace.span(name="analysis-agent")  
+        analysis_result = await self.analysis_agent.process(research_result)  
+        analysis_span.end(output=analysis_result)  
+
+        # Agent 3: Synthesis  
+        synthesis_span = main_trace.span(name="synthesis-agent")  
+        final_result = await self.synthesis_agent.process(analysis_result)  
+        synthesis_span.end(output=final_result)  
+
+        main_trace.end(output=final_result)  
+        return final_result  
 \`\`\`
 
 ---
@@ -256,11 +256,11 @@ class MultiAgentTracer:
 With proper implementation of Langfuse tracing, you can:
 
 - ✅ **Monitor** your LLM applications in real-time
-- ✅ **Debug** issues with detailed trace information  
+- ✅ **Debug** issues with detailed trace information
 - ✅ **Optimize** performance and costs
 - ✅ **Scale** your applications with confidence
 
 ### Next Steps
 1. Review the [official documentation](https://langfuse.com/docs)
 2. Join our [Discord community](https://discord.gg/langfuse)
-3. Check out [example projects](https://github.com/langfuse/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -1,3 +1,21 @@
+export const generateDatasetRunItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-run-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateDatasetItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
 export const generateDatasetRunTraceId = (
   datasetName: string,
   itemIndex: number,

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -92,12 +92,26 @@ export class SeederOrchestrator {
         logger.info(
           `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
         );
+        // const now = Date.now();
 
         const traces: TraceRecordInsertType[] = [];
         const observations: ObservationRecordInsertType[] = [];
+        // const datasetRunItems: DatasetRunItemRecordInsertType[] = [];
 
         for (const seedDataset of SEED_DATASETS) {
           for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // // Generate dataset run item data
+            // const datasetRunItem = this.dataGenerator.generateDatasetRunItem(
+            //   {
+            //     datasetName: seedDataset.name,
+            //     itemIndex,
+            //     item: datasetItem,
+            //     runNumber,
+            //     runCreatedAt: now,
+            //   },
+            //   projectId,
+            // );
+
             // Generate trace data
             const trace = this.dataGenerator.generateDatasetTrace(
               {
@@ -123,12 +137,14 @@ export class SeederOrchestrator {
 
             traces.push(trace);
             observations.push(observation);
+            // datasetRunItems.push(datasetRunItem);
           }
         }
 
         try {
           await this.queryBuilder.executeTracesInsert(traces);
           await this.queryBuilder.executeObservationsInsert(observations);
+          // await this.queryBuilder.executeDatasetRunItemsInsert(datasetRunItems);
         } catch (error) {
           logger.error(`✗ Insert failed:`, error);
           throw error;

packages/shared/src/domain/automations.ts

@@ -30,31 +30,41 @@ export type AutomationDomain = {
 };
 
 export type ActionDomain = Omit<Action, "config"> & {
-  config: SafeWebhookActionConfig;
+  config: SafeActionConfig;
 };
 
 export type ActionDomainWithSecrets = Omit<Action, "config"> & {
-  config: WebhookActionConfigWithSecrets;
+  config: ActionConfigWithSecrets;
 };
 
-export const ActionTypeSchema = z.enum(["WEBHOOK"]);
+export const ActionTypeSchema = z.enum(["WEBHOOK", "SLACK"]);
 
 export const AvailableWebhookApiSchema = z.record(
   z.enum(["prompt"]),
   z.enum(["v1"]),
 );
 
+export const RequestHeaderSchema = z.object({
+  secret: z.boolean(),
+  value: z.string(),
+});
+
 export const WebhookActionConfigSchema = z.object({
   type: z.literal("WEBHOOK"),
   url: z.url(),
-  headers: z.record(z.string(), z.string()),
+  headers: z.record(z.string(), z.string()).optional(), // deprecated field, use requestHeaders instead
+  requestHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
+  displayHeaders: z.record(z.string(), RequestHeaderSchema).optional(), // might not exist on legacy webhooks
   apiVersion: AvailableWebhookApiSchema,
   secretKey: z.string(),
   displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
 });
 
 export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
   secretKey: true,
+  headers: true,
+  requestHeaders: true,
 });
 
 export type SafeWebhookActionConfig = z.infer<
@@ -64,20 +74,98 @@ export type SafeWebhookActionConfig = z.infer<
 export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
   secretKey: true,
   displaySecretKey: true,
+  headers: true, // don't use legacy field anymore
+  displayHeaders: true,
 });
 
+export const SlackActionConfigSchema = z.object({
+  type: z.literal("SLACK"),
+  channelId: z.string(),
+  channelName: z.string(),
+  messageTemplate: z.string().optional(),
+});
+
+export type SlackActionConfig = z.infer<typeof SlackActionConfigSchema>;
+
 export const ActionConfigSchema = z.discriminatedUnion("type", [
   WebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export const ActionCreateSchema = z.discriminatedUnion("type", [
   WebhookActionCreateSchema,
+  SlackActionConfigSchema,
+]);
+
+export const SafeActionConfigSchema = z.discriminatedUnion("type", [
+  SafeWebhookActionConfigSchema,
+  SlackActionConfigSchema,
 ]);
 
 export type ActionTypes = z.infer<typeof ActionTypeSchema>;
 export type ActionConfig = z.infer<typeof ActionConfigSchema>;
 export type ActionCreate = z.infer<typeof ActionCreateSchema>;
+export type SafeActionConfig = z.infer<typeof SafeActionConfigSchema>;
 
+export type WebhookActionCreate = z.infer<typeof WebhookActionCreateSchema>;
 export type WebhookActionConfigWithSecrets = z.infer<
   typeof WebhookActionConfigSchema
 >;
+
+export type ActionConfigWithSecrets = z.infer<typeof ActionConfigSchema>;
+
+// Type Guards for Runtime Validation
+// Using existing Zod schemas to provide both compile-time and runtime type safety
+
+/**
+ * Type guard to check if a config is a valid webhook configuration with secrets
+ */
+export function isWebhookActionConfig(
+  config: unknown,
+): config is WebhookActionConfigWithSecrets {
+  return WebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if a config is a valid Slack configuration
+ */
+export function isSlackActionConfig(
+  config: unknown,
+): config is SlackActionConfig {
+  return SlackActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Type guard to check if an entire action has valid webhook configuration
+ */
+export function isWebhookAction(action: {
+  type: string;
+  config: unknown;
+}): action is { type: "WEBHOOK"; config: WebhookActionConfigWithSecrets } {
+  return action.type === "WEBHOOK" && isWebhookActionConfig(action.config);
+}
+
+/**
+ * Type guard for safe webhook config (without secrets)
+ */
+export function isSafeWebhookActionConfig(
+  config: unknown,
+): config is SafeWebhookActionConfig {
+  return SafeWebhookActionConfigSchema.safeParse(config).success;
+}
+
+/**
+ * Converts webhook config with secrets to safe config by only including allowed fields
+ */
+export function convertToSafeWebhookConfig(
+  webhookConfig: WebhookActionConfigWithSecrets,
+): SafeWebhookActionConfig {
+  return {
+    type: webhookConfig.type,
+    url: webhookConfig.url,
+    displayHeaders: webhookConfig.displayHeaders,
+    apiVersion: webhookConfig.apiVersion,
+    displaySecretKey: webhookConfig.displaySecretKey,
+    lastFailingExecutionId: webhookConfig.lastFailingExecutionId,
+  };
+}

packages/shared/src/domain/dataset-run-items.ts

@@ -0,0 +1,28 @@
+import z from "zod/v4";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const DatasetRunItemSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  datasetRunId: z.string(),
+  datasetItemId: z.string(),
+  datasetId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().nullable(),
+  error: z.string().nullable(),
+  // timestamps
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  // dataset run fields
+  datasetRunName: z.string(),
+  datasetRunDescription: z.string().nullable(),
+  datasetRunMetadata: MetadataDomain,
+  datasetRunCreatedAt: z.date(),
+  // dataset item fields
+  datasetItemInput: jsonSchema,
+  datasetItemExpectedOutput: jsonSchema,
+  datasetItemMetadata: MetadataDomain,
+});
+
+export type DatasetRunItemDomain = z.infer<typeof DatasetRunItemSchema>;

packages/shared/src/domain/webhooks.ts

@@ -3,8 +3,8 @@ import { jsonSchema } from "../utils/zod";
 import { EventActionSchema } from "./automations";
 
 export const WebhookDefaultHeaders = {
-  "Content-Type": "application/json",
-  "User-Agent": "Langfuse/1.0",
+  "content-type": "application/json",
+  "user-agent": "Langfuse/1.0",
 };
 
 export const WebhookOutboundBaseSchema = z.object({

packages/shared/src/env.ts

@@ -15,7 +15,9 @@ const EnvSchema = z.object({
     .default(6379)
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
+  REDIS_USERNAME: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
+  REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
   REDIS_TLS_CA_PATH: z.string().optional(),
   REDIS_TLS_CERT_PATH: z.string().optional(),
@@ -31,8 +33,10 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
     )
     .optional(),
-  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
+  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(300), // 5 minutes
   CLICKHOUSE_URL: z.string().url(),
   CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
   CLICKHOUSE_DB: z.string().default("default"),
@@ -46,6 +50,14 @@ const EnvSchema = z.object({
     .nonnegative()
     .default(15_000),
   LANGFUSE_INGESTION_QUEUE_SHARD_COUNT: z.coerce.number().positive().default(1),
+  LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT: z.coerce
+    .number()
+    .positive()
+    .default(1),
+  LANGFUSE_TRACE_DELETE_DELAY_MS: z.coerce
+    .number()
+    .nonnegative()
+    .default(5_000),
   SALT: z.string().optional(), // used by components imported by web package
   LANGFUSE_LOG_LEVEL: z
     .enum(["trace", "debug", "info", "warn", "error", "fatal"])
@@ -84,6 +96,9 @@ const EnvSchema = z.object({
   LANGFUSE_S3_MEDIA_UPLOAD_SSE: z.enum(["AES256", "aws:kms"]).optional(),
   LANGFUSE_S3_MEDIA_UPLOAD_SSE_KMS_KEY_ID: z.string().optional(),
   LANGFUSE_USE_AZURE_BLOB: z.enum(["true", "false"]).default("false"),
+  LANGFUSE_AZURE_SKIP_CONTAINER_CHECK: z
+    .enum(["true", "false"])
+    .default("true"),
   LANGFUSE_USE_GOOGLE_CLOUD_STORAGE: z.enum(["true", "false"]).default("false"),
   LANGFUSE_GOOGLE_CLOUD_STORAGE_CREDENTIALS: z.string().optional(),
   STRIPE_SECRET_KEY: z.string().optional(),
@@ -107,7 +122,13 @@ const EnvSchema = z.object({
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
   LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
   LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
-
+  // Dataset Run Items Migration Environment Variables
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH: z
+    .enum(["true", "false"])
+    .default("false"),
   LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES: z
     .enum(["true", "false"])
     .default("false"),
@@ -131,6 +152,60 @@ const EnvSchema = z.object({
   LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
     .enum(["true", "false"])
     .default("false"),
+  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_INSERT_INTO_AGGREGATING_MERGE_TREES: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
+  SLACK_CLIENT_ID: z.string().optional(),
+  SLACK_CLIENT_SECRET: z.string().optional(),
+  SLACK_STATE_SECRET: z.string().optional(),
+  HTTPS_PROXY: z.string().optional(),
+
+  LANGFUSE_SERVER_SIDE_IO_CHAR_LIMIT: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(1_000),
+
+  LANGFUSE_CLICKHOUSE_DATA_EXPORT_REQUEST_TIMEOUT_MS: z.coerce
+    .number()
+    .int()
+    .positive()
+    .default(600_000), // 10 minutes
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/features/annotation/types.ts

@@ -1,5 +1,6 @@
 import z from "zod/v4";
 import { type ScoreDataType } from "../../db";
+import { StringNoHTML, StringNoHTMLNonEmpty } from "../../utils/zod";
 
 const NUMERIC: ScoreDataType = "NUMERIC";
 const CATEGORICAL: ScoreDataType = "CATEGORICAL";
@@ -47,12 +48,12 @@ export type ScoreTargetSession = z.infer<typeof ScoreTargetSession>;
 export type ScoreTarget = z.infer<typeof ScoreTarget>;
 
 const CreateAnnotationScoreBase = z.object({
-  name: z.string(),
+  name: StringNoHTMLNonEmpty,
   projectId: z.string(),
   environment: z.string().default("default"),
   scoreTarget: ScoreTarget,
   configId: z.string().optional(),
-  comment: z.string().nullish(),
+  comment: StringNoHTML.nullish(),
   queueId: z.string().nullish(),
 });
 
@@ -83,11 +84,18 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
 // annotation queues
 
 export const CreateQueueData = z.object({
-  name: z.string().min(1).max(35),
-  description: z.string().max(1000).optional(),
+  name: StringNoHTMLNonEmpty.max(35),
+  description: StringNoHTML.max(1000).optional(),
   scoreConfigIds: z.array(z.string()).min(1, {
     message: "At least 1 score config must be selected",
   }),
 });
 
+export const CreateQueueWithAssignmentsData = CreateQueueData.extend({
+  newAssignmentUserIds: z.array(z.string()),
+});
+
 export type CreateQueue = z.infer<typeof CreateQueueData>;
+export type CreateQueueWithAssignments = z.infer<
+  typeof CreateQueueWithAssignmentsData
+>;

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,8 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
+  "observation-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/features/evals/utilities.ts

@@ -27,16 +27,42 @@ export const parseUnknownToString = (value: unknown): string => {
   return String(value);
 };
 
+/**
+ * Recursively parses JSON strings that may have been encoded multiple times.
+ * This handles cases where data has been JSON.stringify'd multiple times.
+ *
+ * @param value - The potentially multi-encoded JSON string
+ * @returns The final parsed object or the original value if parsing fails
+ */
+function parseMultiEncodedJson(value: unknown): unknown {
+  if (typeof value !== "string") {
+    return value;
+  }
+
+  try {
+    const parsed = JSON.parse(value);
+
+    // If result is still a string, it might be double-encoded - recurse
+    if (typeof parsed === "string") {
+      return parseMultiEncodedJson(parsed);
+    }
+
+    return parsed;
+  } catch {
+    // If parsing fails, return original value
+    return value;
+  }
+}
+
 function parseJsonDefault(selectedColumn: unknown, jsonSelector: string) {
+  // selectedColumn should already be preprocessed by preprocessObjectWithJsonFields
+  // so we can directly use it with JSONPath
   const result = JSONPath({
     path: jsonSelector,
-    json:
-      typeof selectedColumn === "string"
-        ? JSON.parse(selectedColumn)
-        : selectedColumn,
+    json: selectedColumn as any, // JSONPath accepts unknown but types are strict
   });
 
-  return result.length > 0 ? result[0] : undefined;
+  return Array.isArray(result) && result.length > 0 ? result[0] : undefined;
 }
 
 export function extractValueFromObject(
@@ -44,7 +70,13 @@ export function extractValueFromObject(
   mapping: z.infer<typeof variableMapping>,
   parseJson?: (selectedColumn: unknown, jsonSelector: string) => unknown, // eslint-disable-line no-unused-vars
 ): { value: string; error: Error | null } {
-  const selectedColumn = obj[mapping.selectedColumnId];
+  let selectedColumn = obj[mapping.selectedColumnId];
+
+  // Simple preprocessing: attempt to parse to valid JSON object
+  if (typeof selectedColumn === "string") {
+    selectedColumn = parseMultiEncodedJson(selectedColumn);
+  }
+
   const jsonParser = parseJson || parseJsonDefault;
 
   let jsonSelectedColumn;

packages/shared/src/features/prompts/validation.ts

@@ -1,12 +1,12 @@
-import { z } from "zod/v4";
+import { StringNoHTMLNonEmpty } from "../../utils/zod";
 
 /**
  * Prompt name validation schema for API, tRPC and client
  */
-export const PromptNameSchema = z
-  .string()
-  .min(1, "Enter a name")
-  .regex(/^[^|]*$/, "Prompt name cannot contain '|' character")
+export const PromptNameSchema = StringNoHTMLNonEmpty.regex(
+  /^[^|]*$/,
+  "Prompt name cannot contain '|' character",
+)
   .regex(/^[^/]/, "Name cannot start with a slash")
   .regex(/^(?!.*\/\/)/, "Name cannot contain consecutive slashes")
   .regex(/^.*[^/]$/, "Name cannot end with a slash")

packages/shared/src/index.ts

@@ -18,6 +18,7 @@ export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
 export * from "./domain/webhooks";
+export * from "./domain/dataset-run-items";
 
 // llm api
 export * from "./server/llm/types";

packages/shared/src/interfaces/cloudConfigSchema.ts

@@ -18,6 +18,21 @@ export const CloudConfigSchema = z.object({
 
   // custom rate limits for an organization
   rateLimitOverrides: CloudConfigRateLimit.optional(),
+
+  // billing alert configuration
+  usageAlerts: z
+    .object({
+      enabled: z.boolean().default(true),
+      type: z.enum(["STRIPE"]).default("STRIPE"),
+      threshold: z.number().int().positive(),
+      alertId: z.string(), // Alert ID for tracking
+      meterId: z.string(), // Meter ID for usage tracking
+      notifications: z.object({
+        email: z.boolean().default(true),
+        recipients: z.array(z.string().email()).default([]),
+      }),
+    })
+    .optional(),
 });
 
 export type CloudConfigSchema = z.infer<typeof CloudConfigSchema>;

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -1,5 +1,9 @@
 import { z } from "zod/v4";
 
+// Sentinel value for Bedrock default credential provider chain
+export const BEDROCK_USE_DEFAULT_CREDENTIALS =
+  "__BEDROCK_DEFAULT_CREDENTIALS__";
+
 export const BedrockConfigSchema = z.object({ region: z.string() });
 export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
 

packages/shared/src/server/auth/types.ts

@@ -1,7 +1,7 @@
 import z from "zod/v4";
 import { Plan, plans } from "../../features/entitlements/plans";
 import { CloudConfigRateLimit } from "../../interfaces/rate-limits";
-import { ApiKeyScope } from "../../";
+import { ApiKeyScope, MakeOptional } from "../../";
 
 const ApiKeyBaseSchema = z.object({
   id: z.string(),
@@ -48,12 +48,25 @@ export type AuthHeaderValidVerificationResult = {
   scope: ApiAccessScope;
 };
 
-export type ApiAccessScope = {
+export type AuthHeaderValidVerificationResultIngestion = {
+  validKey: true;
+  scope: ApiAccessScopeIngestion;
+};
+
+type BaseApiAccessScope = {
   projectId: string | null;
   accessLevel: "organization" | "project" | "scores";
+};
+
+type ApiAccessScopeMetadata = {
   orgId: string;
   plan: Plan;
   rateLimitOverrides: z.infer<typeof CloudConfigRateLimit>;
   apiKeyId: string;
   publicKey: string;
 };
+
+export type ApiAccessScopeIngestion = BaseApiAccessScope &
+  MakeOptional<ApiAccessScopeMetadata>;
+
+export type ApiAccessScope = BaseApiAccessScope & ApiAccessScopeMetadata;

packages/shared/src/server/automations/webhooks.ts

@@ -1,247 +0,0 @@
-import { Job, Processor } from "bullmq";
-import { backOff } from "exponential-backoff";
-import {
-  ActionExecutionStatus,
-  JobConfigState,
-} from "../../../prisma/generated/types";
-import {
-  PromptWebhookOutboundSchema,
-  WebhookDefaultHeaders,
-} from "../../domain";
-import { prisma } from "../../db";
-import { TQueueJobTypes, QueueName, WebhookInput } from "../queues";
-import {
-  getActionByIdWithSecrets,
-  getAutomationById,
-  getConsecutiveAutomationFailures,
-} from "../repositories";
-import { logger } from "..";
-import { createSignatureHeader } from "../../encryption/signature";
-import { decrypt } from "../../encryption";
-import { InternalServerError, LangfuseNotFoundError } from "../../errors";
-
-export const webhookProcessor: Processor = async (
-  job: Job<TQueueJobTypes[QueueName.WebhookQueue]>,
-) => {
-  try {
-    return await executeWebhook(job.data.payload);
-  } catch (error) {
-    logger.error("Error executing WebhookJob", error);
-    throw error;
-  }
-};
-
-// TODO: Webhook outgoing API versioning
-export const executeWebhook = async (input: WebhookInput) => {
-  const executionStart = new Date();
-
-  const { projectId, automationId, executionId } = input;
-  let httpStatus: number | undefined;
-  let responseBody: string | undefined;
-
-  try {
-    logger.debug(`Executing webhook for automation ${automationId}`);
-
-    const automation = await getAutomationById({
-      projectId,
-      automationId,
-    });
-
-    if (!automation) {
-      throw new LangfuseNotFoundError(`Automation ${automationId} not found`);
-    }
-
-    const actionConfig = await getActionByIdWithSecrets({
-      projectId,
-      actionId: automation.action.id,
-    });
-
-    if (!actionConfig) {
-      throw new Error("Action config not found");
-    }
-
-    if (actionConfig.config.type !== "WEBHOOK") {
-      throw new InternalServerError("Action config is not a webhook");
-    }
-
-    // TypeScript now knows actionConfig.config is WebhookActionConfig
-    const webhookConfig = actionConfig.config;
-
-    const validatedPayload = PromptWebhookOutboundSchema.safeParse({
-      id: input.executionId,
-      timestamp: new Date(),
-      type: input.payload.type,
-      apiVersion: "v1",
-      action: input.payload.action,
-      prompt: input.payload.prompt,
-    });
-
-    if (!validatedPayload.success) {
-      throw new InternalServerError(
-        `Invalid webhook payload: ${validatedPayload.error.message}`,
-      );
-    }
-
-    // Prepare webhook payload with prompt always last
-    const { prompt, ...otherFields } = validatedPayload.data;
-    const webhookPayload = JSON.stringify({
-      ...otherFields,
-      prompt,
-    });
-
-    // Prepare headers with signature if secret exists
-    const requestHeaders: Record<string, string> = {
-      ...WebhookDefaultHeaders,
-      ...webhookConfig.headers,
-    };
-
-    if (!webhookConfig.secretKey) {
-      logger.warn(
-        `Webhook config for action ${automation.action.id} has no secret key, failing webhook execution`,
-      );
-      throw new InternalServerError(
-        "Webhook config has no secret key, failing webhook execution",
-      );
-    }
-
-    if (webhookConfig.secretKey) {
-      try {
-        const decryptedSecret = decrypt(webhookConfig.secretKey);
-
-        const signature = createSignatureHeader(
-          webhookPayload,
-          decryptedSecret,
-        );
-        requestHeaders["x-langfuse-signature"] = signature;
-      } catch (error) {
-        logger.error(
-          "Failed to decrypt webhook secret or generate signature",
-          error,
-        );
-        throw new InternalServerError("Failed to generate webhook signature");
-      }
-    }
-
-    await backOff(
-      async () => {
-        logger.debug(
-          `Sending webhook to ${webhookConfig.url} with payload ${JSON.stringify(
-            webhookPayload,
-          )} and headers ${JSON.stringify(requestHeaders)}`,
-        );
-        const res = await fetch(webhookConfig.url, {
-          method: "POST",
-          body: webhookPayload,
-          headers: requestHeaders,
-        });
-
-        httpStatus = res.status;
-        responseBody = await res.text();
-
-        if (res.status !== 200) {
-          logger.warn(
-            `Webhook does not return 200: failed with status ${res.status} for url ${webhookConfig.url} and project ${projectId}. Body: ${responseBody}`,
-          );
-          throw new Error(
-            `Webhook does not return 200: failed with status ${res.status} for url ${webhookConfig.url} and project ${projectId}`,
-          );
-        }
-      },
-      {
-        numOfAttempts: 4, // no retries for webhook calls via BullMQ
-      },
-    );
-
-    // Update action execution status on success
-    await prisma.automationExecution.update({
-      where: {
-        projectId,
-        triggerId: automation.trigger.id,
-        actionId: automation.action.id,
-        id: executionId,
-      },
-      data: {
-        status: ActionExecutionStatus.COMPLETED,
-        startedAt: executionStart,
-        finishedAt: new Date(),
-      },
-    });
-
-    logger.debug(
-      `Webhook executed successfully for action ${automation.action.id}`,
-    );
-  } catch (error) {
-    logger.error("Error executing webhook", error);
-
-    const automation = await getAutomationById({
-      projectId,
-      automationId,
-    });
-
-    if (!automation) {
-      throw new LangfuseNotFoundError(`Automation ${automationId} not found`);
-    }
-
-    const shouldRetryJob =
-      error instanceof LangfuseNotFoundError ||
-      error instanceof InternalServerError;
-
-    if (shouldRetryJob) {
-      logger.warn(
-        `Retrying bullmq for webhook job for action ${automation.action.id}`,
-      );
-      throw error;
-    }
-
-    // Update action execution status and check if we should disable trigger
-    await prisma.$transaction(async (tx) => {
-      // Update execution status
-      await tx.automationExecution.update({
-        where: {
-          id: executionId,
-          projectId,
-          triggerId: automation.trigger.id,
-          actionId: automation.action.id,
-        },
-        data: {
-          status: ActionExecutionStatus.ERROR,
-          startedAt: executionStart,
-          finishedAt: new Date(),
-          error: error instanceof Error ? error.message : "Unknown error",
-          output: httpStatus
-            ? {
-                httpStatus,
-                responseBody: responseBody?.substring(0, 1000),
-              }
-            : undefined,
-        },
-      });
-
-      // Check consecutive failures from execution history
-      const consecutiveFailures = await getConsecutiveAutomationFailures({
-        automationId,
-        projectId,
-      });
-
-      logger.info(
-        `Consecutive failures: ${consecutiveFailures} for trigger ${automation.trigger.id} in project ${projectId}`,
-      );
-
-      // Check if trigger should be disabled (this is the 5th failure, looking for 4 in the past.)
-      if (consecutiveFailures >= 4) {
-        await tx.trigger.update({
-          where: { id: automation.trigger.id, projectId },
-          data: { status: JobConfigState.INACTIVE },
-        });
-
-        logger.warn(
-          `Automation ${automation.trigger.id} disabled after ${consecutiveFailures} consecutive failures in project ${projectId}`,
-        );
-      }
-    });
-
-    logger.debug(
-      `Webhook failed for action ${automation.action.id} in project ${projectId}`,
-    );
-  }
-};

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -1,4 +1,4 @@
-import { instrumentAsync } from "../instrumentation";
+import { instrumentAsync, recordDistribution } from "../instrumentation";
 import * as opentelemetry from "@opentelemetry/api";
 import { env } from "../../env";
 import { logger } from "../logger";
@@ -19,12 +19,17 @@ const executionWrapper = async <T, Y>(
   return [res, duration];
 };
 
+/**
+ * Measures the execution time of two functions and returns the result based on the experiment configuration.
+ * This is used to compare the execution of AggregatingMergeTrees with the existing ReplacingMergeTree execution.
+ */
 export const measureAndReturn = async <T, Y>(args: {
   operationName: string;
   projectId: string;
   input: T;
   existingExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
   newExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
+  minStartTime?: Date;
 }): Promise<Y> => {
   return instrumentAsync(
     {
@@ -32,14 +37,34 @@ export const measureAndReturn = async <T, Y>(args: {
       spanKind: opentelemetry.SpanKind.CLIENT,
     },
     async (currentSpan) => {
-      const { input, existingExecution, newExecution } = args;
+      const { input, existingExecution, newExecution, minStartTime } = args;
 
       if (
         env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES !==
         "true"
       ) {
         currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
-        return existingExecution(input);
+
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newExecution(input);
+          }
+        }
+
+        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
+          ? newExecution(input)
+          : existingExecution(input);
       }
 
       // If not whitelisted, apply sampling logic
@@ -68,6 +93,14 @@ export const measureAndReturn = async <T, Y>(args: {
           durationDifference,
         );
 
+        recordDistribution(
+          "langfuse.experiment.amts.duration_difference_distribution",
+          durationDifference,
+          {
+            operation: args.operationName,
+          },
+        );
+
         if (
           env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
             (p) => p === args.projectId,
@@ -83,6 +116,23 @@ export const measureAndReturn = async <T, Y>(args: {
           );
         }
 
+        // Check for short-term new result experiment
+        if (
+          env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT_SHORT_TERM === "true" &&
+          minStartTime
+        ) {
+          const thirtyDaysAgo = new Date();
+          thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+
+          if (minStartTime >= thirtyDaysAgo) {
+            currentSpan.setAttribute(
+              `langfuse.experiment.amts.short-term`,
+              "true",
+            );
+            return newResult;
+          }
+        }
+
         return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
           ? newResult
           : existingResult;

packages/shared/src/server/clickhouse/schema.ts

@@ -2,6 +2,7 @@ export const ClickhouseTableNames = {
   traces: "traces",
   observations: "observations",
   scores: "scores",
+  dataset_run_items: "dataset_run_items",
 
   // Virtual tables for dashboards
   // TODO: Check if we can do this more elegantly

packages/shared/src/server/clickhouse/schemaUtils.ts

@@ -11,7 +11,8 @@ export type IngestionEntityTypes =
   | "trace"
   | "observation"
   | "score"
-  | "sdk_log";
+  | "sdk_log"
+  | "dataset_run_item";
 
 export const getClickhouseEntityType = (
   eventType: string,
@@ -29,6 +30,8 @@ export const getClickhouseEntityType = (
       return "observation";
     case eventTypes.SCORE_CREATE:
       return "score";
+    case eventTypes.DATASET_RUN_ITEM_CREATE:
+      return "dataset_run_item";
     case eventTypes.SDK_LOG:
       return "sdk_log";
     default:

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/dataset-run-items/datasetExecution.ts

@@ -0,0 +1,94 @@
+import { env } from "../../env";
+import { logger } from "../../server/logger";
+import {
+  DatasetRunItemsExecutionStrategy,
+  DatasetRunItemsOperationType,
+} from "./types";
+/**
+ * Returns the execution strategy for dataset run items based on environment variables.
+ *
+ * Two-phase migration approach:
+ * 1. Dual-write phase: DATASET_RUN_ITEMS_WRITE_TO_CLICKHOUSE=true (write to both databases)
+ * 2. Read migration phase: DATASET_RUN_ITEMS_READ_FROM_CLICKHOUSE=true (read from ClickHouse)
+ */
+function getDatasetRunItemsExecutionStrategy(): DatasetRunItemsExecutionStrategy {
+  return {
+    shouldWriteToClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH === "true",
+    shouldReadFromClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH === "true",
+  };
+}
+
+// Re-export the enum for backward compatibility
+
+/**
+ * Executes the appropriate database operation based on the execution strategy.
+ *
+ * @param postgresExecution - Function to execute PostgreSQL operation
+ * @param clickhouseExecution - Function to execute ClickHouse operation
+ * @param operationType - Type of operation ("read" or "write")
+ * @returns Result from the selected execution strategy
+ */
+export async function executeWithDatasetRunItemsStrategy<TInput, TOutput>({
+  input,
+  operationType,
+  postgresExecution,
+  clickhouseExecution,
+}: {
+  input: TInput;
+  operationType: DatasetRunItemsOperationType;
+  // eslint-disable-next-line no-unused-vars
+  postgresExecution: (input: TInput) => Promise<TOutput>;
+  // eslint-disable-next-line no-unused-vars
+  clickhouseExecution: (input: TInput) => Promise<TOutput>;
+}): Promise<TOutput> {
+  const strategy = getDatasetRunItemsExecutionStrategy();
+
+  if (operationType === DatasetRunItemsOperationType.WRITE) {
+    // For write operations, implement dual-write strategy
+    if (strategy.shouldWriteToClickHouse) {
+      // Dual-write phase: write to both databases
+      const postgresResult = await postgresExecution(input);
+
+      try {
+        await clickhouseExecution(input);
+        logger.debug("Successfully wrote to both PostgreSQL and ClickHouse", {
+          operation: `dataset_run_items_${operationType}`,
+        });
+      } catch (error) {
+        logger.error("ClickHouse write failed during dual-write phase", {
+          error: error instanceof Error ? error.message : String(error),
+          operation: `dataset_run_items_${operationType}`,
+        });
+        // Continue with PostgreSQL result since it succeeded
+      }
+
+      return postgresResult;
+    } else {
+      // Write only to PostgreSQL
+      return await postgresExecution(input);
+    }
+  } else {
+    // For read operations, rely on the strategy
+    const shouldExecuteClickhouse = strategy.shouldReadFromClickHouse;
+
+    if (shouldExecuteClickhouse) {
+      try {
+        return await clickhouseExecution(input);
+      } catch (error) {
+        logger.error(
+          "ClickHouse execution failed, falling back to PostgreSQL",
+          {
+            error: error instanceof Error ? error.message : String(error),
+            operation: `dataset_run_items_${operationType}`,
+          },
+        );
+        // Fallback to PostgreSQL for reliability
+        return await postgresExecution(input);
+      }
+    } else {
+      return await postgresExecution(input);
+    }
+  }
+}

packages/shared/src/server/dataset-run-items/types.ts

@@ -0,0 +1,16 @@
+/**
+ * Types and enums for dataset run items execution.
+ * This file is frontend-safe and doesn't import server-side dependencies.
+ */
+
+export enum DatasetRunItemsOperationType {
+  // eslint-disable-next-line no-unused-vars
+  READ = "read",
+  // eslint-disable-next-line no-unused-vars
+  WRITE = "write",
+}
+
+export type DatasetRunItemsExecutionStrategy = {
+  shouldWriteToClickHouse: boolean;
+  shouldReadFromClickHouse: boolean;
+};

packages/shared/src/server/index.ts

@@ -2,6 +2,7 @@ export * from "./services/StorageService";
 export * from "./services/email/organizationInvitation/sendMembershipInvitationEmail";
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
+export * from "./services/email/billingAlert/sendBillingAlertEmail";
 export * from "./services/PromptService";
 export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
@@ -13,6 +14,7 @@ export * from "./llm/fetchLLMCompletion";
 export * from "./llm/utils";
 export * from "./llm/types";
 export * from "./llm/compileChatMessages";
+export * from "./llm/testModelCall";
 export * from "./utils/DatabaseReadStream";
 export * from "./utils/transforms";
 export * from "./clickhouse/client";
@@ -20,8 +22,8 @@ export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
+export * from "../server/ingestion/modelMatch";
 export * from "./ingestion/processEventBatch";
-export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
@@ -32,6 +34,7 @@ export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -56,6 +59,7 @@ export * from "./logger";
 export * from "./headerPropagation";
 export * from "./queries";
 export * from "./repositories";
+export * from "./utils/rendering";
 export * from "./redis/evalExecutionQueue";
 export * from "./services/sessions-ui-table-service";
 export * from "./services/datasets-ui-table-service";
@@ -63,11 +67,17 @@ export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
 export * from "./clickhouse/measureAndReturn";
+export * from "./services/SlackService";
 
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
-export * from "./automations/webhooks";
+// dataset run items
+export * from "./dataset-run-items/datasetExecution";
+export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
 
 // test utils
 export * from "./test-utils";
+export * from "./utils/headerUtils";
+export * from "./traceDeletionProcessor";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -1,19 +1,23 @@
-import { Model, Prisma } from "@langfuse/shared";
+import { Model, Prisma } from "../../";
 import {
   instrumentAsync,
   logger,
   recordIncrement,
-} from "@langfuse/shared/src/server";
-import { env } from "../env";
-import { redis } from "@langfuse/shared/src/server";
+  redis,
+  safeMultiDel,
+} from "../";
+import { type Cluster } from "ioredis";
+import { env } from "../../env";
 import { Decimal } from "decimal.js";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma } from "../../db";
 
 export type ModelMatchProps = {
   projectId: string;
   model: string;
 };
 
+const MODEL_MATCH_CACHE_LOCKED_KEY = "LOCK:model-match-clear";
+
 export async function findModel(p: ModelMatchProps): Promise<Model | null> {
   return instrumentAsync(
     {
@@ -76,12 +80,20 @@ const getModelFromRedis = async (
   }
 
   try {
+    if (await isModelMatchCacheLocked()) {
+      logger.info(
+        "Model match cache is locked. Skipping model lookup from Redis.",
+      );
+
+      return null;
+    }
+
     const key = getRedisModelKey(p);
     const redisModel = await redis?.get(key);
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;
@@ -178,6 +190,11 @@ export const getRedisModelKey = (p: ModelMatchProps) => {
 };
 
 const getModelMatchKeyPrefix = () => {
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // Use hash tags for Redis cluster compatibility
+    // This ensures all model cache keys are placed on the same hash slot
+    return "{model-match}";
+  }
   return "model-match";
 };
 
@@ -205,3 +222,99 @@ export const redisModelToPrismaModel = (redisModel: string): Model => {
         : null,
   };
 };
+
+export async function clearModelCacheForProject(
+  projectId: string,
+): Promise<void> {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    const pattern = `${getModelMatchKeyPrefix()}:${projectId}:*`;
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared ${keys.length} model cache entries for project ${projectId}`,
+      );
+    }
+  } catch (error) {
+    logger.error(
+      `Error clearing model cache for project ${projectId}: ${error}`,
+    );
+  }
+}
+
+export async function isModelMatchCacheLocked() {
+  try {
+    return Boolean(await redis?.exists(MODEL_MATCH_CACHE_LOCKED_KEY));
+  } catch (err) {
+    logger.error("Failed to check whether model match is locked", err);
+
+    return false;
+  }
+}
+
+export async function clearFullModelCache() {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    // Use lock to protect for concurrent executions
+    // This function is called on worker startup, so we want to avoid all workers triggering this delete
+    if (await isModelMatchCacheLocked()) {
+      logger.info("Model cache clearing already in progress; skipping.");
+
+      return;
+    }
+
+    const startTime = Date.now();
+    logger.info("Clearing full model cache...");
+
+    const tenMinutesInSeconds = 60 * 10;
+    await redis.setex(
+      MODEL_MATCH_CACHE_LOCKED_KEY,
+      tenMinutesInSeconds,
+      "locked",
+    );
+
+    const pattern = getModelMatchKeyPrefix() + "*";
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared full model cache with ${keys.length} keys in ${Date.now() - startTime}ms.`,
+      );
+    } else {
+      logger.info(`No keys found for match pattern '${pattern}'`);
+    }
+  } catch (error) {
+    logger.error(`Error clearing full model cache: ${error}`);
+  } finally {
+    await redis?.del(MODEL_MATCH_CACHE_LOCKED_KEY);
+  }
+}

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -8,7 +8,7 @@ import {
   LangfuseNotFoundError,
   UnauthorizedError,
 } from "../../errors";
-import { AuthHeaderValidVerificationResult } from "../auth/types";
+import { AuthHeaderValidVerificationResultIngestion } from "../auth/types";
 import { getClickhouseEntityType } from "../clickhouse/schemaUtils";
 import {
   getCurrentSpan,
@@ -21,11 +21,16 @@ import { logger } from "../logger";
 import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
 import { redis } from "../redis/redis";
-import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+import {
+  eventTypes,
+  createIngestionEventSchema,
+  IngestionEventType,
+} from "./types";
 import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -57,7 +62,7 @@ export type TokenCountDelegate = (p: {
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -69,24 +74,38 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
   return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
 };
 
+/**
+ * Options for event batch processing.
+ * @property delay - Delay in ms to wait before processing events in the batch.
+ * @property source - Source of the events for metrics tracking (e.g., "otel", "api").
+ * @property isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ */
+type ProcessEventBatchOptions = {
+  delay?: number | null;
+  source?: "api" | "otel";
+  isLangfuseInternal?: boolean;
+};
+
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
- * @param authCheck - AuthHeaderValidVerificationResult
- * @param delay - (Optional) Delay in ms to wait before processing events in the batch.
- * @param source - (Optional) Source of the events for metrics tracking (e.g., "otel", "api").
+ * @param authCheck - AuthHeaderValidVerificationResultIngestion
+ * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
-  authCheck: AuthHeaderValidVerificationResult,
-  delay: number | null = null,
-  source: "api" | "otel" = "api",
+  authCheck: AuthHeaderValidVerificationResultIngestion,
+  options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
   errors: {
@@ -96,6 +115,8 @@ export const processEventBatch = async (
     error?: string;
   }[];
 }> => {
+  const { delay = null, source = "api", isLangfuseInternal = false } = options;
+
   // add context of api call to the span
   const currentSpan = getCurrentSpan();
   recordIncrement("langfuse.ingestion.event", input.length, { source });
@@ -108,8 +129,10 @@ export const processEventBatch = async (
     "langfuse.project.id",
     authCheck.scope.projectId ?? "",
   );
-  currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
-  currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
+  if (authCheck.scope.orgId)
+    currentSpan?.setAttribute("langfuse.org.id", authCheck.scope.orgId);
+  if (authCheck.scope.plan)
+    currentSpan?.setAttribute("langfuse.org.plan", authCheck.scope.plan);
 
   /**************
    * VALIDATION *
@@ -121,9 +144,10 @@ export const processEventBatch = async (
   const validationErrors: { id: string; error: unknown }[] = [];
   const authenticationErrors: { id: string; error: unknown }[] = [];
 
-  const batch: z.infer<typeof ingestionEvent>[] = input
+  const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
+  const batch: z.infer<typeof ingestionSchema>[] = input
     .flatMap((event) => {
-      const parsed = ingestionEvent.safeParse(event);
+      const parsed = ingestionSchema.safeParse(event);
       if (!parsed.success) {
         validationErrors.push({
           id:
@@ -239,11 +263,39 @@ export const processEventBatch = async (
       const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
       const queue = IngestionQueue.getInstance({ shardingKey });
 
-      const shouldSkipS3List =
-        getClickhouseEntityType(eventData.type) === "observation" &&
+      const isDatasetRunItemEvent =
+        getClickhouseEntityType(eventData.type) === "dataset_run_item";
+      const isObservationEvent =
+        getClickhouseEntityType(eventData.type) === "observation";
+
+      const isOtelOrSkipS3Project =
         authCheck.scope.projectId !== null &&
-        (projectIdsToSkipS3List.includes(authCheck.scope.projectId) ||
-          source === "otel");
+        (source === "otel" ||
+          projectIdsToSkipS3List.includes(authCheck.scope.projectId));
+
+      const shouldSkipS3List =
+        isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
+
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
 
       return queue
         ? queue.add(
@@ -268,7 +320,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -283,7 +335,7 @@ export const processEventBatch = async (
 
 const isAuthorized = (
   event: IngestionEventType,
-  authScope: AuthHeaderValidVerificationResult,
+  authScope: AuthHeaderValidVerificationResultIngestion,
 ): boolean => {
   if (event.type === eventTypes.SDK_LOG) {
     return true;
@@ -302,7 +354,7 @@ const isAuthorized = (
 /**
  * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
  */
-const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+const sortBatch = (batch: IngestionEventType[]) => {
   const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
     eventTypes.GENERATION_UPDATE,
     eventTypes.SPAN_UPDATE,

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/ingestion/types.ts

@@ -193,178 +193,42 @@ export const UsageDetails = z
   ])
   .nullish();
 
-export const EnvironmentName = z
+const INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  "Only alphanumeric lower case characters, hyphens, and underscores are allowed";
+
+const ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE +
+  " and it must not start with 'langfuse'";
+
+const PublicEnvironmentName = z
   .string()
   .max(40, "Maximum length is 40 characters")
-  .regex(
-    /^(?!langfuse)[a-z0-9-_]+$/,
-    "Only alphanumeric lower case characters, hyphens, and underscores are allowed, and it must not start with 'langfuse'",
-  )
+  .regex(/^(?!langfuse)[a-z0-9-_]+$/, ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
   .default("default");
 
-// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
-// With this setup parsing should be more lightweight and doesn't block other requests.
-// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
-export const TraceBody = z.object({
-  id: idSchema.nullish(),
-  timestamp: stringDateTime,
-  name: z.string().max(1000).nullish(),
-  externalId: z.string().nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  sessionId: z.string().nullish(),
-  userId: z.string().nullish(),
-  environment: EnvironmentName,
-  metadata: jsonSchema.nullish(),
-  release: z.string().nullish(),
-  version: z.string().nullish(),
-  public: z.boolean().nullish(),
-  tags: z.array(z.string()).nullish(),
-});
+const InternalEnvironmentName = z
+  .string()
+  .max(40, "Maximum length is 40 characters")
+  .regex(/^[a-z0-9-_]+$/, INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
+  .default("default");
 
-export const OptionalObservationBody = z.object({
-  traceId: idSchema.nullish(),
-  environment: EnvironmentName,
-  name: z.string().nullish(),
-  startTime: stringDateTime,
-  metadata: jsonSchema.nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  level: ObservationLevel.nullish(),
-  statusMessage: z.string().nullish(),
-  parentObservationId: z.string().nullish(),
-  version: z.string().nullish(),
-});
+/** @deprecated Use PublicEnvironmentName or InternalEnvironmentName instead */
+export const EnvironmentName = PublicEnvironmentName;
 
-export const CreateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const UpdateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const CreateSpanBody = CreateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const UpdateSpanBody = UpdateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const CreateGenerationBody = CreateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-export const UpdateGenerationBody = UpdateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-const BaseScoreBody = z.object({
-  id: idSchema.nullish(),
-  name: NonEmptyString,
-  traceId: z.string().nullish(),
-  sessionId: z.string().nullish(),
-  datasetRunId: z.string().nullish(),
-  environment: EnvironmentName,
-  observationId: z.string().nullish(),
-  comment: z.string().nullish(),
-  metadata: jsonSchema.nullish(),
-  source: z
-    .enum(["API", "EVAL", "ANNOTATION"])
-    .default("API" as ScoreSourceType),
-});
-
-/**
- * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
- */
-export const ScoreBody = applyScoreValidation(
-  z.discriminatedUnion("dataType", [
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number(),
-        dataType: z.literal("NUMERIC"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.string(),
-        dataType: z.literal("CATEGORICAL"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number().refine((value) => value === 0 || value === 1, {
-          message:
-            "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
-        }),
-        dataType: z.literal("BOOLEAN"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.union([z.string(), z.number()]),
-        dataType: z.undefined(),
-        configId: z.string().nullish(),
-      }),
-    ),
-  ]),
-);
+export const eventTypes = {
+  TRACE_CREATE: "trace-create",
+  SCORE_CREATE: "score-create",
+  EVENT_CREATE: "event-create",
+  SPAN_CREATE: "span-create",
+  SPAN_UPDATE: "span-update",
+  GENERATION_CREATE: "generation-create",
+  GENERATION_UPDATE: "generation-update",
+  SDK_LOG: "sdk-log",
+  DATASET_RUN_ITEM_CREATE: "dataset-run-item-create",
+  // LEGACY, only required for backwards compatibility
+  OBSERVATION_CREATE: "observation-create",
+  OBSERVATION_UPDATE: "observation-update",
+} as const;
 
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
@@ -475,83 +339,366 @@ export const SdkLogEvent = z.object({
   id: z.string().nullish(), // Not used, but makes downstream processing easier.
 });
 
-export const eventTypes = {
-  TRACE_CREATE: "trace-create",
-  SCORE_CREATE: "score-create",
-  EVENT_CREATE: "event-create",
-  SPAN_CREATE: "span-create",
-  SPAN_UPDATE: "span-update",
-  GENERATION_CREATE: "generation-create",
-  GENERATION_UPDATE: "generation-update",
-  SDK_LOG: "sdk-log",
-  // LEGACY, only required for backwards compatibility
-  OBSERVATION_CREATE: "observation-create",
-  OBSERVATION_UPDATE: "observation-update",
-} as const;
+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 
-const base = z.object({
-  id: idSchema,
-  timestamp: z.string().datetime({ offset: true }),
-  metadata: jsonSchema.nullish(),
-});
-export const traceEvent = base.extend({
-  type: z.literal(eventTypes.TRACE_CREATE),
-  body: TraceBody,
-});
-export type TraceEventType = z.infer<typeof traceEvent>;
+// Complete schema factory - single source of truth for ALL schemas
+const createAllIngestionSchemas = ({
+  isPublic = true,
+}: {
+  isPublic: boolean;
+}) => {
+  const environmentSchema = isPublic
+    ? PublicEnvironmentName
+    : InternalEnvironmentName;
 
-export const eventCreateEvent = base.extend({
-  type: z.literal(eventTypes.EVENT_CREATE),
-  body: CreateEventEvent,
-});
-export const spanCreateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_CREATE),
-  body: CreateSpanBody,
-});
-export const spanUpdateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_UPDATE),
-  body: UpdateSpanBody,
-});
-export const generationCreateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_CREATE),
-  body: CreateGenerationBody,
-});
-export const generationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_UPDATE),
-  body: UpdateGenerationBody,
-});
-export const scoreEvent = base.extend({
-  type: z.literal(eventTypes.SCORE_CREATE),
-  body: ScoreBody,
-});
-export type ScoreEventType = z.infer<typeof scoreEvent>;
-export const sdkLogEvent = base.extend({
-  type: z.literal(eventTypes.SDK_LOG),
-  body: SdkLogEvent,
-});
-export const legacyObservationCreateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_CREATE),
-  body: LegacyObservationBody,
-});
-export const legacyObservationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_UPDATE),
-  body: LegacyObservationBody,
-});
+  // Base schemas with environment
+  const TraceBody = z.object({
+    id: idSchema.nullish(),
+    timestamp: stringDateTime,
+    name: z.string().max(1000).nullish(),
+    externalId: z.string().nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    sessionId: z.string().nullish(),
+    userId: z.string().nullish(),
+    environment: environmentSchema,
+    metadata: jsonSchema.nullish(),
+    release: z.string().nullish(),
+    version: z.string().nullish(),
+    public: z.boolean().nullish(),
+    tags: z.array(z.string()).nullish(),
+  });
 
-export const ingestionEvent = z.discriminatedUnion("type", [
-  traceEvent,
-  scoreEvent,
-  eventCreateEvent,
-  spanCreateEvent,
-  spanUpdateEvent,
-  generationCreateEvent,
-  generationUpdateEvent,
-  sdkLogEvent,
-  // LEGACY, only required for backwards compatibility
-  legacyObservationCreateEvent,
-  legacyObservationUpdateEvent,
-]);
+  const OptionalObservationBody = z.object({
+    traceId: idSchema.nullish(),
+    environment: environmentSchema,
+    name: z.string().nullish(),
+    startTime: stringDateTime,
+    metadata: jsonSchema.nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    level: ObservationLevel.nullish(),
+    statusMessage: z.string().nullish(),
+    parentObservationId: z.string().nullish(),
+    version: z.string().nullish(),
+  });
+
+  // Derivative schemas
+  const CreateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const UpdateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const CreateSpanBody = CreateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const UpdateSpanBody = UpdateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const CreateGenerationBody = CreateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const UpdateGenerationBody = UpdateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const BaseScoreBody = z.object({
+    id: idSchema.nullish(),
+    name: NonEmptyString,
+    traceId: z.string().nullish(),
+    sessionId: z.string().nullish(),
+    datasetRunId: z.string().nullish(),
+    environment: environmentSchema,
+    observationId: z.string().nullish(),
+    comment: z.string().nullish(),
+    metadata: jsonSchema.nullish(),
+    source: z
+      .enum(["API", "EVAL", "ANNOTATION"])
+      .default("API" as ScoreSourceType),
+  });
+
+  const ScoreBody = applyScoreValidation(
+    z.discriminatedUnion("dataType", [
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number(),
+          dataType: z.literal("NUMERIC"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.string(),
+          dataType: z.literal("CATEGORICAL"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number().refine((value) => value === 0 || value === 1, {
+            message:
+              "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
+          }),
+          dataType: z.literal("BOOLEAN"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.union([z.string(), z.number()]),
+          dataType: z.undefined(),
+          configId: z.string().nullish(),
+        }),
+      ),
+    ]),
+  );
+
+  const DatasetRunItemBody = z.object({
+    // Core identifiers
+    id: idSchema.nullish(),
+    traceId: z.string(),
+    observationId: z.string().nullish(),
+    error: z.string().nullish(),
+    // Metadata (optional)
+    createdAt: stringDateTime.nullish(),
+    // Dataset identification
+    datasetId: z.string(),
+    // Run identification
+    runId: z.string(),
+    // Dataset item identification
+    datasetItemId: z.string(),
+  });
+
+  // Event schemas
+  const base = z.object({
+    id: idSchema,
+    timestamp: z.string().datetime({ offset: true }),
+    metadata: jsonSchema.nullish(),
+  });
+
+  const traceEvent = base.extend({
+    type: z.literal(eventTypes.TRACE_CREATE),
+    body: TraceBody,
+  });
+
+  const eventCreateEvent = base.extend({
+    type: z.literal(eventTypes.EVENT_CREATE),
+    body: CreateEventEvent,
+  });
+
+  const spanCreateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_CREATE),
+    body: CreateSpanBody,
+  });
+
+  const spanUpdateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_UPDATE),
+    body: UpdateSpanBody,
+  });
+
+  const generationCreateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_CREATE),
+    body: CreateGenerationBody,
+  });
+
+  const generationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_UPDATE),
+    body: UpdateGenerationBody,
+  });
+
+  const scoreEvent = base.extend({
+    type: z.literal(eventTypes.SCORE_CREATE),
+    body: ScoreBody,
+  });
+
+  const baseDatasetRunItemCreateEvent = base.extend({
+    type: z.literal(eventTypes.DATASET_RUN_ITEM_CREATE),
+    body: DatasetRunItemBody,
+  });
+
+  const datasetRunItemCreateEvent = isPublic
+    ? baseDatasetRunItemCreateEvent.refine(() => false, {
+        message: "Dataset run item creation is only allowed for internal usage",
+      })
+    : baseDatasetRunItemCreateEvent;
+
+  const sdkLogEvent = base.extend({
+    type: z.literal(eventTypes.SDK_LOG),
+    body: SdkLogEvent,
+  });
+
+  const legacyObservationCreateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_CREATE),
+    body: LegacyObservationBody,
+  });
+
+  const legacyObservationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_UPDATE),
+    body: LegacyObservationBody,
+  });
+
+  const ingestionEvent = z.discriminatedUnion("type", [
+    traceEvent,
+    scoreEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    sdkLogEvent,
+    datasetRunItemCreateEvent,
+    // LEGACY, only required for backwards compatibility
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+  ]);
+
+  return {
+    // Body schemas
+    TraceBody,
+    OptionalObservationBody,
+    CreateEventEvent,
+    UpdateEventEvent,
+    CreateSpanBody,
+    UpdateSpanBody,
+    CreateGenerationBody,
+    UpdateGenerationBody,
+    BaseScoreBody,
+    ScoreBody,
+    // Event schemas
+    traceEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    scoreEvent,
+    datasetRunItemCreateEvent,
+    sdkLogEvent,
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+    // Complete schema
+    ingestionEvent,
+  };
+};
+
+// Create both public and internal schema instances
+const publicSchemas = createAllIngestionSchemas({ isPublic: true });
+const internalSchemas = createAllIngestionSchemas({ isPublic: false });
+
+// Export individual schemas for backwards compatibility
+export const TraceBody = publicSchemas.TraceBody;
+export const OptionalObservationBody = publicSchemas.OptionalObservationBody;
+export const CreateEventEvent = publicSchemas.CreateEventEvent;
+export const UpdateEventEvent = publicSchemas.UpdateEventEvent;
+export const CreateSpanBody = publicSchemas.CreateSpanBody;
+export const UpdateSpanBody = publicSchemas.UpdateSpanBody;
+export const CreateGenerationBody = publicSchemas.CreateGenerationBody;
+export const UpdateGenerationBody = publicSchemas.UpdateGenerationBody;
+export const BaseScoreBody = publicSchemas.BaseScoreBody;
+
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = publicSchemas.ScoreBody;
+
+// Export individual event schemas for backwards compatibility
+export const traceEvent = publicSchemas.traceEvent;
+export const eventCreateEvent = publicSchemas.eventCreateEvent;
+export const spanCreateEvent = publicSchemas.spanCreateEvent;
+export const spanUpdateEvent = publicSchemas.spanUpdateEvent;
+export const generationCreateEvent = publicSchemas.generationCreateEvent;
+export const generationUpdateEvent = publicSchemas.generationUpdateEvent;
+export const scoreEvent = publicSchemas.scoreEvent;
+export const sdkLogEvent = publicSchemas.sdkLogEvent;
+export const datasetRunItemCreateEvent =
+  publicSchemas.datasetRunItemCreateEvent;
+export const legacyObservationCreateEvent =
+  publicSchemas.legacyObservationCreateEvent;
+export const legacyObservationUpdateEvent =
+  publicSchemas.legacyObservationUpdateEvent;
+
+/** @deprecated Use createIngestionEventSchema() instead */
+export const ingestionEvent = publicSchemas.ingestionEvent;
+
+/**
+ * Type definitions for both schema variants (public and internal).
+ * These types are equivalent to the return types of createIngestionEventSchema() and all exported schemas,
+ * since the factory patterns only differ in environment validation rules, not in the actual TypeScript types.
+ * The environment field remains `string` in all cases - only the validation logic differs.
+ */
 export type IngestionEventType = z.infer<typeof ingestionEvent>;
+export type TraceEventType = z.infer<typeof traceEvent>;
+export type ScoreEventType = z.infer<typeof scoreEvent>;
+export type DatasetRunItemEventType = z.infer<typeof datasetRunItemCreateEvent>;
+
+/**
+ * Creates an ingestion event schema with appropriate environment validation.
+ * @param isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ * @returns The ingestion event schema.
+ */
+export const createIngestionEventSchema = (isLangfuseInternal = false) => {
+  return isLangfuseInternal
+    ? internalSchemas.ingestionEvent
+    : publicSchemas.ingestionEvent;
+};
 
 export type ObservationEvent =
   | z.infer<typeof legacyObservationCreateEvent>

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -19,10 +19,12 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
+import { env } from "../../env";
 import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
   VertexAIConfigSchema,
+  BEDROCK_USE_DEFAULT_CREDENTIALS,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
@@ -40,6 +42,9 @@ import {
 } from "./types";
 import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
+import { HttpsProxyAgent } from "https-proxy-agent";
+
+const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
 
 type ProcessTracedEvents = () => Promise<void>;
 
@@ -137,7 +142,7 @@ export async function fetchLLMCompletion(
     const handler = new CallbackHandler({
       _projectId: traceParams.projectId,
       _isLocalEventExportEnabled: true,
-      tags: traceParams.tags,
+      environment: traceParams.environment,
     });
     finalCallbacks.push(handler);
 
@@ -149,6 +154,7 @@ export async function fetchLLMCompletion(
         await processEventBatch(
           JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
           traceParams.authCheck,
+          { isLangfuseInternal: true },
         );
       } catch (e) {
         logger.error("Failed to process traced events", { error: e });
@@ -191,11 +197,12 @@ export async function fetchLLMCompletion(
       )
         return new SystemMessage(safeContent);
 
-      if (message.type === ChatMessageType.ToolResult)
+      if (message.type === ChatMessageType.ToolResult) {
         return new ToolMessage({
           content: safeContent,
           tool_call_id: message.toolCallId,
         });
+      }
 
       return new AIMessage({
         content: safeContent,
@@ -211,6 +218,10 @@ export async function fetchLLMCompletion(
     (m) => m.content.length > 0 || "tool_calls" in m,
   );
 
+  // Common proxy configuration for all adapters
+  const proxyUrl = env.HTTPS_PROXY;
+  const proxyAgent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined;
+
   let chatModel:
     | ChatOpenAI
     | ChatAnthropic
@@ -226,7 +237,11 @@ export async function fetchLLMCompletion(
       maxTokens: modelParams.max_tokens,
       topP: modelParams.top_p,
       callbacks: finalCallbacks,
-      clientOptions: { maxRetries, timeout: 1000 * 60 * 2 }, // 2 minutes timeout
+      clientOptions: {
+        maxRetries,
+        timeout: 1000 * 60 * 2, // 2 minutes timeout
+        ...(proxyAgent && { httpAgent: proxyAgent }),
+      },
     });
   } else if (modelParams.adapter === LLMAdapter.OpenAI) {
     chatModel = new ChatOpenAI({
@@ -241,6 +256,7 @@ export async function fetchLLMCompletion(
       configuration: {
         baseURL,
         defaultHeaders: extraHeaders,
+        ...(proxyAgent && { httpAgent: proxyAgent }),
       },
       timeout: 1000 * 60 * 2, // 2 minutes timeout
     });
@@ -258,11 +274,16 @@ export async function fetchLLMCompletion(
       timeout: 1000 * 60 * 2, // 2 minutes timeout
       configuration: {
         defaultHeaders: extraHeaders,
+        ...(proxyAgent && { httpAgent: proxyAgent }),
       },
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
-    const credentials = BedrockCredentialSchema.parse(JSON.parse(apiKey));
+    // Handle both explicit credentials and default provider chain
+    const credentials =
+      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS && !isLangfuseCloud
+        ? undefined // undefined = use AWS SDK default credential provider chain
+        : BedrockCredentialSchema.parse(JSON.parse(apiKey));
 
     chatModel = new ChatBedrockConverse({
       model: modelParams.model,
@@ -306,22 +327,6 @@ export async function fetchLLMCompletion(
       maxRetries,
       apiKey,
     });
-  } else if (modelParams.adapter === LLMAdapter.Atla) {
-    // Atla models do not support:
-    // - temperature
-    // - max_tokens
-    // - top_p
-    chatModel = new ChatOpenAI({
-      openAIApiKey: apiKey,
-      modelName: modelParams.model,
-      callbacks: finalCallbacks,
-      maxRetries,
-      configuration: {
-        baseURL: baseURL,
-        defaultHeaders: extraHeaders,
-      },
-      timeout: 1000 * 60, // 1 minute timeout
-    });
   } else {
     // eslint-disable-next-line no-unused-vars
     const _exhaustiveCheck: never = modelParams.adapter;
@@ -382,6 +387,7 @@ export async function fetchLLMCompletion(
           },
           configuration: {
             baseURL,
+            ...(proxyAgent && { httpAgent: proxyAgent }),
           },
           timeout: 1000 * 60 * 2, // 2 minutes timeout
         })

packages/shared/src/server/llm/testModelCall.ts

@@ -0,0 +1,52 @@
+import { z as zodV3 } from "zod/v3";
+import {
+  ChatMessageRole,
+  ChatMessageType,
+  LLMApiKeySchema,
+  type ModelConfig,
+} from "./types";
+import { decrypt } from "../../encryption";
+import { fetchLLMCompletion } from "./fetchLLMCompletion";
+import { decryptAndParseExtraHeaders } from "./utils";
+import z from "zod/v4";
+
+export const testModelCall = async ({
+  provider,
+  model,
+  apiKey,
+  prompt,
+  modelConfig,
+}: {
+  provider: string;
+  model: string;
+  apiKey: z.infer<typeof LLMApiKeySchema>;
+  prompt?: string;
+  modelConfig?: ModelConfig | null;
+}): Promise<void> => {
+  (
+    await fetchLLMCompletion({
+      streaming: false,
+      apiKey: decrypt(apiKey.secretKey), // decrypt the secret key
+      extraHeaders: decryptAndParseExtraHeaders(apiKey.extraHeaders),
+      baseURL: apiKey.baseURL ?? undefined,
+      messages: [
+        {
+          role: ChatMessageRole.User,
+          content: prompt ?? "mock content",
+          type: ChatMessageType.User,
+        },
+      ],
+      modelParams: {
+        provider: provider,
+        model: model,
+        adapter: apiKey.adapter,
+        ...modelConfig,
+      },
+      structuredOutputSchema: zodV3.object({
+        score: zodV3.string(),
+        reasoning: zodV3.string(),
+      }),
+      config: apiKey.config,
+    })
+  ).completion;
+};

packages/shared/src/server/llm/types.ts

@@ -113,6 +113,7 @@ export enum ChatMessageRole {
   User = "user",
   Assistant = "assistant",
   Tool = "tool",
+  Model = "model", // Google Gemini assistant format
 }
 
 // Thought: should placeholder not semantically be part of this, because it can be
@@ -124,6 +125,7 @@ export enum ChatMessageType {
   AssistantText = "assistant-text",
   AssistantToolCall = "assistant-tool-call",
   ToolResult = "tool-result",
+  ModelText = "model-text",
   PublicAPICreated = "public-api-created",
   Placeholder = "placeholder",
 }
@@ -156,6 +158,13 @@ export const AssistantTextMessageSchema = z.object({
 });
 export type AssistantTextMessage = z.infer<typeof AssistantTextMessageSchema>;
 
+export const ModelMessageSchema = z.object({
+  type: z.literal(ChatMessageType.ModelText),
+  role: z.literal(ChatMessageRole.Model),
+  content: z.string(),
+});
+export type ModelMessage = z.infer<typeof ModelMessageSchema>;
+
 export const AssistantToolCallMessageSchema = z.object({
   type: z.literal(ChatMessageType.AssistantToolCall),
   role: z.literal(ChatMessageRole.Assistant),
@@ -193,6 +202,7 @@ export const ChatMessageSchema = z.union([
   AssistantTextMessageSchema,
   AssistantToolCallMessageSchema,
   ToolResultMessageSchema,
+  ModelMessageSchema,
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
@@ -226,18 +236,12 @@ export type PromptVariable = { name: string; value: string; isUsed: boolean };
 export enum LLMAdapter {
   Anthropic = "anthropic",
   OpenAI = "openai",
-  Atla = "atla",
   Azure = "azure",
   Bedrock = "bedrock",
   VertexAI = "google-vertex-ai",
   GoogleAIStudio = "google-ai-studio",
 }
 
-export const SYSTEM_ROLES: string[] = [
-  ChatMessageRole.System,
-  ChatMessageRole.Developer,
-];
-
 export const TextPromptContentSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
@@ -290,6 +294,12 @@ export const openAIModels = [
   "gpt-4.1-mini-2025-04-14",
   "gpt-4.1-nano",
   "gpt-4.1-nano-2025-04-14",
+  "gpt-5",
+  "gpt-5-2025-08-07",
+  "gpt-5-mini",
+  "gpt-5-mini-2025-08-07",
+  "gpt-5-nano",
+  "gpt-5-nano-2025-08-07",
   "o3",
   "o3-2025-04-16",
   "o4-mini",
@@ -327,6 +337,7 @@ export type OpenAIModel = (typeof openAIModels)[number];
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const anthropicModels = [
   "claude-sonnet-4-20250514",
+  "claude-opus-4-1-20250805",
   "claude-opus-4-20250514",
   "claude-3-7-sonnet-20250219",
   "claude-3-5-sonnet-20241022",
@@ -373,8 +384,6 @@ export const googleAIStudioModels = [
   "gemini-1.5-flash-8b",
 ] as const;
 
-export const atlaModels = ["atla-selene", "atla-selene-20250214"] as const;
-
 export type AnthropicModel = (typeof anthropicModels)[number];
 export type VertexAIModel = (typeof vertexAIModels)[number];
 export const supportedModels = {
@@ -384,7 +393,6 @@ export const supportedModels = {
   [LLMAdapter.GoogleAIStudio]: googleAIStudioModels,
   [LLMAdapter.Azure]: [],
   [LLMAdapter.Bedrock]: [],
-  [LLMAdapter.Atla]: atlaModels,
 } as const;
 
 export type LLMFunctionCall = {
@@ -419,11 +427,17 @@ export type LLMApiKey =
     ? z.infer<typeof LLMApiKeySchema>
     : never;
 
+// NOTE: This string is whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
+export const PROMPT_EXPERIMENT_ENVIRONMENT =
+  "langfuse-prompt-experiment" as const;
+
+type PromptExperimentEnvironment = typeof PROMPT_EXPERIMENT_ENVIRONMENT;
+
 export type TraceParams = {
   traceName: string;
   traceId: string;
   projectId: string;
-  tags: string[];
+  environment: PromptExperimentEnvironment;
   tokenCountDelegate: TokenCountDelegate;
   authCheck: AuthHeaderValidVerificationResult;
 };

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -6,6 +6,7 @@ export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
   tablePrefix?: string,
+  useTracesAmtCompatMode: boolean = false,
 ) => {
   const prefix = tablePrefix ? `${tablePrefix}.` : "";
 
@@ -13,9 +14,11 @@ export const clickhouseSearchCondition = (
     !searchType || searchType.includes("id")
       ? `${prefix}id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
-    searchType && searchType.includes("content")
+    searchType && searchType.includes("content") && !useTracesAmtCompatMode
       ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
-      : null,
+      : searchType && searchType.includes("content") && useTracesAmtCompatMode
+        ? `finalizeAggregation(${prefix}input) ILIKE {searchString: String} OR finalizeAggregation(${prefix}output) ILIKE {searchString: String}`
+        : null,
   ].filter(Boolean);
 
   return {

packages/shared/src/server/queries/createSessionsAllQuery.ts

@@ -1,88 +0,0 @@
-import { z } from "zod/v4";
-
-import { Prisma } from "@prisma/client";
-import { tableColumnsToSqlFilterAndPrefix } from "../filterToPrisma";
-import { singleFilter } from "../../interfaces/filters";
-import { orderBy } from "../../interfaces/orderBy";
-import { orderByToPrismaSql } from "../orderByToPrisma";
-import { sessionsViewCols } from "../../tableDefinitions";
-
-const GetSessionTableSQLParamsSchema = z.object({
-  projectId: z.string(),
-  filter: z.array(singleFilter).nullable(),
-  orderBy: orderBy,
-  page: z.number(),
-  limit: z.number(),
-});
-type GetSessionTableSQLParams = z.infer<typeof GetSessionTableSQLParamsSchema>;
-
-export const createSessionsAllQuery = (
-  select: Prisma.Sql,
-  params: GetSessionTableSQLParams,
-  options?: {
-    ignoreOrderBy?: boolean; // used by session.metrics and session.all.totalCount
-    sessionIdList?: string[]; // used by session.metrics
-  },
-): Prisma.Sql => {
-  const { projectId, filter, orderBy, page, limit } =
-    GetSessionTableSQLParamsSchema.parse(params);
-
-  const filterCondition = tableColumnsToSqlFilterAndPrefix(
-    filter ?? [],
-    sessionsViewCols,
-    "sessions",
-  );
-  const orderByCondition = orderByToPrismaSql(orderBy, sessionsViewCols);
-
-  const sessionIdFilter = options?.sessionIdList
-    ? Prisma.sql`AND s.id IN (${Prisma.join(options?.sessionIdList)})`
-    : Prisma.sql``;
-
-  const sql = Prisma.sql`
-      SELECT
-        ${select}
-      FROM
-        trace_sessions AS s
-        LEFT JOIN LATERAL (
-          SELECT
-            t.session_id,
-            MAX(t. "timestamp") AS "max_timestamp",
-            MIN(t. "timestamp") AS "min_timestamp",
-            array_agg(t.id) AS "traceIds",
-            array_agg(DISTINCT t.user_id) AS "userIds",
-            count(t.id)::int AS "countTraces",
-            array_agg(DISTINCT u.tag) AS "tags"
-          FROM
-            traces t
-          LEFT JOIN LATERAL (
-            SELECT DISTINCT UNNEST(t.tags) AS tag) AS u ON TRUE
-          WHERE
-            t.project_id = ${projectId}
-            AND t.session_id = s.id
-          GROUP BY
-            t.session_id) AS t ON TRUE
-        LEFT JOIN LATERAL (
-          SELECT
-            EXTRACT(EPOCH FROM COALESCE(MAX(o. "end_time"), MAX(o. "start_time"), t. "max_timestamp")) - EXTRACT(EPOCH FROM COALESCE(MIN(o. "start_time"), t. "min_timestamp"))::double precision AS "sessionDuration",
-            SUM(COALESCE(o. "calculated_input_cost", 0)) AS "inputCost",
-            SUM(COALESCE(o. "calculated_output_cost", 0)) AS "outputCost",
-            SUM(COALESCE(o. "calculated_total_cost", 0)) AS "totalCost",
-            SUM(o.prompt_tokens) AS "promptTokens",
-            SUM(o.completion_tokens) AS "completionTokens",
-            SUM(o.total_tokens) AS "totalTokens"
-          FROM
-            observations_view o
-          WHERE
-            o.project_id = ${projectId}
-            AND o.trace_id = ANY (t. "traceIds")) AS o ON TRUE
-      WHERE
-        s. "project_id" = ${projectId}
-        ${filterCondition}
-        ${sessionIdFilter}
-      ${options?.ignoreOrderBy ? Prisma.sql`` : orderByCondition}
-      LIMIT ${limit}
-      OFFSET ${page * limit}
-    `;
-
-  return sql;
-};

packages/shared/src/server/queries/index.ts

@@ -1,4 +1,3 @@
-export { createSessionsAllQuery } from "./createSessionsAllQuery";
 export {
   type FullObservations,
   type FullObservationsWithScores,

packages/shared/src/server/queues.ts

@@ -40,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -101,6 +116,24 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
+    z.object({
+      actionId: z.literal("observation-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -144,6 +177,7 @@ export const WebhookInputSchema = z.object({
   payload: WebhookOutboundEnvelopeSchema,
 });
 
+export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   z.object({
     entityType: z.literal("prompt-version"),
@@ -154,8 +188,6 @@ export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
   }),
   // Add other entity types here in the future
 ]);
-
-export type WebhookInput = z.infer<typeof WebhookInputSchema>;
 export type EntityChangeEventType = z.infer<typeof EntityChangeEventSchema>;
 
 export type CreateEvalQueueEventType = z.infer<
@@ -165,6 +197,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -192,6 +225,13 @@ export type DeadLetterRetryQueueEventType = z.infer<
 
 export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
 
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -214,6 +254,7 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
   WebhookQueue = "webhook-queue",
   EntityChangeQueue = "entity-change-queue",
@@ -241,6 +282,7 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
   WebhookJob = "webhook-job",
   EntityChangeJob = "entity-change-job",
@@ -265,6 +307,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -282,6 +330,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -306,6 +355,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/experimentCreateQueue.ts

@@ -34,7 +34,7 @@ export class ExperimentCreateQueue {
               attempts: 10,
               backoff: {
                 type: "exponential",
-                delay: 1000,
+                delay: 10_000, // 10 seconds
               },
             },
           },

packages/shared/src/server/redis/getQueue.ts

@@ -6,7 +6,6 @@ import { DatasetRunItemUpsertQueue } from "./datasetRunItemUpsert";
 import { EvalExecutionQueue } from "./evalExecutionQueue";
 import { ExperimentCreateQueue } from "./experimentCreateQueue";
 import { SecondaryIngestionQueue } from "./ingestionQueue";
-import { TraceUpsertQueue } from "./traceUpsert";
 import { TraceDeleteQueue } from "./traceDelete";
 import { ProjectDeleteQueue } from "./projectDelete";
 import { PostHogIntegrationQueue } from "./postHogIntegrationQueue";
@@ -23,11 +22,15 @@ import { ScoreDeleteQueue } from "./scoreDelete";
 import { DeadLetterRetryQueue } from "./dlqRetryQueue";
 import { WebhookQueue } from "./webhookQueue";
 import { EntityChangeQueue } from "./entityChangeQueue";
+import { DatasetDeleteQueue } from "./datasetDelete";
 
-// IngestionQueue is sharded and requires a sharding key
-// Use IngestionQueue.getInstance({ shardName: queueName }) directly instead
+// IngestionQueue and TraceUpsert are sharded and require a sharding key
+// Use IngestionQueue.getInstance({ shardName: queueName }) or TraceUpsertQueue.getInstance({ shardName: queueName }) directly instead
 export function getQueue(
-  queueName: Exclude<QueueName, QueueName.IngestionQueue>,
+  queueName: Exclude<
+    QueueName,
+    QueueName.IngestionQueue | QueueName.TraceUpsert
+  >,
 ): Queue | null {
   switch (queueName) {
     case QueueName.BatchExport:
@@ -36,12 +39,12 @@ export function getQueue(
       return CloudUsageMeteringQueue.getInstance();
     case QueueName.DatasetRunItemUpsert:
       return DatasetRunItemUpsertQueue.getInstance();
+    case QueueName.DatasetDelete:
+      return DatasetDeleteQueue.getInstance();
     case QueueName.EvaluationExecution:
       return EvalExecutionQueue.getInstance();
     case QueueName.ExperimentCreate:
       return ExperimentCreateQueue.getInstance();
-    case QueueName.TraceUpsert:
-      return TraceUpsertQueue.getInstance();
     case QueueName.TraceDelete:
       return TraceDeleteQueue.getInstance();
     case QueueName.ProjectDelete:

packages/shared/src/server/redis/redis.ts

@@ -6,6 +6,7 @@ import { logger } from "../logger";
 const defaultRedisOptions: Partial<RedisOptions> = {
   maxRetriesPerRequest: null,
   enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
+  keyPrefix: env.REDIS_KEY_PREFIX ?? undefined,
 };
 
 export const redisQueueRetryOptions: Partial<RedisOptions> = {
@@ -76,6 +77,7 @@ const createRedisClusterInstance = (
       callback(null, address);
     },
     redisOptions: {
+      username: env.REDIS_USERNAME || undefined,
       password: env.REDIS_AUTH || undefined,
       ...defaultRedisOptions,
       ...additionalOptions,
@@ -128,6 +130,7 @@ export const createNewRedisInstance = (
       ? new Redis({
           host: String(env.REDIS_HOST),
           port: Number(env.REDIS_PORT),
+          username: env.REDIS_USERNAME || undefined,
           password: String(env.REDIS_AUTH),
           ...defaultRedisOptions,
           ...additionalOptions,
@@ -156,6 +159,24 @@ export const getQueuePrefix = (queueName: string): string | undefined => {
   return undefined;
 };
 
+/**
+ * Execute multiple Redis DEL operations safely in cluster mode
+ */
+export const safeMultiDel = async (
+  redis: Redis | Cluster | null,
+  keys: string[],
+): Promise<void> => {
+  if (!redis || keys.length === 0) return;
+
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // In cluster mode, delete keys in separate commands to avoid CROSSSLOT errors
+    await Promise.all(keys.map(async (key: string) => redis.del(key)));
+  } else {
+    // In single-node mode, can delete all keys at once
+    await redis.del(keys);
+  }
+};
+
 const createRedisClient = () => {
   try {
     return createNewRedisInstance();

packages/shared/src/server/redis/traceUpsert.ts

@@ -6,45 +6,92 @@ import {
   getQueuePrefix,
 } from "./redis";
 import { logger } from "../logger";
+import { getShardIndex } from "./sharding";
+import { env } from "../../env";
 
 export class TraceUpsertQueue {
-  private static instance: Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null =
-    null;
+  private static instances: Map<
+    number,
+    Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null
+  > = new Map();
 
-  public static getInstance(): Queue<
-    TQueueJobTypes[QueueName.TraceUpsert]
-  > | null {
-    if (TraceUpsertQueue.instance) return TraceUpsertQueue.instance;
+  public static getShardNames() {
+    return Array.from(
+      { length: env.LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT },
+      (_, i) => `${QueueName.TraceUpsert}${i > 0 ? `-${i}` : ""}`,
+    );
+  }
+
+  static getShardIndexFromShardName(
+    shardName: string | undefined,
+  ): number | null {
+    if (!shardName) return null;
+
+    // Extract shard index from shard name
+    const shardIndex =
+      shardName === QueueName.TraceUpsert
+        ? 0
+        : parseInt(shardName.replace(`${QueueName.TraceUpsert}-`, ""), 10);
+
+    if (isNaN(shardIndex)) return null;
+    return shardIndex;
+  }
+
+  /**
+   * Get the trace upsert queue instance for the given sharding key or shard name.
+   * @param shardingKey - ShardingKey is being hashed and randomly allocated to a shard. Should be `projectId-traceId`.
+   * @param shardName - Name of the shard. Should be `trace-upsert-queue-${shardIndex}` or plainly `trace-upsert-queue` for the first shard.
+   */
+  public static getInstance({
+    shardingKey,
+    shardName,
+  }: {
+    shardingKey?: string;
+    shardName?: string;
+  } = {}): Queue<TQueueJobTypes[QueueName.TraceUpsert]> | null {
+    const shardIndex =
+      TraceUpsertQueue.getShardIndexFromShardName(shardName) ??
+      (env.REDIS_CLUSTER_ENABLED === "true" && shardingKey
+        ? getShardIndex(
+            shardingKey,
+            env.LANGFUSE_TRACE_UPSERT_QUEUE_SHARD_COUNT,
+          )
+        : 0);
+
+    // Check if we already have an instance for this shard
+    if (TraceUpsertQueue.instances.has(shardIndex)) {
+      return TraceUpsertQueue.instances.get(shardIndex) || null;
+    }
 
     const newRedis = createNewRedisInstance({
       enableOfflineQueue: false,
       ...redisQueueRetryOptions,
     });
 
-    TraceUpsertQueue.instance = newRedis
-      ? new Queue<TQueueJobTypes[QueueName.TraceUpsert]>(
-          QueueName.TraceUpsert,
-          {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.TraceUpsert),
-            defaultJobOptions: {
-              removeOnComplete: 100, // Important: If not true, new jobs for that ID would be ignored as jobs in the complete set are still considered as part of the queue
-              removeOnFail: 100_000,
-              attempts: 5,
-              delay: 15_000, // 15 seconds
-              backoff: {
-                type: "exponential",
-                delay: 5000,
-              },
+    const name = `${QueueName.TraceUpsert}${shardIndex > 0 ? `-${shardIndex}` : ""}`;
+    const queueInstance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.TraceUpsert]>(name, {
+          connection: newRedis,
+          prefix: getQueuePrefix(name),
+          defaultJobOptions: {
+            removeOnComplete: 100,
+            removeOnFail: 100_000,
+            attempts: 5,
+            delay: 15_000, // 15 seconds
+            backoff: {
+              type: "exponential",
+              delay: 5000,
             },
           },
-        )
+        })
       : null;
 
-    TraceUpsertQueue.instance?.on("error", (err) => {
-      logger.error("TraceUpsertQueue error", err);
+    queueInstance?.on("error", (err) => {
+      logger.error(`TraceUpsertQueue shard ${shardIndex} error`, err);
     });
 
-    return TraceUpsertQueue.instance;
+    TraceUpsertQueue.instances.set(shardIndex, queueInstance);
+
+    return queueInstance;
   }
 }

packages/shared/src/server/redis/webhookQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions } from "./redis";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
 import { logger } from "../logger";
 
 export class WebhookQueue {
@@ -23,6 +27,7 @@ export class WebhookQueue {
           QueueName.WebhookQueue,
           {
             connection: newRedis,
+            prefix: getQueuePrefix(QueueName.WebhookQueue),
             defaultJobOptions: {
               removeOnComplete: true,
               removeOnFail: 100_000,

packages/shared/src/server/repositories/automation-repository.ts

@@ -2,19 +2,25 @@ import {
   Action,
   ActionExecutionStatus,
   JobConfigState,
+  Prisma,
   prisma,
   Trigger,
 } from "../../db";
 import {
   TriggerEventSource,
-  WebhookActionConfigWithSecrets,
   TriggerDomain,
   TriggerEventAction,
   ActionDomain,
   AutomationDomain,
-  SafeWebhookActionConfig,
+  ActionDomainWithSecrets,
+  SafeActionConfig,
+  isWebhookActionConfig,
+  WebhookActionConfigWithSecrets,
+  isSafeWebhookActionConfig,
+  convertToSafeWebhookConfig,
 } from "../../domain/automations";
 import { FilterState } from "../../types";
+import { decryptSecretHeaders, mergeHeaders } from "../utils/headerUtils";
 
 export const getActionByIdWithSecrets = async ({
   projectId,
@@ -22,7 +28,7 @@ export const getActionByIdWithSecrets = async ({
 }: {
   projectId: string;
   actionId: string;
-}) => {
+}): Promise<ActionDomainWithSecrets | null> => {
   const actionConfig = await prisma.action.findFirst({
     where: {
       id: actionId,
@@ -34,18 +40,41 @@ export const getActionByIdWithSecrets = async ({
     return null;
   }
 
-  const config = actionConfig.config as WebhookActionConfigWithSecrets;
-  return {
-    ...actionConfig,
-    config: {
-      type: config.type,
-      url: config.url,
-      headers: config.headers,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-      secretKey: config.secretKey,
-    },
-  };
+  if (isWebhookActionConfig(actionConfig.config)) {
+    const config = actionConfig.config; // Type guard ensures this is WebhookActionConfigWithSecrets
+
+    // Decrypt secret headers for webhook execution using new structure
+    const decryptedHeaders = config.requestHeaders
+      ? decryptSecretHeaders(
+          mergeHeaders(config.headers, config.requestHeaders),
+        )
+      : config.headers
+        ? Object.entries(config.headers).reduce(
+            (acc, [key, value]) => {
+              acc[key] = { secret: false, value };
+              return acc;
+            },
+            {} as Record<string, { secret: boolean; value: string }>,
+          )
+        : {};
+
+    return {
+      ...actionConfig,
+      config: {
+        type: config.type,
+        url: config.url,
+        requestHeaders: decryptedHeaders,
+        displayHeaders: getDisplayHeaders(config),
+        apiVersion: config.apiVersion,
+        displaySecretKey: config.displaySecretKey,
+        secretKey: config.secretKey,
+        lastFailingExecutionId: config.lastFailingExecutionId,
+      },
+    };
+  }
+
+  // For SLACK and others, return as stored (already safe)
+  return actionConfig as ActionDomainWithSecrets;
 };
 
 export const getActionById = async ({
@@ -114,18 +143,37 @@ const convertTriggerToDomain = (trigger: Trigger): TriggerDomain => {
   };
 };
 
+const getDisplayHeaders = (config: WebhookActionConfigWithSecrets) => {
+  let displayHeaders = config.displayHeaders;
+  if (!displayHeaders && config.headers) {
+    // Convert legacy headers to displayHeaders format
+    displayHeaders = Object.entries(config.headers).reduce(
+      (acc, [key, value]) => {
+        acc[key] = { secret: false, value };
+        return acc;
+      },
+      {} as Record<string, { secret: boolean; value: string }>,
+    );
+  }
+  return displayHeaders;
+};
+
 const convertActionToDomain = (action: Action): ActionDomain => {
-  const config = action.config as WebhookActionConfigWithSecrets;
+  if (isWebhookActionConfig(action.config)) {
+    const config = action.config;
+    config.displayHeaders = getDisplayHeaders(config);
+
+    return {
+      ...action,
+      config: convertToSafeWebhookConfig(config),
+    };
+  }
+
+  // For SLACK (or future types) return config as-is
   return {
     ...action,
-    config: {
-      type: config.type,
-      url: config.url,
-      headers: config.headers,
-      apiVersion: config.apiVersion,
-      displaySecretKey: config.displaySecretKey,
-    } as SafeWebhookActionConfig,
-  };
+    config: action.config as SafeActionConfig,
+  } as ActionDomain;
 };
 
 export const getAutomationById = async ({
@@ -197,28 +245,49 @@ export const getConsecutiveAutomationFailures = async ({
   automationId: string;
   projectId: string;
 }): Promise<number> => {
-  // First get the automation to extract triggerId and actionId
-  const automation = await prisma.automation.findFirst({
-    where: {
-      id: automationId,
-      projectId,
-    },
+  const automation = await getAutomationById({
+    automationId,
+    projectId,
   });
 
   if (!automation) {
     return 0;
   }
 
-  const { triggerId, actionId } = automation;
-  const executions = await prisma.automationExecution.findMany({
-    where: {
-      triggerId,
-      actionId,
-      projectId,
-      status: {
-        in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
-      },
+  // Build where clause - if lastFailingExecutionId is set, only consider executions newer than it
+  const whereClause: Prisma.AutomationExecutionWhereInput = {
+    triggerId: automation.trigger.id,
+    actionId: automation.action.id,
+    projectId,
+    status: {
+      in: [ActionExecutionStatus.ERROR, ActionExecutionStatus.COMPLETED],
     },
+  };
+
+  // If there's a lastFailingExecutionId, we need to get executions that are newer than that execution
+  if (
+    isSafeWebhookActionConfig(automation.action.config) &&
+    automation.action.config.lastFailingExecutionId
+  ) {
+    // First get the timestamp of the last failing execution
+    const lastFailingExecution = await prisma.automationExecution.findUnique({
+      where: {
+        id: automation.action.config.lastFailingExecutionId,
+      },
+      select: {
+        createdAt: true,
+      },
+    });
+
+    if (lastFailingExecution) {
+      whereClause.createdAt = {
+        gt: lastFailingExecution.createdAt,
+      };
+    }
+  }
+
+  const executions = await prisma.automationExecution.findMany({
+    where: whereClause,
     orderBy: {
       createdAt: "desc",
     },

packages/shared/src/server/repositories/clickhouse.ts

@@ -36,7 +36,7 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
 export async function upsertClickhouse<
   T extends Record<string, unknown>,
 >(opts: {
-  table: "scores" | "traces" | "observations";
+  table: "scores" | "traces" | "observations" | "traces_null";
   records: T[];
   eventBodyMapper: (body: T) => Record<string, unknown>; // eslint-disable-line no-unused-vars
   tags?: Record<string, string>;