chore: release v3.87.1

* fix(trace-view): better markdown spacing for density

* better spacing

.claude/settings.json

@@ -0,0 +1,14 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(find:*)",
+      "Bash(rg:*)",
+      "Bash(grep:*)",
+      "Bash(ls:*)",
+      "Bash(cat:*)",
+      "Bash(head:*)",
+      "Bash(tail:*)"
+    ],
+    "deny": []
+  }
+}

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = .git,*.pdf,*.svg,package-lock.json,*.prisma,pnpm-lock.yaml
-ignore-words-list = afterall,vertx,notIn
+ignore-words-list = afterall,vertx,notIn,alue
 

.cursor/Dockerfile

@@ -30,7 +30,7 @@ RUN corepack enable && \
 
 # ---------- golang-migrate ----------------------------------------------------
 # CLI used for database migrations during development.
-ENV MIGRATE_VERSION=4.18.2
+ENV MIGRATE_VERSION=4.18.3
 RUN wget -qO- "https://github.com/golang-migrate/migrate/releases/download/v${MIGRATE_VERSION}/migrate.linux-amd64.tar.gz" \
     | tar -xz -C /usr/local/bin && \
     chmod +x /usr/local/bin/migrate

.cursor/rules/global.mdc

@@ -0,0 +1,20 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+## Project setup
+- This project is a Turborepo monorepo
+- We build two containers, web (./web) and worker (./worker)
+- We have shared code between these two in the shared package (./packages/shared). For that package, we have different entry points [package.json](mdc:packages/shared/package.json).
+
+
+## Domain layer
+The most important domain objects are in [observations.ts](mdc:packages/shared/src/domain/observations.ts), [traces.ts](mdc:packages/shared/src/domain/traces.ts), [scores.ts](mdc:packages/shared/src/domain/scores.ts).
+
+
+## Database schema
+We use Postgres and Clickhouse.
+- The postgres schema is in [schema.prisma](mdc:packages/shared/prisma/schema.prisma)
+- The clickhouse schema is in [0001_traces.up.sql](mdc:packages/shared/clickhouse/migrations/clustered/0001_traces.up.sql), [0002_observations.up.sql](mdc:packages/shared/clickhouse/migrations/clustered/0002_observations.up.sql), [0003_scores.up.sql](mdc:packages/shared/clickhouse/migrations/clustered/0003_scores.up.sql)
+

.cursor/rules/public-api.mdc

@@ -1,6 +1,6 @@
 ---
 description: How to create new public api routes for Langfuse
-globs: 
+globs:
 alwaysApply: false
 ---
 
@@ -13,4 +13,5 @@ alwaysApply: false
   - Add end-to-end test, similar to [datasets-api.servertest.ts](mdc:web/src/__tests__/async/datasets-api.servertest.ts)
   - Add fern configuration in /fern, learn more about structure here: https://buildwithfern.com/learn/api-definition/fern/overview
   - Prompt user to regenerate the OpenAPI spec via the fern CLI
-  - Pagination starts at 1, query typing defined in publicApiPaginationZod, return meta in paginationMetaResponseZod
+  - Pagination starts at 1, query typing defined in publicApiPaginationZod, return meta in paginationMetaResponseZod
+- For tests, please look at [this directory](mdc:web/src/__tests__/async/) for examples

.cursor/rules/tests.mdc

@@ -0,0 +1,9 @@
+---
+description: 
+globs: 
+alwaysApply: true
+---
+# Writing tests
+
+- When writing tests, focus on decoupling each `it` or `test` block to ensure that they can run independently and concurrently. Tests must never depend on the action or outcome of previous or subsequent tests.
+- When writing tests, especially in the __tests__/async directory, ensure that you avoid `pruneDatabase` calls.

.devcontainer.json

@@ -1,6 +1,6 @@
 {
-    "name": "langfuse-development",
-    "forwardPorts": [3000, 5432, 6379, 8123, 9000],
-    "onCreateCommand": "npm install -g pnpm@9.5.0",
-    "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
+  "name": "langfuse-development",
+  "forwardPorts": [3000, 5432, 6379, 8123, 9000],
+  "onCreateCommand": "npm install -g pnpm@9.5.0",
+  "postCreateCommand": "curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz && git restore LICENSE README.md && chmod +x migrate && sudo mv migrate /usr/bin && cp .env.dev.example .env && npm install -g @anthropic-ai/claude-code && pnpm i"
 }

.env.test.example

@@ -0,0 +1,12 @@
+# Test Environment Configuration
+# Copy this file to .env.test for test database isolation
+# Only overrides specific test variables - other values inherited from .env
+
+# PostgreSQL - Test Database
+DIRECT_URL="postgresql://postgres:postgres@localhost:5432/langfuse_test"
+DATABASE_URL="postgresql://postgres:postgres@localhost:5432/langfuse_test"
+
+# ClickHouse - Use Default Database for now, nothing set
+
+# Redis - Test Database (database 1 for isolation)
+REDIS_CONNECTION_STRING="redis://:myredissecret@127.0.0.1:6379/1"

.github/PULL_REQUEST_TEMPLATE.md

@@ -28,7 +28,7 @@ Fixes # (issue)
 <!-- Remove bullet points below that don't apply to you -->
 
 - I haven't read the [contributing guide](https://github.com/langfuse/langfuse/blob/main/CONTRIBUTING.md)
-- My code doesn't follow the style guidelines of this project (`npm run prettier`)
+- My code doesn't follow the style guidelines of this project (`pnpm run format`)
 - I haven't commented my code, particularly in hard-to-understand areas
 - I haven't checked if my PR needs changes to the documentation
 - I haven't checked if my changes generate no new warnings (`npm run lint`)

.github/dependabot.yml

@@ -36,6 +36,7 @@ updates:
         patterns:
           - "express"
           - "@types/express"
+          - "@types/express-serve-static-core"
       observability:
         patterns:
           - "dd-trace"

.github/workflows/_deploy_ecs_service.yml

@@ -15,10 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - name: Get app name
         uses: winterjung/split@v2
         id: split

.github/workflows/pipeline.yml

@@ -52,6 +52,49 @@ jobs:
       - name: lint web
         run: pnpm run lint
 
+  prettier-check:
+    runs-on: ubuntu-latest
+    needs:
+      - pre-job
+    if: needs.pre-job.outputs.should_skip != 'true'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: pnpm/action-setup@v3
+        with:
+          version: 9.5.0
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: "pnpm-lock.yaml"
+      - name: install dependencies
+        run: |
+          pnpm i
+      - name: Load default env
+        run: |
+          cp .env.dev.example .env
+      - name: Check formatting on changed files
+        run: |
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            BASE_SHA=${{ github.event.pull_request.base.sha }}
+          else
+            BASE_SHA=$(git merge-base origin/main HEAD)
+          fi
+          
+          echo "Checking files changed from $BASE_SHA to HEAD"
+          
+          # Get changed files
+          CHANGED_FILES=$(git diff --name-only $BASE_SHA HEAD -- '*.js' '*.jsx' '*.ts' '*.tsx' '*.css' | tr '\n' ' ')
+          
+          if [ -n "$CHANGED_FILES" ] && [ "$CHANGED_FILES" != " " ]; then
+            echo "Files to check: $CHANGED_FILES"
+            pnpm prettier --check --experimental-cli $CHANGED_FILES
+          else
+            echo "No JS/TS/CSS files changed - skipping prettier check"
+          fi
+
   test-docker-build:
     timeout-minutes: 20
     runs-on: ubuntu-latest
@@ -100,14 +143,10 @@ jobs:
         node-version: [20]
         postgres-version: [12, 15]
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -133,6 +172,7 @@ jobs:
           cp .env.dev.example .env
           grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' .env.dev.example > .env
           echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1" >> .env
+          echo "LANGFUSE_CACHE_PROMPT_ENABLED=false" >> .env
           echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1" >> .env
       - name: Run dev containers
         run: |
@@ -178,14 +218,10 @@ jobs:
         postgres-version: [12, 15]
         deploy-mode: ["", "-azure", "-redis-cluster"]
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - uses: actions/checkout@v4
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - uses: pnpm/action-setup@v3
@@ -257,10 +293,6 @@ jobs:
         postgres-version: [12, 15]
         deploy-mode: ["", "-azure", "-redis-cluster"]
     steps:
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 10
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
         with:
@@ -282,7 +314,7 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
@@ -343,7 +375,7 @@ jobs:
           cp .env.dev.example web/.env
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Run + migrate
@@ -397,14 +429,12 @@ jobs:
           pnpm install
       - name: Install golang-migrate for Clickhouse migrations
         run: |
-          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.2/migrate.linux-amd64.tar.gz | tar xvz
+          curl -L https://github.com/golang-migrate/migrate/releases/download/v4.18.3/migrate.linux-amd64.tar.gz | tar xvz
           sudo mv migrate /usr/bin/migrate
           which migrate
       - name: Load default env
         run: |
           cp .env.dev.example .env
-          echo "LANGFUSE_CACHE_API_KEY_ENABLED=true" >> .env
-          echo "LANGFUSE_CACHE_PROMPT_ENABLED=true" >> .env
       - name: Run + migrate
         run: |
           docker compose -f docker-compose.dev.yml up -d
@@ -442,6 +472,7 @@ jobs:
     needs:
       [
         lint,
+        prettier-check,
         tests-web-sync,
         tests-worker,
         e2e-tests,
@@ -452,11 +483,11 @@ jobs:
     if: always()
     steps:
       - name: Successful deploy
-        if: ${{ !(contains(needs.*.result, 'failure')) }}
+        if: ${{ !(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
         run: exit 0
         working-directory: .
       - name: Failing deploy
-        if: ${{ contains(needs.*.result, 'failure') }}
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
         run: exit 1
         working-directory: .
 

.github/workflows/snyk.yml

@@ -1,65 +1,28 @@
 name: Snyk Container
 on:
-  pull_request:
-    branches:
-      - "**"
   push:
-    branches:
-      - main
-  # Snyk cannot upload results in merge group. Hence, we only run on PRs and when pushingon the main branch https://github.com/github/codeql-action/issues/1572
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+    branches: ["main"]
 
 jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Build a Docker image
-        run: docker compose -f docker-compose.build.yml up -d
+      - uses: actions/checkout@v3
 
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-web)
-        continue-on-error: true
+      - name: Scan web image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-web
-          args: --file=web/Dockerfile
+          image: langfuse/langfuse # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)
 
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: web
-
-      - name: Run Snyk to check Docker image for vulnerabilities (langfuse-worker)
-        continue-on-error: true
+      - name: Scan worker image with Snyk
         uses: snyk/actions/docker@master
+        continue-on-error: true # let upload step always run
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: langfuse-langfuse-worker
-          args: --file=worker/Dockerfile
-
-      # Workaround for https://github.com/github/codeql-action/issues/2187
-      - name: Replace security-severity undefined for license-related findings
-        run: "sed -i 's/\"security-severity\": \"undefined\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Replace security-severity null for license-related findings
-        run: "sed -i 's/\"security-severity\": \"null\"/\"security-severity\": \"0\"/g' snyk.sarif"
-
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: snyk.sarif
-          category: worker
+          image: langfuse/langfuse-worker # pulled from Docker Hub
+          args: --severity-threshold=high # (any extra CLI flags, but NO --file)

.gitignore

@@ -42,6 +42,7 @@ yarn-error.log*
 !.env.dev-azure.example
 !.env.dev-redis-cluster.example
 !.env.prod.example
+!.env.test.example
 
 # vercel
 .vercel
@@ -66,4 +67,7 @@ node_modules
 .yarn
 .turbo
 
-web/test-results/*
+web/test-results/*
+
+# local config files
+*.local.*

.husky/pre-commit

@@ -11,12 +11,13 @@ if [ "$current_branch" = "$protected_branch" ]; then
     echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
     read -r answer < /dev/tty
     if [ "$answer" != "${answer#[Yy]}" ]; then
-        exit 0 # Commit will proceed
+        # Commit approved, check formatting. On files changed, block commit
+        pnpm run format:check
     else
         echo "Commit to $protected_branch branch has been canceled."
         exit 1 # Commit will be blocked
     fi
 fi
 
-# If not the protected branch, proceed with the commit
-exit 0
+# If not the protected branch, check formatting (on changed files, block)
+pnpm run format:check

.prettierignore

@@ -0,0 +1,2 @@
+# Generated code
+packages/shared/prisma/generated

AGENTS.md

@@ -8,6 +8,8 @@ Langfuse is an **open source LLM engineering** platform for developing, monitori
 
 ## Tests
 - Codex cannot run the test suite because it depends on Docker-based infrastructure that is unavailable in this environment.
+- When writing tests, focus on decoupling each `it` or `test` block to ensure that they can run independently and concurrently. Tests must never depend on the action or outcome of previous or subsequent tests.
+- When writing tests, especially in the __tests__/async directory, ensure that you avoid `pruneDatabase` calls.
 
 ## Cursor Rules
 - Additional folder-specific rules live in `.cursor/rules/`.

CLAUDE.md

@@ -94,6 +94,7 @@ pnpm run test --filter=worker -- $TEST_FILE_NAME -t "$TEST_NAME"
 
 ### Utilities
 ```bash
+pnpm run format            # Format code across entire project
 pnpm run nuke              # Remove all node_modules, build files, wipe database, docker containers. **USE WITH CAUTION**
 ```
 
@@ -152,6 +153,8 @@ pnpm run nuke              # Remove all node_modules, build files, wipe database
 - Jest for API tests, Playwright for E2E tests
 - For backend/API changes, tests must pass before pushes
 - Add tests for new API endpoints and features
+- When writing tests, focus on decoupling each `it` or `test` block to ensure that they can run independently and concurrently. Tests must never depend on the action or outcome of previous or subsequent tests.
+- When writing tests, especially in the __tests__/async directory, ensure that you avoid `pruneDatabase` calls.
 
 ### Code Conventions
 - **Pages Router** (not App Router)
@@ -186,3 +189,6 @@ To get a project, use the `get_project` capability with the full project name as
 
 ## TypeScript Best Practices
 - In TypeScript, if possible, don't use the `any` type
+
+## General Coding Guidelines
+- For easier code reviews, prefer not to move functions etc around within a file unless necessary or instructed to do so

CONTRIBUTING.md

@@ -124,16 +124,23 @@ Requirements
    cd langfuse
    ```
 
-3. Create an env file
+3. Install dependencies and set up pre-commit hooks
+
+   ```bash
+   pnpm install
+   pnpm run prepare  # Sets up Husky pre-commit hooks for code formatting
+   ```
+
+4. Create an env file
 
    ```bash
     cp .env.dev.example .env
    ```
 
-4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it.
+4. Run the entire infrastructure in dev mode. **Note**: if you have an existing database, this command wipes it. Also, this will fail on the very first run. Please run it again.
 
    ```bash
-   pnpm run dx # first run only (resets db, node_modules, ...)
+   pnpm run dx # first run only (resets db, docker containers, etc...)
    pnpm run dev # any subsequent runs
    ```
 
@@ -149,8 +156,8 @@ Requirements
    - Username: `demo@langfuse.com`
    - Password: `password`
 
-
 To get comprehensive example data, you can use the `seed` command:
+
 ```sh
 pnpm run db:seed:examples
 ```
@@ -209,31 +216,55 @@ On the main branch, we adhere to the best practices of [conventional commits](ht
 All tests run in the CI and must pass before merging.
 All tests run against a running langfuse instance and **write/delete real data from the database**.
 
+### Test Database Setup
+
+Per default, the tests use the local development database. Therefore, wiping your data in the process.
+For proper test isolation, create a `.env.test` file in the root directory:
+
+```bash
+cp .env.test.example .env.test
+```
+
+Then, a different PostgreSQL and Redis are used for the tests.
+The `.env.test` file only overrides the set values and falls back on `.env` for all undefined values.
+
+- **PostgreSQL**: Uses separate `langfuse_test` database for isolation
+- **ClickHouse**: Uses shared `default` database for now
+- **Redis**: Uses database 1 instead of 0 for isolation (Redis data is not cleaned between tests)
+
+Tests automatically create the PostgreSQL test database if it doesn't exist and clean up data between runs.
+
 ### Tests in the `web` package (public API)
-We're using Jest with in the `web` package. Therefore, if you want to provide an argument to the test runner, do it directly without an intermittent ` -- `.
+
+We're using Jest with in the `web` package. Therefore, if you want to provide an argument to the test runner, do it directly without an intermittent `--`.
 
 There are three types of unit tests:
+
 - `test-sync`
 - `test-async`
 - `test-client`
 
 To run a specific test, for example the test: `"should handle special characters in prompt names"` in `prompts.v2.servertest.ts`, run:
+
 ```sh
 cd web  # or with --filter=web
 pnpm test-sync --testPathPattern="prompts\.v2\.servertest" --testNamePattern="should handle special characters in prompt names"
 ```
 
 To run all tests:
+
 ```sh
 pnpm run test
 ```
 
 Run interactively in watch mode (not recommended!)
+
 ```sh
 pnpm run test:watch
 ```
 
 ### Tests in the `worker` package
+
 For the `worker` package, we're using `vitest` to run unit tests.
 
 ```sh

README.cn.md

@@ -94,7 +94,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
   
 - [提示管理](https://langfuse.com/docs/prompts/get-started) 帮助你集中管理、版本控制并协作迭代提示。得益于服务器和客户端的高效缓存，你可以在不增加延迟的情况下反复迭代提示。
 
-- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为“裁判”、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
+- [评估](https://langfuse.com/docs/scores/overview) 是 LLM 应用开发流程的关键组成部分，Langfuse 能够满足你的多样需求。它支持 LLM 作为"裁判"、用户反馈收集、手动标注以及通过 API/SDK 实现自定义评估流程。
 
 - [数据集](https://langfuse.com/docs/datasets/overview) 为评估你的 LLM 应用提供测试集和基准。它们支持持续改进、部署前测试、结构化实验、灵活评估，并能与 LangChain、LlamaIndex 等框架无缝整合。
 
@@ -135,7 +135,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 
 - [虚拟机](https://langfuse.com/self-hosting/docker-compose)：使用 Docker Compose 在单台虚拟机上部署 Langfuse。
 
-- 【计划中】：针对各云平台的部署指南，欢迎在以下讨论中投票和评论：[AWS](https://github.com/orgs/langfuse/discussions/4645)、[Google Cloud](https://github.com/langfuse/discussions/4646)、[Azure](https://github.com/orgs/langfuse/discussions/4647)。
+- Terraform 模板: [AWS](https://langfuse.com/self-hosting/aws)、[Azure](https://langfuse.com/self-hosting/azure)、[GCP](https://langfuse.com/self-hosting/gcp)
 
 请参阅 [自托管文档](https://langfuse.com/self-hosting) 了解更多关于架构和配置选项的信息。
 
@@ -155,6 +155,7 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | [LiteLLM](https://langfuse.com/docs/integrations/litellm)               | Python, JS/TS (仅代理)     | 允许使用任何 LLM 替代 GPT。支持 Azure、OpenAI、Cohere、Anthropic、Ollama、VLLM、Sagemaker、HuggingFace、Replicate（100+ LLMs）。         |
 | [Vercel AI SDK](https://langfuse.com/docs/integrations/vercel-ai-sdk)     | JS/TS                    | 基于 TypeScript 的工具包，帮助开发者使用 React、Next.js、Vue、Svelte 和 Node.js 构建 AI 驱动的应用。                                           |
 | [API](https://langfuse.com/docs/api)                                    |                          | 直接调用公共 API。提供 OpenAPI 规格。                                                                                                       |
+| [Google VertexAI 和 Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 模型                | 在 Google 上运行基础模型和微调模型。                                                                         |
 
 ### 与 Langfuse 集成的软件包：
 
@@ -162,9 +163,9 @@ Langfuse 是一个 **开源 LLM 工程** 平台。它帮助团队协作 **开发
 | ---------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------- |
 | [Instructor](https://langfuse.com/docs/integrations/instructor)         | 库                  | 用于获取结构化 LLM 输出（JSON、Pydantic）的库。                                                         |
 | [DSPy](https://langfuse.com/docs/integrations/dspy)                     | 库                  | 一个系统性优化语言模型提示和权重的框架。                                                                 |
+| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 库                  | 构建 LLM 应用的 Python 工具包。                                                                           |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 模型（本地）         | 在你的机器上轻松运行开源 LLM。                                                                            |
-| [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 模型                | 在 AWS 上运行基础模型和微调模型。                                                                         |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 代理框架             | 用于构建分布式代理的开源 LLM 平台。                                                                       |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 聊天/代理界面         | 基于 JS/TS 的无代码构建器，用于定制化 LLM 流程。                                                          |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 聊天/代理界面         | 基于 Python 的 LangChain 用户界面，采用 react-flow 设计，提供便捷的实验与原型构建体验。                   |
@@ -253,7 +254,7 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 
 - 我们的 [文档](https://langfuse.com/docs) 是查找答案的最佳起点。内容全面，我们投入大量时间进行维护。你也可以通过 GitHub 提出文档修改建议。
 - [Langfuse 常见问题](https://langfuse.com/faq) 解答了最常见的问题。
-- 使用 “[Ask AI](https://langfuse.com/docs/ask-ai)” 立即获取问题答案。
+- 使用 "Ask AI" 立即获取问题答案。
 
 支持渠道：
 
@@ -351,4 +352,4 @@ _[Langfuse 中的公共示例追踪](https://cloud.langfuse.com/project/cloramnk
 所有数据均不会与第三方共享，也不包含任何敏感信息。我们对这一过程保持高度透明，你可以在 [此处](/web/src/features/telemetry/index.ts) 查看我们收集的具体数据。
 
 你可以通过设置 `TELEMETRY_ENABLED=false` 来选择退出。
-````
+```

README.ja.md

@@ -141,9 +141,7 @@ Langfuseチームによるマネージドデプロイメント。充実した無
 - **[VM](https://langfuse.com/self-hosting/docker-compose):**  
   Docker Composeを使用して、単一の仮想マシン上でLangfuseを実行します。
 
-- **Planned:**  
-  クラウド固有のデプロイガイドは計画中です。以下のスレッドに対して投票やコメントをお願いします:  
-  [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform テンプレート: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 [セルフホスティングのドキュメント](https://langfuse.com/self-hosting)を参照し、アーキテクチャや設定オプションの詳細をご確認ください。
 

README.kr.md

@@ -128,7 +128,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 
 - [Kubernetes (Helm)](https://langfuse.com/self-hosting/kubernetes-helm): Helm을 사용해 Kubernetes 클러스터에서 Langfuse를 실행합니다. 이는 권장되는 프로덕션 배포 방식입니다.
 - [VM](https://langfuse.com/self-hosting/docker-compose): Docker Compose를 사용해 단일 가상 머신에서 Langfuse를 실행합니다.
-- 예정: 클라우드별 배포 가이드 – 아래 스레드에서 투표 및 댓글을 남겨주세요: [AWS](https://github.com/orgs/langfuse/discussions/4645), [Google Cloud](https://github.com/orgs/langfuse/discussions/4646), [Azure](https://github.com/orgs/langfuse/discussions/4647).
+- Terraform 템플릿: [AWS](https://langfuse.com/self-hosting/aws), [Azure](https://langfuse.com/self-hosting/azure), [GCP](https://langfuse.com/self-hosting/gcp)
 
 자세한 내용은 [자체 호스팅 문서](https://langfuse.com/self-hosting)를 참조하세요.
 
@@ -158,6 +158,7 @@ Langfuse 팀이 관리하는 배포 방식으로, 후한 무료 플랜(취미
 | [Mirascope](https://langfuse.com/docs/integrations/mirascope)           | 라이브러리         | LLM 애플리케이션 구축을 위한 Python 툴킷입니다.                                                                    |
 | [Ollama](https://langfuse.com/docs/integrations/ollama)                 | 모델 (로컬)        | 자신의 컴퓨터에서 오픈 소스 LLM을 손쉽게 실행할 수 있습니다.                                                      |
 | [Amazon Bedrock](https://langfuse.com/docs/integrations/amazon-bedrock) | 모델               | AWS에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
+| [Google VertexAI and Gemini](https://langfuse.com/docs/integrations/google-vertex-ai) | 모델               | Google에서 기본 및 파인튜닝된 모델을 실행합니다.                                                                      |
 | [AutoGen](https://langfuse.com/docs/integrations/autogen)               | 에이전트 프레임워크 | 분산 에이전트 구축을 위한 오픈 소스 LLM 플랫폼입니다.                                                              |
 | [Flowise](https://langfuse.com/docs/integrations/flowise)               | 채팅/에이전트 UI    | 맞춤형 LLM 플로우를 위한 JS/TS 코드 없는(no-code) 빌더입니다.                                                       |
 | [Langflow](https://langfuse.com/docs/integrations/langflow)             | 채팅/에이전트 UI    | react-flow를 활용하여 실험 및 프로토타이핑을 손쉽게 할 수 있도록 디자인된 LangChain용 Python 기반 UI입니다.          |

docker-compose.build.yml

@@ -78,7 +78,7 @@ services:
       retries: 3
 
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: docker.io/clickhouse/clickhouse-server
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
@@ -98,7 +98,7 @@ services:
       start_period: 1s
 
   minio:
-    image: minio/minio
+    image: docker.io/minio/minio
     entrypoint: sh
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
@@ -118,7 +118,7 @@ services:
       start_period: 1s
 
   redis:
-    image: redis:7
+    image: docker.io/redis:7
     restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
@@ -131,7 +131,7 @@ services:
       retries: 10
 
   postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

docker-compose.dev-azure.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:24.3
+    image: docker.io/clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
@@ -24,7 +24,7 @@ services:
       - langfuse_azurite_data:/data
 
   redis:
-    image: redis:7.2.4
+    image: docker.io/redis:7.2.4
     restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
@@ -32,7 +32,7 @@ services:
       - 6379:6379
 
   postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

docker-compose.dev-redis-cluster.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:24.3
+    image: docker.io/clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
@@ -16,7 +16,7 @@ services:
       - postgres
 
   minio:
-    image: minio/minio
+    image: docker.io/minio/minio
     entrypoint: sh
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
@@ -36,7 +36,7 @@ services:
       start_period: 1s
 
   postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

docker-compose.dev.yml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:24.3
+    image: docker.io/clickhouse/clickhouse-server:24.3
     user: "101:101"
     environment:
       CLICKHOUSE_DB: default
@@ -16,7 +16,7 @@ services:
       - postgres
 
   minio:
-    image: minio/minio
+    image: docker.io/minio/minio
     entrypoint: sh
     # create the 'langfuse' bucket before starting the service
     command: -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
@@ -36,7 +36,7 @@ services:
       start_period: 1s
 
   redis:
-    image: redis:7.2.4
+    image: docker.io/redis:7.2.4
     restart: always
     command: >
       --requirepass ${REDIS_AUTH:-myredissecret}
@@ -44,7 +44,7 @@ services:
       - 127.0.0.1:6379:6379
 
   postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

docker-compose.yml

@@ -5,7 +5,7 @@
 # External connections from other machines will not be able to reach these services directly.
 services:
   langfuse-worker:
-    image: langfuse/langfuse-worker:3
+    image: docker.io/langfuse/langfuse-worker:3
     restart: always
     depends_on: &langfuse-depends-on
       postgres:
@@ -62,9 +62,11 @@ services:
       REDIS_TLS_CA: ${REDIS_TLS_CA:-/certs/ca.crt}
       REDIS_TLS_CERT: ${REDIS_TLS_CERT:-/certs/redis.crt}
       REDIS_TLS_KEY: ${REDIS_TLS_KEY:-/certs/redis.key}
+      EMAIL_FROM_ADDRESS: ${EMAIL_FROM_ADDRESS:-}
+      SMTP_CONNECTION_URL: ${SMTP_CONNECTION_URL:-}
 
   langfuse-web:
-    image: langfuse/langfuse:3
+    image: docker.io/langfuse/langfuse:3
     restart: always
     depends_on: *langfuse-depends-on
     ports:
@@ -84,7 +86,7 @@ services:
       LANGFUSE_INIT_USER_PASSWORD: ${LANGFUSE_INIT_USER_PASSWORD:-}
 
   clickhouse:
-    image: clickhouse/clickhouse-server
+    image: docker.io/clickhouse/clickhouse-server
     restart: always
     user: "101:101"
     environment:
@@ -105,7 +107,7 @@ services:
       start_period: 1s
 
   minio:
-    image: minio/minio
+    image: docker.io/minio/minio
     restart: always
     entrypoint: sh
     # create the 'langfuse' bucket before starting the service
@@ -126,7 +128,7 @@ services:
       start_period: 1s
 
   redis:
-    image: redis:7
+    image: docker.io/redis:7
     restart: always
     # CHANGEME: row below to secure redis password
     command: >
@@ -140,7 +142,7 @@ services:
       retries: 10
 
   postgres:
-    image: postgres:${POSTGRES_VERSION:-latest}
+    image: docker.io/postgres:${POSTGRES_VERSION:-latest}
     restart: always
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]

ee/package.json

@@ -41,14 +41,9 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-config-standard": "^17.1.0",
     "eslint-plugin-prettier": "^5.1.3",
-    "prettier": "^3.3.3",
+    "prettier": "^3.6.2",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
     "typescript": "^5.4.5"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

fern/apis/server/definition/dataset-run-items.yml

@@ -27,7 +27,7 @@ service:
           limit:
             type: optional<integer>
             docs: limit of items per page
-          response: PaginatedDatasetRunItems
+      response: PaginatedDatasetRunItems
 
 types:
   CreateDatasetRunItemRequest:

fern/apis/server/definition/observations.yml

@@ -32,6 +32,9 @@ service:
           userId: optional<string>
           type: optional<string>
           traceId: optional<string>
+          level:
+            type: optional<commons.ObservationLevel>
+            docs: Optional filter for observations with a specific level (e.g. "DEBUG", "DEFAULT", "WARNING", "ERROR").
           parentObservationId: optional<string>
           environment:
             type: optional<string>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "langfuse",
-  "version": "3.78.2",
+  "version": "3.87.1",
   "author": "engineering@langfuse.com",
   "license": "MIT",
   "private": true,
@@ -17,9 +17,9 @@
     "db:seed": "turbo run db:seed",
     "db:seed:examples": "turbo run db:seed:examples",
     "nuke": "bash ./scripts/nuke.sh",
-    "dx": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx-f": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset -f && SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
-    "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset:test && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx-f": "pnpm i && pnpm run infra:dev:prune && pnpm run infra:dev:up --pull always && pnpm --filter=shared run db:reset:test && pnpm --filter=shared run db:reset -f && SKIP_CONFIRM=1 pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
+    "dx:skip-infra": "pnpm i && pnpm --filter=shared run db:reset:test && pnpm --filter=shared run db:reset && pnpm --filter=shared run ch:reset && pnpm --filter=shared run db:seed:examples && pnpm run dev",
     "build": "turbo run build",
     "start": "turbo run start",
     "dev": "turbo run dev",
@@ -27,6 +27,8 @@
     "dev:web": "turbo run dev --filter=web",
     "dev:web-turbo": "turbo run dev --filter=web -- --turbo",
     "lint": "turbo run lint",
+    "format": "prettier --write \"**/*.{js,jsx,ts,tsx,css}\" --experimental-cli",
+    "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx,css}\" --experimental-cli",
     "test": "turbo run test",
     "release": "dotenv -e ../.env -- release-it",
     "prepare": "husky"
@@ -36,9 +38,9 @@
     "braces": "3.0.3",
     "dotenv-cli": "^7.4.2",
     "husky": "^9.0.11",
-    "prettier": "^3.3.3",
+    "prettier": "^3.6.2",
     "release-it": "^19.0.3",
-    "turbo": "^2.5.4"
+    "turbo": "^2.5.5"
   },
   "release-it": {
     "git": {

packages/config-eslint/library.js

@@ -35,5 +35,13 @@ module.exports = {
     {
       files: ["*.js?(x)", "*.ts?(x)"],
     },
+    {
+      files: ["*.ts", "*.mts", "*.cts", "*.tsx"],
+      // no-undef doesn't make sense in TS, see:
+      // https://typescript-eslint.io/troubleshooting/faqs/eslint/#i-get-errors-from-the-no-undef-rule-about-global-variables-not-being-defined-even-though-there-are-no-typescript-errors
+      rules: {
+        "no-undef": "off",
+      },
+    },
   ],
 };

packages/config-eslint/package.json

@@ -13,7 +13,7 @@
     "@vercel/style-guide": "^6.0.0",
     "eslint-config-next": "^14.2.15",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^2.5.4",
+    "eslint-config-turbo": "^2.5.5",
     "eslint-plugin-only-warn": "^1.1.0",
     "typescript": "^5.4.5"
   }

packages/shared/clickhouse/migrations/unclustered/0006_add_user_id_index.down.sql

@@ -1 +1 @@
-ALTER TABLE traces ON CLUSTER default DROP INDEX IF EXISTS idx_user_id;
+ALTER TABLE traces DROP INDEX IF EXISTS idx_user_id;

packages/shared/clickhouse/scripts/up.sh

@@ -5,8 +5,30 @@
 
 # Check if CLICKHOUSE_URL is configured
 if [ -z "${CLICKHOUSE_URL}" ]; then
-  echo "Info: CLICKHOUSE_URL not configured, skipping migration."
-  exit 0
+  echo "Error: CLICKHOUSE_URL is not configured."
+  echo "Please set CLICKHOUSE_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_MIGRATION_URL is configured
+if [ -z "${CLICKHOUSE_MIGRATION_URL}" ]; then
+  echo "Error: CLICKHOUSE_MIGRATION_URL is not configured."
+  echo "Please set CLICKHOUSE_MIGRATION_URL in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_USER is set
+if [ -z "${CLICKHOUSE_USER}" ]; then
+  echo "Error: CLICKHOUSE_USER is not set."
+  echo "Please set CLICKHOUSE_USER in your environment variables."
+  exit 1
+fi
+
+# Check if CLICKHOUSE_PASSWORD is set
+if [ -z "${CLICKHOUSE_PASSWORD}" ]; then
+  echo "Error: CLICKHOUSE_PASSWORD is not set."
+  echo "Please set CLICKHOUSE_PASSWORD in your environment variables."
+  exit 1
 fi
 
 # Check if golang-migrate is installed

packages/shared/package.json

@@ -38,6 +38,7 @@
     "db:migrate": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma migrate dev",
     "db:push": "DISABLE_ERD=false dotenv -e ../../.env -- npx prisma db push",
     "db:reset": "dotenv -e ../../.env npx -- prisma migrate reset",
+    "db:reset:test": "if [ -f ../../.env.test ]; then dotenv -e ../../.env.test -e ../../.env -- npx prisma migrate reset --force; fi",
     "db:deploy": "dotenv -e ../../.env npx -- prisma migrate deploy",
     "db:seed": "dotenv -e ../../.env -- npx prisma db seed",
     "db:generate": "dotenv -e ../../.env -- npx prisma generate",
@@ -60,7 +61,7 @@
     "@aws-sdk/lib-storage": "^3.675.0",
     "@aws-sdk/s3-request-presigner": "^3.679.0",
     "@azure/storage-blob": "^12.26.0",
-    "@clickhouse/client": "^1.11.2",
+    "@clickhouse/client": "^1.12.0",
     "@google-cloud/storage": "^7.15.2",
     "@langchain/anthropic": "^0.3.22",
     "@langchain/aws": "^0.1.11",
@@ -78,13 +79,13 @@
     "bullmq": "^5.34.10",
     "dd-trace": "^5.36.0",
     "decimal.js": "^10.4.3",
-    "exponential-backoff": "^3.1.1",
+    "exponential-backoff": "^3.1.2",
     "https-proxy-agent": "^7.0.6",
     "ioredis": "^5.4.1",
     "jsonpath-plus": "10.3.0",
     "kysely": "^0.27.4",
     "langchain": "^0.3.28",
-    "langfuse-langchain": "3.38.1",
+    "langfuse-langchain": "3.38.4",
     "lodash": "^4.17.21",
     "lossless-json": "^4.1.1",
     "next-auth": "^4.24.11",
@@ -111,23 +112,16 @@
     "eslint-plugin-prettier": "^5.1.3",
     "kysely-codegen": "^0.16.8",
     "nodemon": "^3.1.7",
-    "prettier": "^3.3.3",
+    "prettier": "^3.6.2",
     "prisma": "^6.10.1",
     "prisma-erd-generator": "^1.11.2",
     "prisma-kysely": "^1.8.0",
     "ts-node": "^10.9.2",
     "tsc-watch": "^6.2.0",
-    "tsx": "^4.19.1",
-    "typescript": "^5.4.5",
-    "vitest": "^2.1.2"
+    "typescript": "^5.4.5"
   },
   "peerDependencies": {
     "@types/react": "~18.2.79",
     "react": "~18.2.0"
-  },
-  "pnpm": {
-    "overrides": {
-      "nanoid": "^3.3.8"
-    }
   }
 }

packages/shared/prisma/migrations/20250704170658_add_automations/migration.sql

@@ -0,0 +1,111 @@
+-- CreateEnum
+CREATE TYPE "ActionType" AS ENUM ('WEBHOOK');
+
+-- CreateEnum
+CREATE TYPE "ActionExecutionStatus" AS ENUM ('COMPLETED', 'ERROR', 'PENDING', 'CANCELLED');
+
+-- CreateTable
+CREATE TABLE "actions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "type" "ActionType" NOT NULL,
+    "config" JSONB NOT NULL,
+
+    CONSTRAINT "actions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "triggers" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+    "eventSource" TEXT NOT NULL,
+    "eventActions" TEXT[],
+    "filter" JSONB,
+    "status" "JobConfigState" NOT NULL DEFAULT 'ACTIVE',
+
+    CONSTRAINT "triggers_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "automations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "trigger_id" TEXT NOT NULL,
+    "action_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "project_id" TEXT NOT NULL,
+
+    CONSTRAINT "automations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "automation_executions" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "source_id" TEXT NOT NULL,
+    "automation_id" TEXT NOT NULL,
+    "trigger_id" TEXT NOT NULL,
+    "action_id" TEXT NOT NULL,
+    "project_id" TEXT NOT NULL,
+    "status" "ActionExecutionStatus" NOT NULL DEFAULT 'PENDING',
+    "input" JSONB NOT NULL,
+    "output" JSONB,
+    "started_at" TIMESTAMP(3),
+    "finished_at" TIMESTAMP(3),
+    "error" TEXT,
+
+    CONSTRAINT "automation_executions_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "actions_project_id_idx" ON "actions"("project_id");
+
+-- CreateIndex
+CREATE INDEX "triggers_project_id_idx" ON "triggers"("project_id");
+
+-- CreateIndex
+CREATE INDEX "automations_project_id_action_id_trigger_id_idx" ON "automations"("project_id", "action_id", "trigger_id");
+
+-- CreateIndex
+CREATE INDEX "automations_project_id_name_idx" ON "automations"("project_id", "name");
+
+-- CreateIndex
+CREATE INDEX "automation_executions_trigger_id_idx" ON "automation_executions"("trigger_id");
+
+-- CreateIndex
+CREATE INDEX "automation_executions_action_id_idx" ON "automation_executions"("action_id");
+
+-- CreateIndex
+CREATE INDEX "automation_executions_project_id_idx" ON "automation_executions"("project_id");
+
+-- AddForeignKey
+ALTER TABLE "actions" ADD CONSTRAINT "actions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "triggers" ADD CONSTRAINT "triggers_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automations" ADD CONSTRAINT "automations_trigger_id_fkey" FOREIGN KEY ("trigger_id") REFERENCES "triggers"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automations" ADD CONSTRAINT "automations_action_id_fkey" FOREIGN KEY ("action_id") REFERENCES "actions"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automations" ADD CONSTRAINT "automations_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automation_executions" ADD CONSTRAINT "automation_executions_automation_id_fkey" FOREIGN KEY ("automation_id") REFERENCES "automations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automation_executions" ADD CONSTRAINT "automation_executions_trigger_id_fkey" FOREIGN KEY ("trigger_id") REFERENCES "triggers"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automation_executions" ADD CONSTRAINT "automation_executions_action_id_fkey" FOREIGN KEY ("action_id") REFERENCES "actions"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "automation_executions" ADD CONSTRAINT "automation_executions_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/shared/prisma/migrations/20250709113103_add_blob_export_schedule_type/migration.sql

@@ -0,0 +1,6 @@
+-- CreateEnum
+CREATE TYPE "BlobStorageExportMode" AS ENUM ('FULL_HISTORY', 'FROM_TODAY', 'FROM_CUSTOM_DATE');
+
+-- AlterTable
+ALTER TABLE "blob_storage_integrations" ADD COLUMN     "export_mode" "BlobStorageExportMode" NOT NULL DEFAULT 'FULL_HISTORY',
+ADD COLUMN     "export_start_date" TIMESTAMP(3);

packages/shared/prisma/migrations/20250711105322_prices_add_project_id/migration.sql

@@ -0,0 +1,13 @@
+-- AlterTable
+ALTER TABLE "prices"
+    ADD COLUMN "project_id" TEXT;
+
+-- AddForeignKey
+ALTER TABLE "prices"
+    ADD CONSTRAINT "prices_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "projects" ("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- BackfillData
+UPDATE "prices"
+SET "project_id" = (SELECT "models"."project_id"
+                    FROM "models"
+                    WHERE "models"."id" = "prices"."model_id");

packages/shared/prisma/migrations/20250711134738_add_patch_llm_tool_schema_audit_logs_background_migration/migration.sql

@@ -0,0 +1,2 @@
+INSERT INTO background_migrations (id, name, script, args)
+VALUES ('3445cac4-d9d5-4750-8b65-351135c1b85e', '20250711_1347_patch_llm_tool_schema_audit_logs', 'patchLLMToolAndLLLMSchemaAuditLogs', '{}');

packages/shared/prisma/migrations/20250714151410_add_trace_session_combined_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "trace_sessions_project_id_created_at_idx" ON "trace_sessions"("project_id", "created_at" DESC);

packages/shared/prisma/migrations/20250714151410_remove_trace_session_created_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_created_at_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_project_id_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_project_id_idx";

packages/shared/prisma/migrations/20250714151410_remove_trace_session_updated_at_idx/migration.sql

@@ -0,0 +1,2 @@
+-- DropIndex
+DROP INDEX CONCURRENTLY IF EXISTS "trace_sessions_updated_at_idx";

packages/shared/prisma/migrations/20250724114251_add_webhooks_datasets/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "datasets" ADD COLUMN     "remote_experiment_payload" JSONB,
+ADD COLUMN     "remote_experiment_url" TEXT;

packages/shared/prisma/migrations/20250724160133_add_session_object_type_annoation_queue_items/migration.sql

@@ -0,0 +1,2 @@
+-- AlterEnum
+ALTER TYPE "AnnotationQueueObjectType" ADD VALUE 'SESSION';

packages/shared/prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
 # It should be added in your version-control system (e.g., Git)
-provider = "postgresql"
+provider = "postgresql"

packages/shared/prisma/schema.prisma

@@ -157,7 +157,12 @@ model Project {
   Dashboard              Dashboard[]
   DashboardWidget        DashboardWidget[]
   TableViewPreset        TableViewPreset[]
+  actions                Action[]
+  triggers               Trigger[]
+  automationExecutions   AutomationExecution[]
+  Automation             Automation[]
   DefaultLlmModel        DefaultLlmModel[]
+  Price                  Price[]
 
   @@index([orgId])
   @@map("projects")
@@ -306,9 +311,7 @@ model TraceSession {
   environment String   @default("default")
 
   @@id([id, projectId])
-  @@index([projectId])
-  @@index([createdAt])
-  @@index([updatedAt])
+  @@index([projectId, createdAt(sort: Desc)])
   @@map("trace_sessions")
 }
 
@@ -536,6 +539,7 @@ enum AnnotationQueueStatus {
 enum AnnotationQueueObjectType {
   TRACE
   OBSERVATION
+  SESSION
 }
 
 model CronJobs {
@@ -548,16 +552,18 @@ model CronJobs {
 }
 
 model Dataset {
-  id           String        @default(cuid())
-  projectId    String        @map("project_id")
-  name         String
-  description  String?
-  metadata     Json?
-  project      Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
-  createdAt    DateTime      @default(now()) @map("created_at")
-  updatedAt    DateTime      @default(now()) @updatedAt @map("updated_at")
-  datasetItems DatasetItem[]
-  datasetRuns  DatasetRuns[]
+  id                      String        @default(cuid())
+  projectId               String        @map("project_id")
+  name                    String
+  description             String?
+  metadata                Json?
+  remoteExperimentUrl     String?       @map("remote_experiment_url")
+  remoteExperimentPayload Json?         @map("remote_experiment_payload")
+  project                 Project       @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  createdAt               DateTime      @default(now()) @map("created_at")
+  updatedAt               DateTime      @default(now()) @updatedAt @map("updated_at")
+  datasetItems            DatasetItem[]
+  datasetRuns             DatasetRuns[]
 
   @@id([id, projectId])
   @@unique([projectId, name])
@@ -751,6 +757,8 @@ model Price {
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
   modelId   String   @map("model_id") // Model is already linked to project (or default), so we don't need projectId here
   Model     Model    @relation(fields: [modelId], references: [id], onDelete: Cascade)
+  projectId String?  @map("project_id")
+  project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
   usageType String   @map("usage_type")
   price     Decimal
 
@@ -963,6 +971,8 @@ model BlobStorageIntegration {
   enabled         Boolean
   exportFrequency String                         @map("export_frequency")
   fileType        BlobStorageIntegrationFileType @default(CSV) @map("file_type")
+  exportMode      BlobStorageExportMode          @default(FULL_HISTORY) @map("export_mode")
+  exportStartDate DateTime?                      @map("export_start_date")
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
@@ -986,6 +996,14 @@ enum BlobStorageIntegrationType {
   @@map("BlobStorageIntegrationType")
 }
 
+enum BlobStorageExportMode {
+  FULL_HISTORY
+  FROM_TODAY
+  FROM_CUSTOM_DATE
+
+  @@map("BlobStorageExportMode")
+}
+
 model BatchExport {
   id        String   @id @default(cuid())
   createdAt DateTime @default(now()) @map("created_at")
@@ -1214,3 +1232,109 @@ model TableViewPreset {
   @@unique([projectId, tableName, name])
   @@map("table_view_presets")
 }
+
+model Action {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  type ActionType
+
+  // Configuration specific to each action type
+  config Json // Structured JSON for different action types
+  // For WEBHOOK: { version: "1.0", url: "...", method: "POST", headers: {...}, secretId: "..." }
+  // For ANNOTATION_QUEUE: { version: "1.0", queueId: "..." }
+
+  automations          Automation[]
+  automationExecutions AutomationExecution[]
+
+  @@index([projectId])
+  @@map("actions")
+}
+
+model Trigger {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  // When should this trigger fire
+  eventSource  String // trace, prompt, etc.
+  eventActions String[] // created, updated, deleted
+  filter       Json? // Filter conditions (format: { field: "name", operator: "equals", value: "my_trace" })
+
+  // Additional attributes
+  status JobConfigState @default(ACTIVE) @map("status")
+
+  // Link to executions
+  automationExecutions AutomationExecution[]
+  automations          Automation[]
+
+  @@index([projectId])
+  @@map("triggers")
+}
+
+model Automation {
+  id                  String                @id @default(cuid())
+  name                String                @map("name")
+  trigger             Trigger               @relation(fields: [triggerId], references: [id], onDelete: Cascade)
+  triggerId           String                @map("trigger_id")
+  action              Action                @relation(fields: [actionId], references: [id], onDelete: Cascade)
+  actionId            String                @map("action_id")
+  createdAt           DateTime              @default(now()) @map("created_at")
+  projectId           String                @map("project_id")
+  project             Project               @relation(fields: [projectId], references: [id], onDelete: Cascade)
+  AutomationExecution AutomationExecution[]
+
+  @@index([projectId, actionId, triggerId])
+  @@index([projectId, name])
+  @@map("automations")
+}
+
+enum ActionType {
+  WEBHOOK
+  // More action types can be added as needed
+}
+
+enum ActionExecutionStatus {
+  COMPLETED
+  ERROR
+  PENDING
+  CANCELLED
+}
+
+model AutomationExecution {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @default(now()) @updatedAt @map("updated_at")
+  sourceId  String   @map("source_id")
+
+  automationId String     @map("automation_id")
+  automation   Automation @relation(fields: [automationId], references: [id], onDelete: Cascade)
+
+  triggerId String  @map("trigger_id")
+  trigger   Trigger @relation(fields: [triggerId], references: [id], onDelete: Cascade)
+
+  actionId String @map("action_id")
+  action   Action @relation(fields: [actionId], references: [id], onDelete: Cascade)
+
+  projectId String  @map("project_id")
+  project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+
+  status     ActionExecutionStatus @default(PENDING) @map("status")
+  input      Json                  @map("input")
+  output     Json?                 @map("output")
+  startedAt  DateTime?             @map("started_at")
+  finishedAt DateTime?             @map("finished_at")
+  error      String?               @map("error")
+
+  @@index([triggerId])
+  @@index([actionId])
+  @@index([projectId])
+  @@map("automation_executions")
+}

packages/shared/scripts/seeder/seed-postgres.ts

@@ -4,15 +4,16 @@ import { hash } from "bcryptjs";
 import { v4 } from "uuid";
 import { encrypt } from "../../src/encryption";
 import {
-	type JobConfiguration,
-	JobExecutionStatus,
-	PrismaClient,
-	type Project,
-	ScoreDataType,
+  type JobConfiguration,
+  JobExecutionStatus,
+  PrismaClient,
+  type Project,
+  ScoreDataType,
 } from "../../src/index";
 import { getDisplaySecretKey, hashSecretKey, logger } from "../../src/server";
 import { redis } from "../../src/server/redis/redis";
-import {EVAL_TRACE_COUNT,
+import {
+  EVAL_TRACE_COUNT,
   FAILED_EVAL_TRACE_INTERVAL,
   SEED_CHAT_ML_PROMPTS,
   SEED_DATASETS,
@@ -22,6 +23,7 @@ import {EVAL_TRACE_COUNT,
   SEED_TEXT_PROMPTS,
 } from "./utils/postgres-seed-constants";
 import {
+  generateDatasetItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -516,6 +518,7 @@ export async function createDatasets(
             description: data.description,
             projectId,
             metadata: data.metadata,
+            id: `${datasetName}-${projectId.slice(-8)}`,
           },
         }));
 
@@ -531,13 +534,23 @@ export async function createDatasets(
         const datasetItem = await prisma.datasetItem.upsert({
           where: {
             id_projectId: {
-              id: `${dataset.id}-${index}`,
+              id: generateDatasetItemId(
+                datasetName,
+                index,
+                projectId,
+                SEED_DATASETS.indexOf(data) || 0,
+              ),
               projectId,
             },
           },
           create: {
             projectId,
-            id: `${dataset.id}-${index}`,
+            id: generateDatasetItemId(
+              datasetName,
+              index,
+              projectId,
+              SEED_DATASETS.indexOf(data) || 0,
+            ),
             datasetId: dataset.id,
             sourceTraceId: sourceTraceId ?? null,
             sourceObservationId: null,
@@ -553,14 +566,14 @@ export async function createDatasets(
       for (let datasetRunNumber = 0; datasetRunNumber < 3; datasetRunNumber++) {
         const datasetRun = await prisma.datasetRuns.upsert({
           where: {
-            datasetId_projectId_name: {
-              datasetId: dataset.id,
+            id_projectId: {
+              id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
               projectId,
-              name: `demo-dataset-run-${datasetRunNumber}`,
             },
           },
           create: {
             projectId,
+            id: `demo-dataset-run-${datasetRunNumber}-${projectId.slice(-8)}`,
             name: `demo-dataset-run-${datasetRunNumber}`,
             description: Math.random() > 0.5 ? "Dataset run description" : "",
             datasetId: dataset.id,

packages/shared/scripts/seeder/utils/clickhouse-builder.ts

@@ -2,6 +2,8 @@ import {
   TraceRecordInsertType,
   ObservationRecordInsertType,
   ScoreRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItemsCh,
 } from "../../../src/server";
 import { SEED_TEXT_PROMPTS } from "./postgres-seed-constants";
 import {
@@ -42,6 +44,16 @@ export class ClickHouseQueryBuilder {
     return await createObservationsCh(observations);
   }
 
+  /**
+   * Creates INSERT query for dataset run items data using VALUES syntax.
+   * Use for: Small datasets, dataset run items that link to postgres data (e.g. dataset runs)
+   */
+  async executeDatasetRunItemsInsert(
+    datasetRunItems: DatasetRunItemRecordInsertType[],
+  ): Promise<InsertResult> {
+    return await createDatasetRunItemsCh(datasetRunItems);
+  }
+
   /**
    * Creates INSERT query for score data using VALUES syntax.
    * Use for: Small datasets, scores with custom values and metadata.

packages/shared/scripts/seeder/utils/data-generators.ts

@@ -6,6 +6,8 @@ import {
   REALISTIC_MODELS,
 } from "./clickhouse-seed-constants";
 import {
+  generateDatasetItemId,
+  generateDatasetRunItemId,
   generateDatasetRunTraceId,
   generateEvalObservationId,
   generateEvalScoreId,
@@ -23,6 +25,8 @@ import {
   ObservationRecordInsertType,
   ScoreRecordInsertType,
   TraceRecordInsertType,
+  DatasetRunItemRecordInsertType,
+  createDatasetRunItem,
 } from "../../../src/server";
 
 /**
@@ -60,6 +64,49 @@ export class DataGenerator {
     return Math.floor(Math.random() * (max - min + 1)) + min;
   }
 
+  /**
+   * Creates dataset run items for dataset runs.
+   * Use for: Dataset experiment scenarios.
+   */
+  generateDatasetRunItem(
+    input: DatasetItemInput & { runCreatedAt: number },
+    projectId: string,
+  ): DatasetRunItemRecordInsertType {
+    const datasetRunItemId = generateDatasetRunItemId(
+      input.datasetName,
+      input.itemIndex,
+      projectId,
+      input.runNumber || 0,
+    );
+
+    // TODO: there are too many dataset run items in the postgres database?
+    return createDatasetRunItem({
+      id: datasetRunItemId,
+      project_id: projectId,
+      trace_id: generateDatasetRunTraceId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_id: `${input.datasetName}-${projectId.slice(-8)}`,
+      dataset_run_id: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_name: `demo-dataset-run-${input.runNumber}-${projectId.slice(-8)}`,
+      dataset_run_created_at: input.runCreatedAt,
+      dataset_run_description:
+        (input.runNumber || 0) % 2 === 0 ? "Dataset run description" : "",
+      dataset_run_metadata: { key: "value" },
+      dataset_item_id: generateDatasetItemId(
+        input.datasetName,
+        input.itemIndex,
+        projectId,
+        input.runNumber || 0,
+      ),
+      dataset_item_input: input.item.input,
+      dataset_item_expected_output: input.item.output,
+    });
+  }
+
   /**
    * Creates traces from dataset items for experiment runs.
    * Use for: Dataset experiments scenarios.

packages/shared/scripts/seeder/utils/markdown.txt

@@ -11,29 +11,29 @@
 ## 🎯 Getting Started
 
 ### Prerequisites
-\`\`\`bash
-npm install @langfuse/core
-pip install langfuse
+\`\`\`bash  
+npm install @langfuse/core  
+pip install langfuse  
 \`\`\`
 
 ### Quick Setup
 1. **Initialize your project**
-   \`\`\`typescript
-   import { Langfuse } from 'langfuse'
-   
-   const langfuse = new Langfuse({
-     secretKey: process.env.LANGFUSE_SECRET_KEY,
-     publicKey: process.env.LANGFUSE_PUBLIC_KEY,
-     baseUrl: 'https://cloud.langfuse.com'
-   })
+   \`\`\`typescript  
+   import { Langfuse } from 'langfuse'  
+
+   const langfuse = new Langfuse({  
+     secretKey: process.env.LANGFUSE_SECRET_KEY,  
+     publicKey: process.env.LANGFUSE_PUBLIC_KEY,  
+     baseUrl: 'https://cloud.langfuse.com'  
+   })  
    \`\`\`
 
 2. **Create your first trace**
-   \`\`\`python
-   from langfuse import Langfuse
-   
-   langfuse = Langfuse()
-   trace = langfuse.trace(name="chat-application")
+   \`\`\`python  
+   from langfuse import Langfuse  
+
+   langfuse = Langfuse()  
+   trace = langfuse.trace(name="chat-application")  
    \`\`\`
 
 ---
@@ -72,75 +72,75 @@ graph TD
 > **Note:** Traces are the foundation of observability in LLM applications.
 
 #### Creating Traces
-\`\`\`typescript
-// Basic trace creation
-const trace = langfuse.trace({
-  name: "user-query-processing",
-  userId: "user-123",
-  sessionId: "session-456",
-  metadata: {
-    environment: "production",
-    version: "2.1.0"
-  }
-})
+\`\`\`typescript  
+// Basic trace creation  
+const trace = langfuse.trace({  
+  name: "user-query-processing",  
+  userId: "user-123",  
+  sessionId: "session-456",  
+  metadata: {  
+    environment: "production",  
+    version: "2.1.0"  
+  }  
+})  
 
-// Nested observations
-const span = trace.span({
-  name: "document-retrieval",
-  input: { query: "What is machine learning?" },
-  metadata: { vectorStore: "pinecone" }
-})
+// Nested observations  
+const span = trace.span({  
+  name: "document-retrieval",  
+  input: { query: "What is machine learning?" },  
+  metadata: { vectorStore: "pinecone" }  
+})  
 
-const generation = span.generation({
-  name: "answer-generation",
-  model: "gpt-4",
-  input: retrievedDocs,
-  output: generatedAnswer,
-  usage: {
-    promptTokens: 1250,
-    completionTokens: 420,
-    totalTokens: 1670
-  }
-})
+const generation = span.generation({  
+  name: "answer-generation",  
+  model: "gpt-4",  
+  input: retrievedDocs,  
+  output: generatedAnswer,  
+  usage: {  
+    promptTokens: 1250,  
+    completionTokens: 420,  
+    totalTokens: 1670  
+  }  
+})  
 \`\`\`
 
 ### Advanced Features
 
 #### 🔄 Async Processing
-\`\`\`python
-import asyncio
-from langfuse import Langfuse
+\`\`\`python  
+import asyncio  
+from langfuse import Langfuse  
 
-async def process_batch():
-    langfuse = Langfuse()
-    
-    tasks = []
-    for item in batch_items:
-        task = asyncio.create_task(
-            process_item_with_tracing(langfuse, item)
-        )
-        tasks.append(task)
-    
-    results = await asyncio.gather(*tasks)
-    return results
+async def process_batch():  
+    langfuse = Langfuse()  
+
+    tasks = []  
+    for item in batch_items:  
+        task = asyncio.create_task(  
+            process_item_with_tracing(langfuse, item)  
+        )  
+        tasks.append(task)  
+
+    results = await asyncio.gather(*tasks)  
+    return results  
 \`\`\`
 
 #### 🎯 Custom Scoring
-\`\`\`typescript
-// Automated scoring
-trace.score({
-  name: "relevance",
-  value: 0.95,
-  comment: "Highly relevant response"
-})
+\`\`\`typescript  
+// Automated scoring  
+trace.score({  
+  name: "relevance",  
+  value: 0.95,  
+  comment: "Highly relevant response"  
+})  
 
-// Human feedback scoring
-trace.score({
-  name: "user-satisfaction", 
-  value: 1,
-  source: "user-feedback",
-  comment: "User rated 5/5 stars"
-})
+// Human feedback scoring  
+trace.score({  
+  name: "user-satisfaction",  
+  value: 1,  
+  source: "user-feedback",  
+  comment: "User rated 5/5 stars"  
+})  
 \`\`\`
 
 ---
@@ -156,20 +156,20 @@ trace.score({
 - **User Satisfaction**: Quality metrics
 
 #### Dashboard Setup
-\`\`\`yaml
-# monitoring-config.yml
-dashboards:
-  - name: "LLM Performance"
-    panels:
-      - type: "time-series"
-        title: "Response Latency"
-        query: "avg(response_time) by (model)"
-      - type: "stat"
-        title: "Daily Token Usage" 
-        query: "sum(tokens_used)"
-      - type: "table"
-        title: "Top Errors"
-        query: "topk(10, count by (error_type))"
+\`\`\`yaml  
+# monitoring-config.yml  
+dashboards:  
+  - name: "LLM Performance"  
+    panels:  
+      - type: "time-series"  
+        title: "Response Latency"  
+        query: "avg(response_time) by (model)"  
+      - type: "stat"  
+        title: "Daily Token Usage"  
+        query: "sum(tokens_used)"  
+      - type: "table"  
+        title: "Top Errors"  
+        query: "topk(10, count by (error_type))"  
 \`\`\`
 
 ### 🔐 Security Considerations
@@ -177,40 +177,40 @@ dashboards:
 > ⚠️ **Important**: Never log sensitive user data in traces
 
 #### Data Sanitization
-\`\`\`python
-def sanitize_input(data):
-    """Remove PII from trace data"""
-    sanitized = data.copy()
-    
-    # Remove email addresses
-    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', 
-                      '[EMAIL_REDACTED]', sanitized)
-    
-    # Remove phone numbers
-    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b', 
-                      '[PHONE_REDACTED]', sanitized)
-    
-    return sanitized
+\`\`\`python  
+def sanitize_input(data):  
+    """Remove PII from trace data"""  
+    sanitized = data.copy()  
+
+    # Remove email addresses  
+    sanitized = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b',  
+                      '[EMAIL_REDACTED]', sanitized)  
+
+    # Remove phone numbers  
+    sanitized = re.sub(r'\\b\\d{3}-\\d{3}-\\d{4}\\b',  
+                      '[PHONE_REDACTED]', sanitized)  
+
+    return sanitized  
 \`\`\`
 
 ### 🚀 Performance Optimization
 
 #### Batch Processing
-\`\`\`typescript
-// Efficient batch uploads
-const batchSize = 100
-const traces = []
+\`\`\`typescript  
+// Efficient batch uploads  
+const batchSize = 100  
+const traces = []  
 
-for (let i = 0; i < data.length; i += batchSize) {
-  const batch = data.slice(i, i + batchSize)
-  const processedBatch = await Promise.all(
-    batch.map(item => processWithLangfuse(item))
-  )
-  traces.push(...processedBatch)
-}
+for (let i = 0; i < data.length; i += batchSize) {  
+  const batch = data.slice(i, i + batchSize)  
+  const processedBatch = await Promise.all(  
+    batch.map(item => processWithLangfuse(item))  
+  )  
+  traces.push(...processedBatch)  
+}  
 
-// Flush all traces at once
-await langfuse.flushAsync()
+// Flush all traces at once  
+await langfuse.flushAsync()  
 \`\`\`
 
 ---
@@ -219,34 +219,34 @@ await langfuse.flushAsync()
 
 ### Multi-Agent System Tracing
 \`\`\`python
-class MultiAgentTracer:
-    def __init__(self):
-        self.langfuse = Langfuse()
-    
-    async def orchestrate_agents(self, task):
-        # Main orchestration trace
-        main_trace = self.langfuse.trace(
-            name="multi-agent-orchestration",
-            input={"task": task}
-        )
-        
-        # Agent 1: Research
-        research_span = main_trace.span(name="research-agent")
-        research_result = await self.research_agent.process(task)
-        research_span.end(output=research_result)
-        
+class MultiAgentTracer:  
+    def __init__(self):  
+        self.langfuse = Langfuse()  
+
+    async def orchestrate_agents(self, task):  
+        # Main orchestration trace  
+        main_trace = self.langfuse.trace(  
+            name="multi-agent-orchestration",  
+            input={"task": task}  
+        )  
+
+        # Agent 1: Research  
+        research_span = main_trace.span(name="research-agent")  
+        research_result = await self.research_agent.process(task)  
+        research_span.end(output=research_result)  
+
         # Agent 2: Analysis  
-        analysis_span = main_trace.span(name="analysis-agent")
-        analysis_result = await self.analysis_agent.process(research_result)
-        analysis_span.end(output=analysis_result)
-        
-        # Agent 3: Synthesis
-        synthesis_span = main_trace.span(name="synthesis-agent")
-        final_result = await self.synthesis_agent.process(analysis_result)
-        synthesis_span.end(output=final_result)
-        
-        main_trace.end(output=final_result)
-        return final_result
+        analysis_span = main_trace.span(name="analysis-agent")  
+        analysis_result = await self.analysis_agent.process(research_result)  
+        analysis_span.end(output=analysis_result)  
+
+        # Agent 3: Synthesis  
+        synthesis_span = main_trace.span(name="synthesis-agent")  
+        final_result = await self.synthesis_agent.process(analysis_result)  
+        synthesis_span.end(output=final_result)  
+
+        main_trace.end(output=final_result)  
+        return final_result  
 \`\`\`
 
 ---
@@ -256,11 +256,11 @@ class MultiAgentTracer:
 With proper implementation of Langfuse tracing, you can:
 
 - ✅ **Monitor** your LLM applications in real-time
-- ✅ **Debug** issues with detailed trace information  
+- ✅ **Debug** issues with detailed trace information
 - ✅ **Optimize** performance and costs
 - ✅ **Scale** your applications with confidence
 
 ### Next Steps
 1. Review the [official documentation](https://langfuse.com/docs)
 2. Join our [Discord community](https://discord.gg/langfuse)
-3. Check out [example projects](https://github.com/langfuse/langfuse)
+3. Check out [example projects](https://github.com/langfuse/langfuse)

packages/shared/scripts/seeder/utils/postgres-seed-constants.ts

@@ -271,6 +271,68 @@ export const SEED_TEXT_PROMPTS = [
     labels: ["production", "latest"],
     tags: ["tag1", "tag2"],
   },
+  {
+    id: `prompt-with-many-labels`,
+    createdBy: "user-1",
+    prompt:
+      "This is a comprehensive prompt for testing multiple label scenarios. It demonstrates how prompts can be tagged with numerous labels for organization, categorization, and filtering purposes. Use this prompt to understand how label management works at scale. Variables: {{input}}",
+    name: "prompt-with-many-labels",
+    version: 1,
+    labels: [
+      "production",
+      "latest",
+      "v1",
+      "v2",
+      "stable",
+      "beta",
+      "alpha",
+      "test",
+      "development",
+      "staging",
+      "experimental",
+      "feature",
+      "bugfix",
+      "hotfix",
+      "critical",
+      "high-priority",
+      "medium-priority",
+      "low-priority",
+      "urgent",
+      "customer-facing",
+      "internal",
+      "public",
+      "private",
+      "confidential",
+      "ai",
+      "nlp",
+      "chatbot",
+      "assistant",
+      "automation",
+      "ml",
+      "data",
+      "analytics",
+      "monitoring",
+      "logging",
+      "debug",
+      "performance",
+      "security",
+      "compliance",
+      "audit",
+      "review",
+      "approved",
+      "rejected",
+      "pending",
+      "archived",
+      "deprecated",
+      "legacy",
+      "migration",
+      "upgrade",
+      "downgrade",
+      "template",
+      "example",
+    ],
+    tags: [],
+  },
 ];
 
 export const SEED_CHAT_ML_PROMPTS = [

packages/shared/scripts/seeder/utils/seed-helpers.ts

@@ -1,3 +1,21 @@
+export const generateDatasetRunItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-run-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
+export const generateDatasetItemId = (
+  datasetName: string,
+  itemIndex: number,
+  projectId: string,
+  runNumber: number,
+) => {
+  return `dataset-item-${datasetName}-${itemIndex}-${projectId.slice(-8)}-${runNumber}`;
+};
+
 export const generateDatasetRunTraceId = (
   datasetName: string,
   itemIndex: number,

packages/shared/scripts/seeder/utils/seeder-orchestrator.ts

@@ -92,12 +92,26 @@ export class SeederOrchestrator {
         logger.info(
           `Processing run ${runNumber + 1}/${numberOfRuns} for project ${projectId}`,
         );
+        // const now = Date.now();
 
         const traces: TraceRecordInsertType[] = [];
         const observations: ObservationRecordInsertType[] = [];
+        // const datasetRunItems: DatasetRunItemRecordInsertType[] = [];
 
         for (const seedDataset of SEED_DATASETS) {
           for (const [itemIndex, datasetItem] of seedDataset.items.entries()) {
+            // // Generate dataset run item data
+            // const datasetRunItem = this.dataGenerator.generateDatasetRunItem(
+            //   {
+            //     datasetName: seedDataset.name,
+            //     itemIndex,
+            //     item: datasetItem,
+            //     runNumber,
+            //     runCreatedAt: now,
+            //   },
+            //   projectId,
+            // );
+
             // Generate trace data
             const trace = this.dataGenerator.generateDatasetTrace(
               {
@@ -123,12 +137,14 @@ export class SeederOrchestrator {
 
             traces.push(trace);
             observations.push(observation);
+            // datasetRunItems.push(datasetRunItem);
           }
         }
 
         try {
           await this.queryBuilder.executeTracesInsert(traces);
           await this.queryBuilder.executeObservationsInsert(observations);
+          // await this.queryBuilder.executeDatasetRunItemsInsert(datasetRunItems);
         } catch (error) {
           logger.error(`✗ Insert failed:`, error);
           throw error;

packages/shared/src/db.ts

@@ -88,7 +88,8 @@ declare const globalThis: {
   kyselyPrismaGlobal: { $kysely: Kysely<DB> } | undefined;
 } & typeof global;
 
-if (process.env.NODE_ENV === "development") { // eslint-disable-line turbo/no-undeclared-env-vars
+// eslint-disable-next-line turbo/no-undeclared-env-vars
+if (process.env.NODE_ENV === "development") {
   globalThis.prismaGlobal ??= createPrismaInstance(); // regular instantiation
   globalThis.kyselyPrismaGlobal ??= globalThis.prismaGlobal.$extends(
     kyselyExtension({

packages/shared/src/domain/automations.ts

@@ -0,0 +1,95 @@
+import { Action, Trigger } from "@prisma/client";
+import { FilterState } from "../types";
+import { z } from "zod/v4";
+
+export enum TriggerEventSource {
+  // eslint-disable-next-line no-unused-vars
+  Prompt = "prompt",
+}
+
+export const EventActionSchema = z.enum(["created", "updated", "deleted"]);
+
+export type TriggerEventAction = z.infer<typeof EventActionSchema>;
+
+export const TriggerEventSourceSchema = z.enum([TriggerEventSource.Prompt]);
+
+export type TriggerDomain = Omit<
+  Trigger,
+  "filter" | "eventSource" | "eventActions"
+> & {
+  filter: FilterState;
+  eventSource: TriggerEventSource;
+  eventActions: TriggerEventAction[];
+};
+
+export type AutomationDomain = {
+  id: string;
+  name: string;
+  trigger: TriggerDomain;
+  action: ActionDomain;
+};
+
+export type ActionDomain = Omit<Action, "config"> & {
+  config: SafeWebhookActionConfig;
+};
+
+export type ActionDomainWithSecrets = Omit<Action, "config"> & {
+  config: WebhookActionConfigWithSecrets;
+};
+
+export const ActionTypeSchema = z.enum(["WEBHOOK"]);
+
+export const AvailableWebhookApiSchema = z.record(
+  z.enum(["prompt"]),
+  z.enum(["v1"]),
+);
+
+export const RequestHeaderSchema = z.object({
+  secret: z.boolean(),
+  value: z.string(),
+});
+
+export const WebhookActionConfigSchema = z.object({
+  type: z.literal("WEBHOOK"),
+  url: z.url(),
+  headers: z.record(z.string(), z.string()),
+  requestHeaders: z.record(z.string(), RequestHeaderSchema),
+  displayHeaders: z.record(z.string(), RequestHeaderSchema),
+  apiVersion: AvailableWebhookApiSchema,
+  secretKey: z.string(),
+  displaySecretKey: z.string(),
+  lastFailingExecutionId: z.string().nullish(),
+});
+
+export const SafeWebhookActionConfigSchema = WebhookActionConfigSchema.omit({
+  secretKey: true,
+  headers: true,
+  requestHeaders: true,
+});
+
+export type SafeWebhookActionConfig = z.infer<
+  typeof SafeWebhookActionConfigSchema
+>;
+
+export const WebhookActionCreateSchema = WebhookActionConfigSchema.omit({
+  secretKey: true,
+  displaySecretKey: true,
+  headers: true, // don't use legacy field anymore
+  displayHeaders: true,
+});
+
+export const ActionConfigSchema = z.discriminatedUnion("type", [
+  WebhookActionConfigSchema,
+]);
+
+export const ActionCreateSchema = z.discriminatedUnion("type", [
+  WebhookActionCreateSchema,
+]);
+
+export type ActionTypes = z.infer<typeof ActionTypeSchema>;
+export type ActionConfig = z.infer<typeof ActionConfigSchema>;
+export type ActionCreate = z.infer<typeof ActionCreateSchema>;
+
+export type WebhookActionConfigWithSecrets = z.infer<
+  typeof WebhookActionConfigSchema
+>;

packages/shared/src/domain/dataset-run-items.ts

@@ -0,0 +1,28 @@
+import z from "zod/v4";
+import { jsonSchema } from "../utils/zod";
+import { MetadataDomain } from "./traces";
+
+export const DatasetRunItemSchema = z.object({
+  id: z.string(),
+  projectId: z.string(),
+  datasetRunId: z.string(),
+  datasetItemId: z.string(),
+  datasetId: z.string(),
+  traceId: z.string(),
+  observationId: z.string().nullable(),
+  error: z.string().nullable(),
+  // timestamps
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  // dataset run fields
+  datasetRunName: z.string(),
+  datasetRunDescription: z.string().nullable(),
+  datasetRunMetadata: MetadataDomain,
+  datasetRunCreatedAt: z.date(),
+  // dataset item fields
+  datasetItemInput: jsonSchema,
+  datasetItemExpectedOutput: jsonSchema,
+  datasetItemMetadata: MetadataDomain,
+});
+
+export type DatasetRunItemDomain = z.infer<typeof DatasetRunItemSchema>;

packages/shared/src/domain/index.ts

@@ -2,3 +2,6 @@ export * from "./observations";
 export * from "./traces";
 export * from "./scores";
 export * from "./table-view-presets";
+export * from "./automations";
+export * from "./webhooks";
+export * from "./prompts";

packages/shared/src/domain/prompts.ts

@@ -0,0 +1,21 @@
+import { z } from "zod/v4";
+import { jsonSchemaNullable } from "../utils/zod";
+
+export const PromptDomainSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  version: z.number(),
+  createdAt: z.date(),
+  updatedAt: z.date(),
+  createdBy: z.string(),
+  isActive: z.boolean().nullable(),
+  type: z.string().default("text"),
+  tags: z.array(z.string()).default([]),
+  labels: z.array(z.string()).default([]),
+  prompt: jsonSchemaNullable,
+  config: jsonSchemaNullable,
+  projectId: z.string(),
+  commitMessage: z.string().nullable(),
+});
+
+export type PromptDomain = z.infer<typeof PromptDomainSchema>;

packages/shared/src/domain/webhooks.ts

@@ -0,0 +1,37 @@
+import { z } from "zod/v4";
+import { jsonSchema } from "../utils/zod";
+import { EventActionSchema } from "./automations";
+
+export const WebhookDefaultHeaders = {
+  "content-type": "application/json",
+  "user-agent": "Langfuse/1.0",
+};
+
+export const WebhookOutboundBaseSchema = z.object({
+  id: z.string(),
+  timestamp: z.coerce.date(),
+  type: z.literal("prompt-version"),
+  apiVersion: z.literal("v1"),
+  action: EventActionSchema,
+});
+
+export const PromptWebhookOutboundSchema = z
+  .object({
+    prompt: z.object({
+      id: z.string(),
+      name: z.string(),
+      version: z.number(),
+      projectId: z.string(),
+      labels: z.array(z.string()),
+      prompt: jsonSchema.nullable(),
+      type: z.string(),
+      config: z.record(z.string(), z.any()),
+      commitMessage: z.string().nullable(),
+      tags: z.array(z.string()),
+      createdAt: z.coerce.date(),
+      updatedAt: z.coerce.date(),
+    }),
+  })
+  .and(WebhookOutboundBaseSchema);
+
+export type PromptWebhookOutput = z.infer<typeof PromptWebhookOutboundSchema>;

packages/shared/src/encryption/encryption.ts

@@ -0,0 +1,60 @@
+import crypto from "crypto";
+import { env } from "../env";
+
+const ENCRYPTION_KEY: string | undefined = env.ENCRYPTION_KEY; // Must be 256 bits (32 bytes, 64 hex characters)
+const IV_LENGTH: number = 12; // For AES-GCM, this is always 12
+
+// Alternatively: openssl rand -hex 32
+export function keyGen() {
+  return crypto.randomBytes(32).toString("hex");
+}
+
+/**
+ * Encrypts the given plain text using AES-256-GCM algorithm.
+ *
+ * @param {string} plainText - The text to encrypt.
+ * @returns {string} The encrypted data in hex format, including IV and authentication tag.
+ */
+export function encrypt(plainText: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
+  const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
+  const cipher = crypto.createCipheriv(
+    "aes-256-gcm",
+    Buffer.from(ENCRYPTION_KEY, "hex"),
+    iv,
+  );
+  let encrypted = cipher.update(plainText, "utf8", "hex");
+  encrypted += cipher.final("hex");
+  const authTag = cipher.getAuthTag();
+
+  // Return iv, encrypted data, and authTag as hex, combined in one line
+  return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
+}
+
+export function decrypt(text: string): string {
+  if (!ENCRYPTION_KEY) {
+    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
+  }
+  const [ivHex, encryptedHex, authTagHex] = text.split(":");
+  if (!ivHex || !encryptedHex || !authTagHex) {
+    throw new Error("Invalid or corrupted cipher format");
+  }
+
+  const iv = Buffer.from(ivHex, "hex");
+  const encryptedText = Buffer.from(encryptedHex, "hex");
+  const authTag = Buffer.from(authTagHex, "hex");
+
+  const decipher = crypto.createDecipheriv(
+    "aes-256-gcm",
+    Buffer.from(ENCRYPTION_KEY, "hex"),
+    iv,
+  );
+  decipher.setAuthTag(authTag);
+
+  let decrypted = decipher.update(encryptedText, undefined, "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted.toString();
+}

packages/shared/src/encryption/index.ts

@@ -1,60 +1,2 @@
-import crypto from "crypto";
-import { env } from "../env";
-
-const ENCRYPTION_KEY: string | undefined = env.ENCRYPTION_KEY; // Must be 256 bits (32 bytes, 64 hex characters)
-const IV_LENGTH: number = 12; // For AES-GCM, this is always 12
-
-// Alternatively: openssl rand -hex 32
-export function keyGen() {
-  return crypto.randomBytes(32).toString("hex");
-}
-
-/**
- * Encrypts the given plain text using AES-256-GCM algorithm.
- *
- * @param {string} plainText - The text to encrypt.
- * @returns {string} The encrypted data in hex format, including IV and authentication tag.
- */
-export function encrypt(plainText: string): string {
-  if (!ENCRYPTION_KEY) {
-    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
-  }
-  const iv = crypto.randomBytes(IV_LENGTH); // Directly use Buffer returned by randomBytes
-  const cipher = crypto.createCipheriv(
-    "aes-256-gcm",
-    Buffer.from(ENCRYPTION_KEY, "hex"),
-    iv,
-  );
-  let encrypted = cipher.update(plainText, "utf8", "hex");
-  encrypted += cipher.final("hex");
-  const authTag = cipher.getAuthTag();
-
-  // Return iv, encrypted data, and authTag as hex, combined in one line
-  return iv.toString("hex") + ":" + encrypted + ":" + authTag.toString("hex");
-}
-
-export function decrypt(text: string): string {
-  if (!ENCRYPTION_KEY) {
-    throw new Error("Missing environment variable: `ENCRYPTION_KEY`");
-  }
-  const [ivHex, encryptedHex, authTagHex] = text.split(":");
-  if (!ivHex || !encryptedHex || !authTagHex) {
-    throw new Error("Invalid or corrupted cipher format");
-  }
-
-  const iv = Buffer.from(ivHex, "hex");
-  const encryptedText = Buffer.from(encryptedHex, "hex");
-  const authTag = Buffer.from(authTagHex, "hex");
-
-  const decipher = crypto.createDecipheriv(
-    "aes-256-gcm",
-    Buffer.from(ENCRYPTION_KEY, "hex"),
-    iv,
-  );
-  decipher.setAuthTag(authTag);
-
-  let decrypted = decipher.update(encryptedText, undefined, "utf8");
-  decrypted += decipher.final("utf8");
-
-  return decrypted.toString();
-}
+export * from "./encryption";
+export * from "./signature";

packages/shared/src/encryption/signature.ts

@@ -0,0 +1,41 @@
+import crypto from "crypto";
+
+// Generate a webhook secret key
+export function generateWebhookSecret(): {
+  secretKey: string;
+  displaySecretKey: string;
+} {
+  // Generate 32 random bytes and encode as hex (64 characters)
+  const rawSecret = crypto.randomBytes(32).toString("hex");
+  const secretKey = `lf-whsec_${rawSecret}`;
+  return { secretKey, displaySecretKey: getDisplaySecretKey(secretKey) };
+}
+
+// Create display version of webhook secret
+export function getDisplaySecretKey(secretKey: string): string {
+  if (!secretKey || secretKey.length < 12) {
+    // whsec_ + at least 4 chars
+    return "****";
+  }
+
+  return `lf-whsec_...${secretKey.slice(-4)}`;
+}
+
+// Generate HMAC-SHA256 signature for webhook payload
+export function generateWebhookSignature(
+  payload: string,
+  timestamp: number,
+  secret: string,
+) {
+  const signedPayload = `${timestamp}.${payload}`;
+  return crypto
+    .createHmac("sha256", secret)
+    .update(signedPayload, "utf8")
+    .digest("hex");
+}
+
+export function createSignatureHeader(payload: string, secret: string): string {
+  const timestamp = Math.floor(Date.now() / 1000);
+  const signature = generateWebhookSignature(payload, timestamp, secret);
+  return `t=${timestamp},v1=${signature}`;
+}

packages/shared/src/env.ts

@@ -16,6 +16,7 @@ const EnvSchema = z.object({
     .nullable(),
   REDIS_AUTH: z.string().nullish(),
   REDIS_CONNECTION_STRING: z.string().nullish(),
+  REDIS_KEY_PREFIX: z.string().nullish(),
   REDIS_TLS_ENABLED: z.enum(["true", "false"]).default("false"),
   REDIS_TLS_CA_PATH: z.string().optional(),
   REDIS_TLS_CERT_PATH: z.string().optional(),
@@ -31,8 +32,10 @@ const EnvSchema = z.object({
       "ENCRYPTION_KEY must be 256 bits, 64 string characters in hex format, generate via: openssl rand -hex 32",
     )
     .optional(),
-  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("false"),
-  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(60 * 60),
+  LANGFUSE_CACHE_MODEL_MATCH_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_MODEL_MATCH_TTL_SECONDS: z.coerce.number().default(86400), // 24 hours
+  LANGFUSE_CACHE_PROMPT_ENABLED: z.enum(["true", "false"]).default("true"),
+  LANGFUSE_CACHE_PROMPT_TTL_SECONDS: z.coerce.number().default(300), // 5 minutes
   CLICKHOUSE_URL: z.string().url(),
   CLICKHOUSE_CLUSTER_NAME: z.string().default("default"),
   CLICKHOUSE_DB: z.string().default("default"),
@@ -107,6 +110,68 @@ const EnvSchema = z.object({
   LANGFUSE_CLICKHOUSE_DELETION_TIMEOUT_MS: z.coerce.number().default(240_000), // 4 minutes
   LANGFUSE_CLICKHOUSE_QUERY_MAX_ATTEMPTS: z.coerce.number().default(3), // Maximum attempts for socket hang up errors
   LANGFUSE_SKIP_S3_LIST_FOR_OBSERVATIONS_PROJECT_IDS: z.string().optional(),
+  // Dataset Run Items Migration Environment Variables
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS: z
+    .string()
+    .optional()
+    .transform((s) =>
+      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
+    ),
+  LANGFUSE_EXPERIMENT_SAMPLING_RATE: z.coerce
+    .number()
+    .min(0)
+    .max(1)
+    .default(0.1),
+  LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS: z
+    .string()
+    .optional()
+    .transform((s) =>
+      s ? s.split(",").map((s) => s.toLowerCase().trim()) : [],
+    ),
+  LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT: z
+    .enum(["true", "false"])
+    .default("false"),
+  LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS: z
+    .string()
+    .optional()
+    .transform((val) => {
+      try {
+        if (!val) return new Map<string, number>();
+
+        const map = new Map<string, number>();
+        const parts = val.split(",");
+
+        for (const part of parts) {
+          const [projectId, sampleRateStr] = part.split(":");
+
+          if (!projectId || sampleRateStr === undefined) {
+            throw new Error(`Invalid format: ${part}`);
+          }
+
+          // Validate sample rate is between 0 and 1
+          const sampleRate = z.coerce
+            .number()
+            .min(0)
+            .max(1)
+            .parse(sampleRateStr);
+
+          map.set(projectId, sampleRate);
+        }
+
+        return map;
+      } catch (err) {
+        return new Map<string, number>();
+      }
+    }),
 });
 
 export const env: z.infer<typeof EnvSchema> =

packages/shared/src/errors/BaseError.ts

@@ -7,7 +7,7 @@ export class BaseError extends Error {
     name: string,
     httpCode: number,
     description: string,
-    isOperational: boolean
+    isOperational: boolean,
   ) {
     super(description);
     Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain

packages/shared/src/features/annotation/types.ts

@@ -1,5 +1,6 @@
 import z from "zod/v4";
 import { type ScoreDataType } from "../../db";
+import { StringNoHTML, StringNoHTMLNonEmpty } from "../../utils/zod";
 
 const NUMERIC: ScoreDataType = "NUMERIC";
 const CATEGORICAL: ScoreDataType = "CATEGORICAL";
@@ -47,12 +48,12 @@ export type ScoreTargetSession = z.infer<typeof ScoreTargetSession>;
 export type ScoreTarget = z.infer<typeof ScoreTarget>;
 
 const CreateAnnotationScoreBase = z.object({
-  name: z.string(),
+  name: StringNoHTMLNonEmpty,
   projectId: z.string(),
   environment: z.string().default("default"),
   scoreTarget: ScoreTarget,
   configId: z.string().optional(),
-  comment: z.string().nullish(),
+  comment: StringNoHTML.nullish(),
   queueId: z.string().nullish(),
 });
 
@@ -83,8 +84,8 @@ export const UpdateAnnotationScoreData = z.discriminatedUnion("dataType", [
 // annotation queues
 
 export const CreateQueueData = z.object({
-  name: z.string().min(1).max(35),
-  description: z.string().max(1000).optional(),
+  name: StringNoHTMLNonEmpty.max(35),
+  description: StringNoHTML.max(1000).optional(),
   scoreConfigIds: z.array(z.string()).min(1, {
     message: "At least 1 score config must be selected",
   }),

packages/shared/src/features/batchAction/types.ts

@@ -14,6 +14,7 @@ const ActionIdSchema = z.enum([
   "score-delete",
   "trace-delete",
   "trace-add-to-annotation-queue",
+  "session-add-to-annotation-queue",
 ]);
 
 export type ActionId = z.infer<typeof ActionIdSchema>;

packages/shared/src/features/prompts/validation.ts

@@ -1,12 +1,12 @@
-import { z } from "zod/v4";
+import { StringNoHTMLNonEmpty } from "../../utils/zod";
 
 /**
  * Prompt name validation schema for API, tRPC and client
  */
-export const PromptNameSchema = z
-  .string()
-  .min(1, "Enter a name")
-  .regex(/^[^|]*$/, "Prompt name cannot contain '|' character")
+export const PromptNameSchema = StringNoHTMLNonEmpty.regex(
+  /^[^|]*$/,
+  "Prompt name cannot contain '|' character",
+)
   .regex(/^[^/]/, "Name cannot start with a slash")
   .regex(/^(?!.*\/\/)/, "Name cannot contain consecutive slashes")
   .regex(/^.*[^/]$/, "Name cannot end with a slash")

packages/shared/src/index.ts

@@ -17,6 +17,7 @@ export * from "./utils/prompts";
 export * from "./features/entitlements/plans";
 export * from "./interfaces/rate-limits";
 export * from "./tableDefinitions/typeHelpers";
+export * from "./domain/webhooks";
 
 // llm api
 export * from "./server/llm/types";
@@ -46,7 +47,13 @@ export * from "./features/prompts/parsePromptDependencyTags";
 export * from "./features/prompts/validation";
 export * from "./features/prompts/types";
 export * from "./features/prompts/constants";
-export { compileChatMessages, compileChatMessagesWithIds, isPlaceholder, type MessagePlaceholderValues, type PromptMessage as ServerPromptMessage } from "./server/llm/compileChatMessages";
+export {
+  compileChatMessages,
+  compileChatMessagesWithIds,
+  isPlaceholder,
+  type MessagePlaceholderValues,
+  type PromptMessage as ServerPromptMessage,
+} from "./server/llm/compileChatMessages";
 
 // export db types only
 export * from "@prisma/client";

packages/shared/src/interfaces/customLLMProviderConfigSchemas.ts

@@ -1,5 +1,9 @@
 import { z } from "zod/v4";
 
+// Sentinel value for Bedrock default credential provider chain
+export const BEDROCK_USE_DEFAULT_CREDENTIALS =
+  "__BEDROCK_DEFAULT_CREDENTIALS__";
+
 export const BedrockConfigSchema = z.object({ region: z.string() });
 export type BedrockConfig = z.infer<typeof BedrockConfigSchema>;
 

packages/shared/src/server/auth/customSsoProvider.ts

@@ -12,9 +12,8 @@ interface CustomSSOUser extends Record<string, any> {
   verified: boolean;
 }
 
-
 export function CustomSSOProvider<P extends CustomSSOUser>(
-  options: OAuthUserConfig<P>
+  options: OAuthUserConfig<P>,
 ): OAuthConfig<P> {
   return {
     id: "custom",

packages/shared/src/server/clickhouse/measureAndReturn.ts

@@ -0,0 +1,98 @@
+import { instrumentAsync } from "../instrumentation";
+import * as opentelemetry from "@opentelemetry/api";
+import { env } from "../../env";
+import { logger } from "../logger";
+
+const executionWrapper = async <T, Y>(
+  input: T,
+  fn: (input: T) => Promise<Y>, // eslint-disable-line no-unused-vars
+  span?: opentelemetry.Span,
+  attributePrefix?: string,
+): Promise<[Y, number]> => {
+  const startTime = Date.now();
+  const res = await fn(input);
+  const duration = Date.now() - startTime;
+  span?.setAttribute(
+    `langfuse.experiment.amts.${attributePrefix}-duration`,
+    duration,
+  );
+  return [res, duration];
+};
+
+export const measureAndReturn = async <T, Y>(args: {
+  operationName: string;
+  projectId: string;
+  input: T;
+  existingExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
+  newExecution: (input: T) => Promise<Y>; // eslint-disable-line no-unused-vars
+}): Promise<Y> => {
+  return instrumentAsync(
+    {
+      name: `experiment-${args.operationName}`,
+      spanKind: opentelemetry.SpanKind.CLIENT,
+    },
+    async (currentSpan) => {
+      const { input, existingExecution, newExecution } = args;
+
+      if (
+        env.LANGFUSE_EXPERIMENT_COMPARE_READ_FROM_AGGREGATING_MERGE_TREES !==
+        "true"
+      ) {
+        currentSpan.setAttribute(`langfuse.experiment.amts.run`, "disabled");
+        return existingExecution(input);
+      }
+
+      // If not whitelisted, apply sampling logic
+      if (
+        !env.LANGFUSE_EXPERIMENT_WHITELISTED_PROJECT_IDS.includes(
+          args.projectId,
+        ) &&
+        Math.random() > env.LANGFUSE_EXPERIMENT_SAMPLING_RATE
+      ) {
+        currentSpan.setAttribute(`langfuse.experiment.amts.run`, "sampled-out");
+        return existingExecution(input);
+      }
+
+      currentSpan.setAttribute(`langfuse.experiment.amts.run`, "true");
+
+      try {
+        const [[existingResult, existingDuration], [newResult, newDuration]] =
+          await Promise.all([
+            executionWrapper(input, existingExecution, currentSpan, "existing"),
+            executionWrapper(input, newExecution, currentSpan, "new"),
+          ]);
+        // Positive duration difference means new is faster
+        const durationDifference = existingDuration - newDuration;
+        currentSpan?.setAttribute(
+          "langfuse.experiment.amts.execution-time-difference",
+          durationDifference,
+        );
+
+        if (
+          env.LANGFUSE_EXPERIMENT_ADD_QUERY_RESULT_TO_SPAN_PROJECT_IDS.some(
+            (p) => p === args.projectId,
+          )
+        ) {
+          currentSpan?.setAttribute(
+            "langfuse.experiment.amts.existing-result",
+            JSON.stringify(existingResult),
+          );
+          currentSpan?.setAttribute(
+            "langfuse.experiment.amts.new-result",
+            JSON.stringify(newResult),
+          );
+        }
+
+        return env.LANGFUSE_EXPERIMENT_RETURN_NEW_RESULT === "true"
+          ? newResult
+          : existingResult;
+      } catch (e) {
+        logger.error(
+          "Failed to run experiment wrapper. Retrying existing query",
+          e,
+        );
+        return existingExecution(input);
+      }
+    },
+  );
+};

packages/shared/src/server/clickhouse/schemaUtils.ts

@@ -11,7 +11,8 @@ export type IngestionEntityTypes =
   | "trace"
   | "observation"
   | "score"
-  | "sdk_log";
+  | "sdk_log"
+  | "dataset_run_item";
 
 export const getClickhouseEntityType = (
   eventType: string,
@@ -29,6 +30,8 @@ export const getClickhouseEntityType = (
       return "observation";
     case eventTypes.SCORE_CREATE:
       return "score";
+    case eventTypes.DATASET_RUN_ITEM_CREATE:
+      return "dataset_run_item";
     case eventTypes.SDK_LOG:
       return "sdk_log";
     default:

packages/shared/src/server/dataset-run-items/addToDeleteQueue.ts

@@ -0,0 +1,34 @@
+import { DatasetDeleteQueue } from "../redis/datasetDelete";
+import { QueueJobs } from "../queues";
+import { redis } from "../redis/redis";
+import { randomUUID } from "crypto";
+
+type DatasetDeletionType = "dataset" | "dataset-runs";
+
+type DatasetDeletionPayload = {
+  deletionType: DatasetDeletionType;
+  projectId: string;
+  datasetId: string;
+  datasetRunIds?: string[];
+};
+
+export const addToDeleteDatasetQueue = async ({
+  deletionType,
+  projectId,
+  datasetId,
+  datasetRunIds = [],
+}: DatasetDeletionPayload) => {
+  if (redis) {
+    await DatasetDeleteQueue.getInstance()?.add(QueueJobs.DatasetDelete, {
+      payload: {
+        deletionType,
+        projectId,
+        datasetId,
+        datasetRunIds,
+      },
+      id: randomUUID(),
+      timestamp: new Date(),
+      name: QueueJobs.DatasetDelete,
+    });
+  }
+};

packages/shared/src/server/dataset-run-items/datasetExecution.ts

@@ -0,0 +1,94 @@
+import { env } from "../../env";
+import { logger } from "../../server/logger";
+import {
+  DatasetRunItemsExecutionStrategy,
+  DatasetRunItemsOperationType,
+} from "./types";
+/**
+ * Returns the execution strategy for dataset run items based on environment variables.
+ *
+ * Two-phase migration approach:
+ * 1. Dual-write phase: DATASET_RUN_ITEMS_WRITE_TO_CLICKHOUSE=true (write to both databases)
+ * 2. Read migration phase: DATASET_RUN_ITEMS_READ_FROM_CLICKHOUSE=true (read from ClickHouse)
+ */
+function getDatasetRunItemsExecutionStrategy(): DatasetRunItemsExecutionStrategy {
+  return {
+    shouldWriteToClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_WRITE_CH === "true",
+    shouldReadFromClickHouse:
+      env.LANGFUSE_EXPERIMENT_DATASET_RUN_ITEMS_READ_CH === "true",
+  };
+}
+
+// Re-export the enum for backward compatibility
+
+/**
+ * Executes the appropriate database operation based on the execution strategy.
+ *
+ * @param postgresExecution - Function to execute PostgreSQL operation
+ * @param clickhouseExecution - Function to execute ClickHouse operation
+ * @param operationType - Type of operation ("read" or "write")
+ * @returns Result from the selected execution strategy
+ */
+export async function executeWithDatasetRunItemsStrategy<TInput, TOutput>({
+  input,
+  operationType,
+  postgresExecution,
+  clickhouseExecution,
+}: {
+  input: TInput;
+  operationType: DatasetRunItemsOperationType;
+  // eslint-disable-next-line no-unused-vars
+  postgresExecution: (input: TInput) => Promise<TOutput>;
+  // eslint-disable-next-line no-unused-vars
+  clickhouseExecution: (input: TInput) => Promise<TOutput>;
+}): Promise<TOutput> {
+  const strategy = getDatasetRunItemsExecutionStrategy();
+
+  if (operationType === DatasetRunItemsOperationType.WRITE) {
+    // For write operations, implement dual-write strategy
+    if (strategy.shouldWriteToClickHouse) {
+      // Dual-write phase: write to both databases
+      const postgresResult = await postgresExecution(input);
+
+      try {
+        await clickhouseExecution(input);
+        logger.debug("Successfully wrote to both PostgreSQL and ClickHouse", {
+          operation: `dataset_run_items_${operationType}`,
+        });
+      } catch (error) {
+        logger.error("ClickHouse write failed during dual-write phase", {
+          error: error instanceof Error ? error.message : String(error),
+          operation: `dataset_run_items_${operationType}`,
+        });
+        // Continue with PostgreSQL result since it succeeded
+      }
+
+      return postgresResult;
+    } else {
+      // Write only to PostgreSQL
+      return await postgresExecution(input);
+    }
+  } else {
+    // For read operations, rely on the strategy
+    const shouldExecuteClickhouse = strategy.shouldReadFromClickHouse;
+
+    if (shouldExecuteClickhouse) {
+      try {
+        return await clickhouseExecution(input);
+      } catch (error) {
+        logger.error(
+          "ClickHouse execution failed, falling back to PostgreSQL",
+          {
+            error: error instanceof Error ? error.message : String(error),
+            operation: `dataset_run_items_${operationType}`,
+          },
+        );
+        // Fallback to PostgreSQL for reliability
+        return await postgresExecution(input);
+      }
+    } else {
+      return await postgresExecution(input);
+    }
+  }
+}

packages/shared/src/server/dataset-run-items/types.ts

@@ -0,0 +1,16 @@
+/**
+ * Types and enums for dataset run items execution.
+ * This file is frontend-safe and doesn't import server-side dependencies.
+ */
+
+export enum DatasetRunItemsOperationType {
+  // eslint-disable-next-line no-unused-vars
+  READ = "read",
+  // eslint-disable-next-line no-unused-vars
+  WRITE = "write",
+}
+
+export type DatasetRunItemsExecutionStrategy = {
+  shouldWriteToClickHouse: boolean;
+  shouldReadFromClickHouse: boolean;
+};

packages/shared/src/server/index.ts

@@ -3,6 +3,7 @@ export * from "./services/email/organizationInvitation/sendMembershipInvitationE
 export * from "./services/email/batchExportSuccess/sendBatchExportSuccessEmail";
 export * from "./services/email/passwordReset/sendResetPasswordVerificationRequest";
 export * from "./services/PromptService";
+export * from "./services/PromptService/types";
 export * from "./services/traces-ui-table-service";
 export * from "./services/InMemoryFilterService";
 export * from "./auth/apiKeys";
@@ -19,17 +20,19 @@ export * from "./clickhouse/schemaUtils";
 export * from "./clickhouse/schema";
 export * from "./repositories/definitions";
 export * from "../server/ingestion/types";
+export * from "../server/ingestion/modelMatch";
 export * from "./ingestion/processEventBatch";
-export * from "../server/ingestion/types";
 export * from "../server/ingestion/validateAndInflateScore";
 export * from "./redis/redis";
 export * from "./redis/traceUpsert";
 export * from "./redis/createEvalQueue";
 export * from "./redis/cloudUsageMeteringQueue";
 export * from "./redis/getQueue";
+export * from "./redis/webhookQueue";
 export * from "./redis/traceDelete";
 export * from "./redis/projectDelete";
 export * from "./redis/scoreDelete";
+export * from "./redis/datasetDelete";
 export * from "./redis/datasetRunItemUpsert";
 export * from "./redis/batchExport";
 export * from "./redis/batchActionQueue";
@@ -44,6 +47,7 @@ export * from "./redis/coreDataS3ExportQueue";
 export * from "./redis/meteringDataPostgresExportQueue";
 export * from "./redis/experimentCreateQueue";
 export * from "./redis/dlqRetryQueue";
+export * from "./redis/entityChangeQueue";
 export * from "./auth/types";
 export * from "./queues";
 export * from "./orderByToPrisma";
@@ -59,9 +63,15 @@ export * from "./services/datasets-ui-table-service";
 export * from "./services/DashboardService";
 export * from "./services/TableViewService";
 export * from "./services/DefaultEvaluationModelService";
-
+export * from "./clickhouse/measureAndReturn";
 export * from "./data-deletion/ingestionFileDeletion";
 export * from "./s3";
 
+// dataset run items
+export * from "./dataset-run-items/datasetExecution";
+export * from "./dataset-run-items/types";
+export * from "./dataset-run-items/addToDeleteQueue";
+
 // test utils
 export * from "./test-utils";
+export * from "./utils/headerUtils";

packages/shared/src/server/ingestion/modelMatch.ts

@@ -1,13 +1,15 @@
-import { Model, Prisma } from "@langfuse/shared";
+import { Model, Prisma } from "../../";
 import {
   instrumentAsync,
   logger,
   recordIncrement,
-} from "@langfuse/shared/src/server";
-import { env } from "../env";
-import { redis } from "@langfuse/shared/src/server";
+  redis,
+  safeMultiDel,
+} from "../";
+import { type Cluster } from "ioredis";
+import { env } from "../../env";
 import { Decimal } from "decimal.js";
-import { prisma } from "@langfuse/shared/src/db";
+import { prisma } from "../../db";
 
 export type ModelMatchProps = {
   projectId: string;
@@ -81,7 +83,7 @@ const getModelFromRedis = async (
     if (redisModel) {
       recordIncrement("langfuse.model_match.cache_hit", 1);
       if (redisModel === NOT_FOUND_TOKEN) {
-        return null;
+        return NOT_FOUND_TOKEN;
       }
       const model = redisModelToPrismaModel(redisModel);
       return model;
@@ -178,6 +180,11 @@ export const getRedisModelKey = (p: ModelMatchProps) => {
 };
 
 const getModelMatchKeyPrefix = () => {
+  if (env.REDIS_CLUSTER_ENABLED === "true") {
+    // Use hash tags for Redis cluster compatibility
+    // This ensures all model cache keys are placed on the same hash slot
+    return "{model-match}";
+  }
   return "model-match";
 };
 
@@ -205,3 +212,35 @@ export const redisModelToPrismaModel = (redisModel: string): Model => {
         : null,
   };
 };
+
+export async function clearModelCacheForProject(
+  projectId: string,
+): Promise<void> {
+  if (env.LANGFUSE_CACHE_MODEL_MATCH_ENABLED === "false" || !redis) {
+    return;
+  }
+
+  try {
+    const pattern = `${getModelMatchKeyPrefix()}:${projectId}:*`;
+
+    const keys =
+      env.REDIS_CLUSTER_ENABLED === "true"
+        ? (
+            await Promise.all(
+              (redis as Cluster)
+                .nodes("master")
+                .map((node) => node.keys(pattern) || []),
+            )
+          ).flat()
+        : await redis.keys(pattern);
+
+    if (keys.length > 0) {
+      await safeMultiDel(redis, keys);
+      logger.info(
+        `Cleared ${keys.length} model cache entries for project ${projectId}`,
+      );
+    }
+  } catch (error) {
+    logger.error(`Error clearing model cache for project ${projectId}`, error);
+  }
+}

packages/shared/src/server/ingestion/processEventBatch.ts

@@ -21,11 +21,16 @@ import { logger } from "../logger";
 import { QueueJobs } from "../queues";
 import { IngestionQueue } from "../redis/ingestionQueue";
 import { redis } from "../redis/redis";
-import { eventTypes, ingestionEvent, IngestionEventType } from "./types";
+import {
+  eventTypes,
+  createIngestionEventSchema,
+  IngestionEventType,
+} from "./types";
 import {
   StorageService,
   StorageServiceFactory,
 } from "../services/StorageService";
+import { isTraceIdInSample } from "./sampling";
 
 let s3StorageServiceClient: StorageService;
 
@@ -45,7 +50,8 @@ const getS3StorageServiceClient = (bucketName: string): StorageService => {
   return s3StorageServiceClient;
 };
 
-export type TokenCountDelegate = (p: { // eslint-disable-line no-unused-vars
+// eslint-disable-next-line no-unused-vars
+export type TokenCountDelegate = (p: {
   model: Model;
   text: unknown;
 }) => number | undefined;
@@ -56,7 +62,7 @@ export type TokenCountDelegate = (p: { // eslint-disable-line no-unused-vars
  * We need the delay around date boundaries to avoid duplicates for out-of-order processing of events.
  * @param delay - Delay overwrite. Used if non-null.
  */
-const getDelay = (delay: number | null) => {
+const getDelay = (delay: number | null, source: "api" | "otel") => {
   if (delay !== null) {
     return delay;
   }
@@ -68,24 +74,38 @@ const getDelay = (delay: number | null) => {
     return env.LANGFUSE_INGESTION_QUEUE_DELAY_MS;
   }
 
+  if (source === "otel") {
+    return 0;
+  }
+
   // Use 5s here to avoid duplicate processing on the worker. If the ingestion delay is set to a lower value,
   // we use this instead.
   // Values should be revisited based on a cost/performance trade-off.
   return Math.min(5000, env.LANGFUSE_INGESTION_QUEUE_DELAY_MS);
 };
 
+/**
+ * Options for event batch processing.
+ * @property delay - Delay in ms to wait before processing events in the batch.
+ * @property source - Source of the events for metrics tracking (e.g., "otel", "api").
+ * @property isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ */
+type ProcessEventBatchOptions = {
+  delay?: number | null;
+  source?: "api" | "otel";
+  isLangfuseInternal?: boolean;
+};
+
 /**
  * Processes a batch of events.
  * @param input - Batch of IngestionEventType. Will validate the types first thing and return errors if they are invalid.
  * @param authCheck - AuthHeaderValidVerificationResult
- * @param delay - (Optional) Delay in ms to wait before processing events in the batch.
- * @param source - (Optional) Source of the events for metrics tracking (e.g., "otel", "api").
+ * @param options - (Optional) Options for the event batch processing.
  */
 export const processEventBatch = async (
   input: unknown[],
   authCheck: AuthHeaderValidVerificationResult,
-  delay: number | null = null,
-  source: "api" | "otel" = "api",
+  options: ProcessEventBatchOptions = {},
 ): Promise<{
   successes: { id: string; status: number }[];
   errors: {
@@ -95,6 +115,8 @@ export const processEventBatch = async (
     error?: string;
   }[];
 }> => {
+  const { delay = null, source = "api", isLangfuseInternal = false } = options;
+
   // add context of api call to the span
   const currentSpan = getCurrentSpan();
   recordIncrement("langfuse.ingestion.event", input.length, { source });
@@ -120,9 +142,10 @@ export const processEventBatch = async (
   const validationErrors: { id: string; error: unknown }[] = [];
   const authenticationErrors: { id: string; error: unknown }[] = [];
 
-  const batch: z.infer<typeof ingestionEvent>[] = input
+  const ingestionSchema = createIngestionEventSchema(isLangfuseInternal);
+  const batch: z.infer<typeof ingestionSchema>[] = input
     .flatMap((event) => {
-      const parsed = ingestionEvent.safeParse(event);
+      const parsed = ingestionSchema.safeParse(event);
       if (!parsed.success) {
         validationErrors.push({
           id:
@@ -238,11 +261,39 @@ export const processEventBatch = async (
       const shardingKey = `${authCheck.scope.projectId}-${eventData.eventBodyId}`;
       const queue = IngestionQueue.getInstance({ shardingKey });
 
-      const shouldSkipS3List =
-        getClickhouseEntityType(eventData.type) === "observation" &&
+      const isDatasetRunItemEvent =
+        getClickhouseEntityType(eventData.type) === "dataset_run_item";
+      const isObservationEvent =
+        getClickhouseEntityType(eventData.type) === "observation";
+
+      const isOtelOrSkipS3Project =
         authCheck.scope.projectId !== null &&
-        (projectIdsToSkipS3List.includes(authCheck.scope.projectId) ||
-          source === "otel");
+        (source === "otel" ||
+          projectIdsToSkipS3List.includes(authCheck.scope.projectId));
+
+      const shouldSkipS3List =
+        isDatasetRunItemEvent || (isObservationEvent && isOtelOrSkipS3Project);
+
+      const { isSampled, isSamplingConfigured } = isTraceIdInSample({
+        projectId: authCheck.scope.projectId,
+        event: eventData.data[0],
+      });
+
+      if (!isSampled) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "out",
+        });
+
+        return;
+      }
+
+      if (isSamplingConfigured) {
+        recordIncrement("langfuse.ingestion.sampling", eventData.data.length, {
+          projectId: authCheck.scope.projectId ?? "<not set>",
+          sampling_decision: "in",
+        });
+      }
 
       return queue
         ? queue.add(
@@ -267,7 +318,7 @@ export const processEventBatch = async (
                 },
               },
             },
-            { delay: getDelay(delay) },
+            { delay: getDelay(delay, source) },
           )
         : Promise.reject("Failed to instantiate queue");
     }),
@@ -301,7 +352,7 @@ const isAuthorized = (
 /**
  * Sorts a batch of ingestion events. Orders by: updating events last, sorted by timestamp asc.
  */
-const sortBatch = (batch: Array<z.infer<typeof ingestionEvent>>) => {
+const sortBatch = (batch: IngestionEventType[]) => {
   const updateEvents: (typeof eventTypes)[keyof typeof eventTypes][] = [
     eventTypes.GENERATION_UPDATE,
     eventTypes.SPAN_UPDATE,

packages/shared/src/server/ingestion/sampling.ts

@@ -0,0 +1,59 @@
+import crypto from "node:crypto";
+import { logger } from "../logger";
+import { env } from "../../env";
+import { IngestionEventType } from "./types";
+
+export function isTraceIdInSample(params: {
+  projectId: string | null;
+  event: IngestionEventType;
+}): { isSampled: boolean; isSamplingConfigured: boolean } {
+  const { projectId, event } = params;
+
+  const sampledProjects = env.LANGFUSE_INGESTION_PROCESSING_SAMPLED_PROJECTS;
+
+  if (!projectId || !sampledProjects.has(projectId))
+    return { isSampled: true, isSamplingConfigured: false };
+
+  const sampleRate = sampledProjects.get(projectId);
+  if (sampleRate === undefined)
+    return { isSampled: true, isSamplingConfigured: true };
+
+  const traceId = parseTraceId(event);
+  if (!traceId) return { isSampled: true, isSamplingConfigured: true };
+
+  return {
+    isSampled: isInSample(traceId, sampleRate),
+    isSamplingConfigured: true,
+  };
+}
+
+function isInSample(traceId: string, sampleRate: number) {
+  if (sampleRate < 0 || sampleRate > 1) {
+    logger.error(`Invalid sample rate ${sampleRate}`);
+
+    // Be conservative and keep the trace ID in sample for invalid configs
+    return true;
+  }
+
+  if (sampleRate === 0) return false;
+  if (sampleRate === 1) return true;
+
+  // Create SHA-256 hash of the input
+  const hash = crypto.createHash("sha256").update(traceId).digest("hex");
+
+  // Take first 8 characters and convert to integer
+  // Equivalent to 4 bytes, 32 bit integer
+  const hashInt = parseInt(hash.substring(0, 8), 16);
+
+  // Convert to a value between 0 and 1 by dividing by largest integer
+  const normalizedHash = hashInt / 0xffffffff;
+
+  // Return true if normalized hash is less than sample rate
+  return normalizedHash < sampleRate;
+}
+
+function parseTraceId(event: IngestionEventType): string | null | undefined {
+  if (event.type === "trace-create") return event.body.id;
+
+  return "traceId" in event.body ? event.body.traceId : null;
+}

packages/shared/src/server/ingestion/types.ts

@@ -193,178 +193,42 @@ export const UsageDetails = z
   ])
   .nullish();
 
-export const EnvironmentName = z
+const INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  "Only alphanumeric lower case characters, hyphens, and underscores are allowed";
+
+const ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE =
+  INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE +
+  " and it must not start with 'langfuse'";
+
+const PublicEnvironmentName = z
   .string()
   .max(40, "Maximum length is 40 characters")
-  .regex(
-    /^(?!langfuse)[a-z0-9-_]+$/,
-    "Only alphanumeric lower case characters, hyphens, and underscores are allowed, and it must not start with 'langfuse'",
-  )
+  .regex(/^(?!langfuse)[a-z0-9-_]+$/, ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
   .default("default");
 
-// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
-// With this setup parsing should be more lightweight and doesn't block other requests.
-// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
-export const TraceBody = z.object({
-  id: idSchema.nullish(),
-  timestamp: stringDateTime,
-  name: z.string().max(1000).nullish(),
-  externalId: z.string().nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  sessionId: z.string().nullish(),
-  userId: z.string().nullish(),
-  environment: EnvironmentName,
-  metadata: jsonSchema.nullish(),
-  release: z.string().nullish(),
-  version: z.string().nullish(),
-  public: z.boolean().nullish(),
-  tags: z.array(z.string()).nullish(),
-});
+const InternalEnvironmentName = z
+  .string()
+  .max(40, "Maximum length is 40 characters")
+  .regex(/^[a-z0-9-_]+$/, INTERNAL_ENVIRONMENT_NAME_REGEX_ERROR_MESSAGE)
+  .default("default");
 
-export const OptionalObservationBody = z.object({
-  traceId: idSchema.nullish(),
-  environment: EnvironmentName,
-  name: z.string().nullish(),
-  startTime: stringDateTime,
-  metadata: jsonSchema.nullish(),
-  input: z.any().nullish(),
-  output: z.any().nullish(),
-  level: ObservationLevel.nullish(),
-  statusMessage: z.string().nullish(),
-  parentObservationId: z.string().nullish(),
-  version: z.string().nullish(),
-});
+/** @deprecated Use PublicEnvironmentName or InternalEnvironmentName instead */
+export const EnvironmentName = PublicEnvironmentName;
 
-export const CreateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const UpdateEventEvent = OptionalObservationBody.extend({
-  id: idSchema,
-});
-
-export const CreateSpanBody = CreateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const UpdateSpanBody = UpdateEventEvent.extend({
-  endTime: stringDateTime,
-});
-
-export const CreateGenerationBody = CreateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-export const UpdateGenerationBody = UpdateSpanBody.extend({
-  completionStartTime: stringDateTime,
-  model: z.string().nullish(),
-  modelParameters: z
-    .record(
-      z.string(),
-      z
-        .union([
-          z.string(),
-          z.number(),
-          z.boolean(),
-          z.array(z.string()),
-          z.record(z.string(), z.string()),
-        ])
-        .nullish(),
-    )
-    .nullish(),
-  usage: usage,
-  usageDetails: UsageDetails,
-  costDetails: CostDetails,
-  promptName: z.string().nullish(),
-  promptVersion: z.number().int().nullish(),
-}).refine((value) => {
-  // ensure that either promptName and promptVersion are set, or none
-
-  if (!value.promptName && !value.promptVersion) return true;
-  if (value.promptName && value.promptVersion) return true;
-  return false;
-});
-
-const BaseScoreBody = z.object({
-  id: idSchema.nullish(),
-  name: NonEmptyString,
-  traceId: z.string().nullish(),
-  sessionId: z.string().nullish(),
-  datasetRunId: z.string().nullish(),
-  environment: EnvironmentName,
-  observationId: z.string().nullish(),
-  comment: z.string().nullish(),
-  metadata: jsonSchema.nullish(),
-  source: z
-    .enum(["API", "EVAL", "ANNOTATION"])
-    .default("API" as ScoreSourceType),
-});
-
-/**
- * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
- */
-export const ScoreBody = applyScoreValidation(
-  z.discriminatedUnion("dataType", [
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number(),
-        dataType: z.literal("NUMERIC"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.string(),
-        dataType: z.literal("CATEGORICAL"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.number().refine((value) => value === 0 || value === 1, {
-          message:
-            "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
-        }),
-        dataType: z.literal("BOOLEAN"),
-        configId: z.string().nullish(),
-      }),
-    ),
-    BaseScoreBody.merge(
-      z.object({
-        value: z.union([z.string(), z.number()]),
-        dataType: z.undefined(),
-        configId: z.string().nullish(),
-      }),
-    ),
-  ]),
-);
+export const eventTypes = {
+  TRACE_CREATE: "trace-create",
+  SCORE_CREATE: "score-create",
+  EVENT_CREATE: "event-create",
+  SPAN_CREATE: "span-create",
+  SPAN_UPDATE: "span-update",
+  GENERATION_CREATE: "generation-create",
+  GENERATION_UPDATE: "generation-update",
+  SDK_LOG: "sdk-log",
+  DATASET_RUN_ITEM_CREATE: "dataset-run-item-create",
+  // LEGACY, only required for backwards compatibility
+  OBSERVATION_CREATE: "observation-create",
+  OBSERVATION_UPDATE: "observation-update",
+} as const;
 
 // LEGACY, only required for backwards compatibility
 export const LegacySpanPostSchema = z.object({
@@ -475,83 +339,366 @@ export const SdkLogEvent = z.object({
   id: z.string().nullish(), // Not used, but makes downstream processing easier.
 });
 
-export const eventTypes = {
-  TRACE_CREATE: "trace-create",
-  SCORE_CREATE: "score-create",
-  EVENT_CREATE: "event-create",
-  SPAN_CREATE: "span-create",
-  SPAN_UPDATE: "span-update",
-  GENERATION_CREATE: "generation-create",
-  GENERATION_UPDATE: "generation-update",
-  SDK_LOG: "sdk-log",
-  // LEGACY, only required for backwards compatibility
-  OBSERVATION_CREATE: "observation-create",
-  OBSERVATION_UPDATE: "observation-update",
-} as const;
+// Using z.any instead of jsonSchema for input/output as we saw huge CPU overhead for large numeric arrays.
+// With this setup parsing should be more lightweight and doesn't block other requests.
+// As we allow plain values, arrays, and objects the JSON parse via bodyParser should suffice.
 
-const base = z.object({
-  id: idSchema,
-  timestamp: z.string().datetime({ offset: true }),
-  metadata: jsonSchema.nullish(),
-});
-export const traceEvent = base.extend({
-  type: z.literal(eventTypes.TRACE_CREATE),
-  body: TraceBody,
-});
-export type TraceEventType = z.infer<typeof traceEvent>;
+// Complete schema factory - single source of truth for ALL schemas
+const createAllIngestionSchemas = ({
+  isPublic = true,
+}: {
+  isPublic: boolean;
+}) => {
+  const environmentSchema = isPublic
+    ? PublicEnvironmentName
+    : InternalEnvironmentName;
 
-export const eventCreateEvent = base.extend({
-  type: z.literal(eventTypes.EVENT_CREATE),
-  body: CreateEventEvent,
-});
-export const spanCreateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_CREATE),
-  body: CreateSpanBody,
-});
-export const spanUpdateEvent = base.extend({
-  type: z.literal(eventTypes.SPAN_UPDATE),
-  body: UpdateSpanBody,
-});
-export const generationCreateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_CREATE),
-  body: CreateGenerationBody,
-});
-export const generationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.GENERATION_UPDATE),
-  body: UpdateGenerationBody,
-});
-export const scoreEvent = base.extend({
-  type: z.literal(eventTypes.SCORE_CREATE),
-  body: ScoreBody,
-});
-export type ScoreEventType = z.infer<typeof scoreEvent>;
-export const sdkLogEvent = base.extend({
-  type: z.literal(eventTypes.SDK_LOG),
-  body: SdkLogEvent,
-});
-export const legacyObservationCreateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_CREATE),
-  body: LegacyObservationBody,
-});
-export const legacyObservationUpdateEvent = base.extend({
-  type: z.literal(eventTypes.OBSERVATION_UPDATE),
-  body: LegacyObservationBody,
-});
+  // Base schemas with environment
+  const TraceBody = z.object({
+    id: idSchema.nullish(),
+    timestamp: stringDateTime,
+    name: z.string().max(1000).nullish(),
+    externalId: z.string().nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    sessionId: z.string().nullish(),
+    userId: z.string().nullish(),
+    environment: environmentSchema,
+    metadata: jsonSchema.nullish(),
+    release: z.string().nullish(),
+    version: z.string().nullish(),
+    public: z.boolean().nullish(),
+    tags: z.array(z.string()).nullish(),
+  });
 
-export const ingestionEvent = z.discriminatedUnion("type", [
-  traceEvent,
-  scoreEvent,
-  eventCreateEvent,
-  spanCreateEvent,
-  spanUpdateEvent,
-  generationCreateEvent,
-  generationUpdateEvent,
-  sdkLogEvent,
-  // LEGACY, only required for backwards compatibility
-  legacyObservationCreateEvent,
-  legacyObservationUpdateEvent,
-]);
+  const OptionalObservationBody = z.object({
+    traceId: idSchema.nullish(),
+    environment: environmentSchema,
+    name: z.string().nullish(),
+    startTime: stringDateTime,
+    metadata: jsonSchema.nullish(),
+    input: z.any().nullish(),
+    output: z.any().nullish(),
+    level: ObservationLevel.nullish(),
+    statusMessage: z.string().nullish(),
+    parentObservationId: z.string().nullish(),
+    version: z.string().nullish(),
+  });
+
+  // Derivative schemas
+  const CreateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const UpdateEventEvent = OptionalObservationBody.extend({
+    id: idSchema,
+  });
+
+  const CreateSpanBody = CreateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const UpdateSpanBody = UpdateEventEvent.extend({
+    endTime: stringDateTime,
+  });
+
+  const CreateGenerationBody = CreateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const UpdateGenerationBody = UpdateSpanBody.extend({
+    completionStartTime: stringDateTime,
+    model: z.string().nullish(),
+    modelParameters: z
+      .record(
+        z.string(),
+        z
+          .union([
+            z.string(),
+            z.number(),
+            z.boolean(),
+            z.array(z.string()),
+            z.record(z.string(), z.string()),
+          ])
+          .nullish(),
+      )
+      .nullish(),
+    usage: usage,
+    usageDetails: UsageDetails,
+    costDetails: CostDetails,
+    promptName: z.string().nullish(),
+    promptVersion: z.number().int().nullish(),
+  }).refine((value) => {
+    // ensure that either promptName and promptVersion are set, or none
+    if (!value.promptName && !value.promptVersion) return true;
+    if (value.promptName && value.promptVersion) return true;
+    return false;
+  });
+
+  const BaseScoreBody = z.object({
+    id: idSchema.nullish(),
+    name: NonEmptyString,
+    traceId: z.string().nullish(),
+    sessionId: z.string().nullish(),
+    datasetRunId: z.string().nullish(),
+    environment: environmentSchema,
+    observationId: z.string().nullish(),
+    comment: z.string().nullish(),
+    metadata: jsonSchema.nullish(),
+    source: z
+      .enum(["API", "EVAL", "ANNOTATION"])
+      .default("API" as ScoreSourceType),
+  });
+
+  const ScoreBody = applyScoreValidation(
+    z.discriminatedUnion("dataType", [
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number(),
+          dataType: z.literal("NUMERIC"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.string(),
+          dataType: z.literal("CATEGORICAL"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.number().refine((value) => value === 0 || value === 1, {
+            message:
+              "Value must be a number equal to either 0 or 1 for data type BOOLEAN",
+          }),
+          dataType: z.literal("BOOLEAN"),
+          configId: z.string().nullish(),
+        }),
+      ),
+      BaseScoreBody.merge(
+        z.object({
+          value: z.union([z.string(), z.number()]),
+          dataType: z.undefined(),
+          configId: z.string().nullish(),
+        }),
+      ),
+    ]),
+  );
+
+  const DatasetRunItemBody = z.object({
+    // Core identifiers
+    id: idSchema.nullish(),
+    traceId: z.string(),
+    observationId: z.string().nullish(),
+    error: z.string().nullish(),
+    // Metadata (optional)
+    createdAt: stringDateTime.nullish(),
+    // Dataset identification
+    datasetId: z.string(),
+    // Run identification
+    runId: z.string(),
+    // Dataset item identification
+    datasetItemId: z.string(),
+  });
+
+  // Event schemas
+  const base = z.object({
+    id: idSchema,
+    timestamp: z.string().datetime({ offset: true }),
+    metadata: jsonSchema.nullish(),
+  });
+
+  const traceEvent = base.extend({
+    type: z.literal(eventTypes.TRACE_CREATE),
+    body: TraceBody,
+  });
+
+  const eventCreateEvent = base.extend({
+    type: z.literal(eventTypes.EVENT_CREATE),
+    body: CreateEventEvent,
+  });
+
+  const spanCreateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_CREATE),
+    body: CreateSpanBody,
+  });
+
+  const spanUpdateEvent = base.extend({
+    type: z.literal(eventTypes.SPAN_UPDATE),
+    body: UpdateSpanBody,
+  });
+
+  const generationCreateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_CREATE),
+    body: CreateGenerationBody,
+  });
+
+  const generationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.GENERATION_UPDATE),
+    body: UpdateGenerationBody,
+  });
+
+  const scoreEvent = base.extend({
+    type: z.literal(eventTypes.SCORE_CREATE),
+    body: ScoreBody,
+  });
+
+  const baseDatasetRunItemCreateEvent = base.extend({
+    type: z.literal(eventTypes.DATASET_RUN_ITEM_CREATE),
+    body: DatasetRunItemBody,
+  });
+
+  const datasetRunItemCreateEvent = isPublic
+    ? baseDatasetRunItemCreateEvent.refine(() => false, {
+        message: "Dataset run item creation is only allowed for internal usage",
+      })
+    : baseDatasetRunItemCreateEvent;
+
+  const sdkLogEvent = base.extend({
+    type: z.literal(eventTypes.SDK_LOG),
+    body: SdkLogEvent,
+  });
+
+  const legacyObservationCreateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_CREATE),
+    body: LegacyObservationBody,
+  });
+
+  const legacyObservationUpdateEvent = base.extend({
+    type: z.literal(eventTypes.OBSERVATION_UPDATE),
+    body: LegacyObservationBody,
+  });
+
+  const ingestionEvent = z.discriminatedUnion("type", [
+    traceEvent,
+    scoreEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    sdkLogEvent,
+    datasetRunItemCreateEvent,
+    // LEGACY, only required for backwards compatibility
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+  ]);
+
+  return {
+    // Body schemas
+    TraceBody,
+    OptionalObservationBody,
+    CreateEventEvent,
+    UpdateEventEvent,
+    CreateSpanBody,
+    UpdateSpanBody,
+    CreateGenerationBody,
+    UpdateGenerationBody,
+    BaseScoreBody,
+    ScoreBody,
+    // Event schemas
+    traceEvent,
+    eventCreateEvent,
+    spanCreateEvent,
+    spanUpdateEvent,
+    generationCreateEvent,
+    generationUpdateEvent,
+    scoreEvent,
+    datasetRunItemCreateEvent,
+    sdkLogEvent,
+    legacyObservationCreateEvent,
+    legacyObservationUpdateEvent,
+    // Complete schema
+    ingestionEvent,
+  };
+};
+
+// Create both public and internal schema instances
+const publicSchemas = createAllIngestionSchemas({ isPublic: true });
+const internalSchemas = createAllIngestionSchemas({ isPublic: false });
+
+// Export individual schemas for backwards compatibility
+export const TraceBody = publicSchemas.TraceBody;
+export const OptionalObservationBody = publicSchemas.OptionalObservationBody;
+export const CreateEventEvent = publicSchemas.CreateEventEvent;
+export const UpdateEventEvent = publicSchemas.UpdateEventEvent;
+export const CreateSpanBody = publicSchemas.CreateSpanBody;
+export const UpdateSpanBody = publicSchemas.UpdateSpanBody;
+export const CreateGenerationBody = publicSchemas.CreateGenerationBody;
+export const UpdateGenerationBody = publicSchemas.UpdateGenerationBody;
+export const BaseScoreBody = publicSchemas.BaseScoreBody;
+
+/**
+ * ScoreBody exactly mirrors `PostScoresBody` in the public API. Please refer there for source of truth.
+ */
+export const ScoreBody = publicSchemas.ScoreBody;
+
+// Export individual event schemas for backwards compatibility
+export const traceEvent = publicSchemas.traceEvent;
+export const eventCreateEvent = publicSchemas.eventCreateEvent;
+export const spanCreateEvent = publicSchemas.spanCreateEvent;
+export const spanUpdateEvent = publicSchemas.spanUpdateEvent;
+export const generationCreateEvent = publicSchemas.generationCreateEvent;
+export const generationUpdateEvent = publicSchemas.generationUpdateEvent;
+export const scoreEvent = publicSchemas.scoreEvent;
+export const sdkLogEvent = publicSchemas.sdkLogEvent;
+export const datasetRunItemCreateEvent =
+  publicSchemas.datasetRunItemCreateEvent;
+export const legacyObservationCreateEvent =
+  publicSchemas.legacyObservationCreateEvent;
+export const legacyObservationUpdateEvent =
+  publicSchemas.legacyObservationUpdateEvent;
+
+/** @deprecated Use createIngestionEventSchema() instead */
+export const ingestionEvent = publicSchemas.ingestionEvent;
+
+/**
+ * Type definitions for both schema variants (public and internal).
+ * These types are equivalent to the return types of createIngestionEventSchema() and all exported schemas,
+ * since the factory patterns only differ in environment validation rules, not in the actual TypeScript types.
+ * The environment field remains `string` in all cases - only the validation logic differs.
+ */
 export type IngestionEventType = z.infer<typeof ingestionEvent>;
+export type TraceEventType = z.infer<typeof traceEvent>;
+export type ScoreEventType = z.infer<typeof scoreEvent>;
+export type DatasetRunItemEventType = z.infer<typeof datasetRunItemCreateEvent>;
+
+/**
+ * Creates an ingestion event schema with appropriate environment validation.
+ * @param isLangfuseInternal - Whether the events are being ingested by Langfuse internally (e.g. traces created for prompt experiments).
+ * @returns The ingestion event schema.
+ */
+export const createIngestionEventSchema = (isLangfuseInternal = false) => {
+  return isLangfuseInternal
+    ? internalSchemas.ingestionEvent
+    : publicSchemas.ingestionEvent;
+};
 
 export type ObservationEvent =
   | z.infer<typeof legacyObservationCreateEvent>

packages/shared/src/server/llm/compileChatMessages.ts

@@ -1,21 +1,26 @@
 import { z } from "zod/v4";
 import { v4 as uuidv4 } from "uuid";
-import { type ChatMessage, type PlaceholderMessage, ChatMessageType, type PromptChatMessageSchema, type ChatMessageWithId, type ChatMessageWithIdNoPlaceholders, ChatMessageSchema } from "./types";
+import {
+  type ChatMessage,
+  type PlaceholderMessage,
+  ChatMessageType,
+  type PromptChatMessageSchema,
+  type ChatMessageWithId,
+  type ChatMessageWithIdNoPlaceholders,
+} from "./types";
 
-export type MessagePlaceholderValues = Record<string, ChatMessage[]>;
+export type MessagePlaceholderValues = Record<string, unknown[]>;
 export type PromptMessage = z.infer<typeof PromptChatMessageSchema>;
 
-export function isPlaceholder(message: PromptMessage): message is PlaceholderMessage {
+export function isPlaceholder(
+  message: PromptMessage,
+): message is PlaceholderMessage {
   return "type" in message && message.type === ChatMessageType.Placeholder;
 }
 
-function validateMessage(message: unknown): message is ChatMessage {
-  return ChatMessageSchema.safeParse(message).success;
-}
-
 function replaceTextVariables(
   content: string,
-  textVariables: Record<string, string>
+  textVariables: Record<string, string>,
 ): string {
   let result = content;
   for (const [varName, varValue] of Object.entries(textVariables)) {
@@ -28,35 +33,44 @@ function replaceTextVariables(
 
 function expandPlaceholder(
   placeholder: PlaceholderMessage,
-  placeholderValues: MessagePlaceholderValues
+  placeholderValues: MessagePlaceholderValues,
 ): ChatMessage[] {
   const replacementMessages = placeholderValues[placeholder.name];
 
   if (!replacementMessages) {
-    throw new Error(`Missing value for message placeholder: ${placeholder.name}`);
+    throw new Error(
+      `Missing value for message placeholder: ${placeholder.name}`,
+    );
   }
 
   if (!Array.isArray(replacementMessages)) {
-    throw new Error(`Placeholder value for '${placeholder.name}' must be an array of messages`);
+    throw new Error(
+      `Placeholder value for '${placeholder.name}' must be an array of messages`,
+    );
   }
 
-  for (const replacementMsg of replacementMessages) {
-    if (!validateMessage(replacementMsg)) {
-      throw new Error(`Invalid message format in placeholder '${placeholder.name}': messages must have 'role' and 'content' properties`);
+  // Allow arbitrary objects - just pass them through as ChatMessage
+  // Users might want to use ChatML with placeholders for any message structure
+  return replacementMessages.map((replacementMsg) => {
+    if (typeof replacementMsg === "object" && replacementMsg !== null) {
+      return replacementMsg as ChatMessage;
     }
-  }
-  return replacementMessages;
+
+    throw new Error(
+      `Invalid message in placeholder '${placeholder.name}': expected object but got ${typeof replacementMsg}`,
+    );
+  });
 }
 
 export function compileChatMessages(
   messages: PromptMessage[],
   placeholderValues: MessagePlaceholderValues,
-  textVariables?: Record<string, string>
+  textVariables?: Record<string, string>,
 ): ChatMessage[] {
   const expandedMessages = messages.flatMap((message) =>
     isPlaceholder(message)
       ? expandPlaceholder(message, placeholderValues)
-      : [message as ChatMessage]
+      : [message as ChatMessage],
   );
 
   // substitute text variables
@@ -65,27 +79,27 @@ export function compileChatMessages(
   }
 
   return expandedMessages.map((message) => {
-    if (!message.content) {
+    if (!message.content || typeof message.content !== "string") {
       return message;
     }
 
     return {
       ...message,
-      content: replaceTextVariables(message.content, textVariables)
+      content: replaceTextVariables(message.content, textVariables),
     };
   });
 }
 
 export function compileChatMessagesWithIds(
   messages: ChatMessageWithId[],
-  placeholderValues: Record<string, ChatMessage[]>,
-  textVariables?: Record<string, string>
+  placeholderValues: MessagePlaceholderValues,
+  textVariables?: Record<string, string>,
 ): ChatMessageWithIdNoPlaceholders[] {
   // TODO: check, is it even important to retain the IDs?
   const expandedMessages = messages.flatMap((message) => {
     if (isPlaceholder(message)) {
       const expandedMsgs = expandPlaceholder(message, placeholderValues);
-      return expandedMsgs.map(msg => ({ ...msg, id: uuidv4() }));
+      return expandedMsgs.map((msg) => ({ ...msg, id: uuidv4() }));
     } else {
       // Preserve message IDs for already non-placeholder messages
       return [message as ChatMessageWithIdNoPlaceholders];
@@ -98,19 +112,22 @@ export function compileChatMessagesWithIds(
   }
 
   return expandedMessages.map((message) => {
-    if (!message.content) {
+    if (!message.content || typeof message.content !== "string") {
       return message;
     }
 
     return {
       ...message,
-      content: replaceTextVariables(message.content, textVariables)
+      content: replaceTextVariables(message.content, textVariables),
     };
   });
 }
 
 export function extractPlaceholderNames(messages: PromptMessage[]): string[] {
   return messages
-    .filter((msg): msg is PlaceholderMessage => "type" in msg && msg.type === ChatMessageType.Placeholder)
-    .map(msg => msg.name);
+    .filter(
+      (msg): msg is PlaceholderMessage =>
+        "type" in msg && msg.type === ChatMessageType.Placeholder,
+    )
+    .map((msg) => msg.name);
 }

packages/shared/src/server/llm/fetchLLMCompletion.ts

@@ -19,10 +19,12 @@ import {
 } from "@langchain/core/output_parsers";
 import { IterableReadableStream } from "@langchain/core/utils/stream";
 import { ChatOpenAI, AzureChatOpenAI } from "@langchain/openai";
+import { env } from "../../env";
 import GCPServiceAccountKeySchema, {
   BedrockConfigSchema,
   BedrockCredentialSchema,
   VertexAIConfigSchema,
+  BEDROCK_USE_DEFAULT_CREDENTIALS,
 } from "../../interfaces/customLLMProviderConfigSchemas";
 import { processEventBatch } from "../ingestion/processEventBatch";
 import { logger } from "../logger";
@@ -41,6 +43,8 @@ import {
 import { CallbackHandler } from "langfuse-langchain";
 import type { BaseCallbackHandler } from "@langchain/core/callbacks/base";
 
+const isLangfuseCloud = Boolean(env.NEXT_PUBLIC_LANGFUSE_CLOUD_REGION);
+
 type ProcessTracedEvents = () => Promise<void>;
 
 type LLMCompletionParams = {
@@ -63,7 +67,8 @@ type FetchLLMCompletionParams = LLMCompletionParams & {
 };
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  params: LLMCompletionParams & {
     streaming: true;
   },
 ): Promise<{
@@ -72,7 +77,8 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  params: LLMCompletionParams & {
     streaming: false;
   },
 ): Promise<{
@@ -81,7 +87,8 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  params: LLMCompletionParams & {
     streaming: false;
     structuredOutputSchema: ZodSchema;
   },
@@ -91,7 +98,8 @@ export async function fetchLLMCompletion(
 }>;
 
 export async function fetchLLMCompletion(
-  params: LLMCompletionParams & { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  params: LLMCompletionParams & {
     tools: LLMToolDefinition[];
     streaming: false;
   },
@@ -133,7 +141,7 @@ export async function fetchLLMCompletion(
     const handler = new CallbackHandler({
       _projectId: traceParams.projectId,
       _isLocalEventExportEnabled: true,
-      tags: traceParams.tags,
+      environment: traceParams.environment,
     });
     finalCallbacks.push(handler);
 
@@ -145,6 +153,7 @@ export async function fetchLLMCompletion(
         await processEventBatch(
           JSON.parse(JSON.stringify(events)), // stringify to emulate network event batch from network call
           traceParams.authCheck,
+          { isLangfuseInternal: true },
         );
       } catch (e) {
         logger.error("Failed to process traced events", { error: e });
@@ -154,28 +163,48 @@ export async function fetchLLMCompletion(
 
   finalCallbacks = finalCallbacks.length > 0 ? finalCallbacks : undefined;
 
+  // Helper function to safely stringify content
+  const safeStringify = (content: any): string => {
+    try {
+      return JSON.stringify(content);
+    } catch {
+      return "[Unserializable content]";
+    }
+  };
+
   let finalMessages: BaseMessage[];
   // VertexAI requires at least 1 user message
   if (modelParams.adapter === LLMAdapter.VertexAI && messages.length === 1) {
-    finalMessages = [new HumanMessage(messages[0].content)];
+    const safeContent =
+      typeof messages[0].content === "string"
+        ? messages[0].content
+        : JSON.stringify(messages[0].content);
+    finalMessages = [new HumanMessage(safeContent)];
   } else {
     finalMessages = messages.map((message) => {
+      // For arbitrary content types, convert to string safely
+      const safeContent =
+        typeof message.content === "string"
+          ? message.content
+          : safeStringify(message.content);
+
       if (message.role === ChatMessageRole.User)
-        return new HumanMessage(message.content);
+        return new HumanMessage(safeContent);
       if (
         message.role === ChatMessageRole.System ||
         message.role === ChatMessageRole.Developer
       )
-        return new SystemMessage(message.content);
+        return new SystemMessage(safeContent);
 
-      if (message.type === ChatMessageType.ToolResult)
+      if (message.type === ChatMessageType.ToolResult) {
         return new ToolMessage({
-          content: message.content,
+          content: safeContent,
           tool_call_id: message.toolCallId,
         });
+      }
 
       return new AIMessage({
-        content: message.content,
+        content: safeContent,
         tool_calls:
           message.type === ChatMessageType.AssistantToolCall
             ? (message.toolCalls as any)
@@ -239,7 +268,11 @@ export async function fetchLLMCompletion(
     });
   } else if (modelParams.adapter === LLMAdapter.Bedrock) {
     const { region } = BedrockConfigSchema.parse(config);
-    const credentials = BedrockCredentialSchema.parse(JSON.parse(apiKey));
+    // Handle both explicit credentials and default provider chain
+    const credentials =
+      apiKey === BEDROCK_USE_DEFAULT_CREDENTIALS && !isLangfuseCloud
+        ? undefined // undefined = use AWS SDK default credential provider chain
+        : BedrockCredentialSchema.parse(JSON.parse(apiKey));
 
     chatModel = new ChatBedrockConverse({
       model: modelParams.model,
@@ -325,7 +358,7 @@ export async function fetchLLMCompletion(
 
     /*
   Workaround OpenAI reasoning models:
-  
+
   This is a temporary workaround to avoid sending unsupported parameters to OpenAI's O1 models.
   O1 models do not support:
   - system messages

packages/shared/src/server/llm/types.ts

@@ -176,7 +176,12 @@ export type ToolResultMessage = z.infer<typeof ToolResultMessageSchema>;
 
 export const PlaceholderMessageSchema = z.object({
   type: z.literal(ChatMessageType.Placeholder),
-  name: z.string().regex(/^[a-zA-Z][a-zA-Z0-9_]*$/, "Placeholder name must start with a letter and contain only alphanumeric characters and underscores"),
+  name: z
+    .string()
+    .regex(
+      /^[a-zA-Z][a-zA-Z0-9_]*$/,
+      "Placeholder name must start with a letter and contain only alphanumeric characters and underscores",
+    ),
 });
 export type PlaceholderMessage = z.infer<typeof PlaceholderMessageSchema>;
 
@@ -191,7 +196,7 @@ export const ChatMessageSchema = z.union([
   z
     .object({
       role: z.union([ChatMessageDefaultRoleSchema, z.string()]), // Users may ingest any string as role via API/SDK
-      content: z.string(),
+      content: z.union([z.string(), z.array(z.any()), z.any()]), // Support arbitrary content types for message placeholders
     })
     .transform((msg) => {
       return {
@@ -202,8 +207,10 @@ export const ChatMessageSchema = z.union([
 ]);
 
 export type ChatMessage = z.infer<typeof ChatMessageSchema>;
-export type ChatMessageWithId = (ChatMessage & { id: string }) | (PlaceholderMessage & { id: string });
-export type ChatMessageWithIdNoPlaceholders = (ChatMessage & { id: string });
+export type ChatMessageWithId =
+  | (ChatMessage & { id: string })
+  | (PlaceholderMessage & { id: string });
+export type ChatMessageWithIdNoPlaceholders = ChatMessage & { id: string };
 
 export const PromptChatMessageSchema = z.union([
   z.object({
@@ -226,11 +233,6 @@ export enum LLMAdapter {
   GoogleAIStudio = "google-ai-studio",
 }
 
-export const SYSTEM_ROLES: string[] = [
-  ChatMessageRole.System,
-  ChatMessageRole.Developer,
-];
-
 export const TextPromptContentSchema = z.string().min(1, "Enter a prompt");
 
 export const PromptContentSchema = z.union([
@@ -335,9 +337,12 @@ export const anthropicModels = [
 
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys
 export const vertexAIModels = [
-  "gemini-2.0-flash",
+  "gemini-2.5-pro",
+  "gemini-2.5-flash",
+  "gemini-2.5-flash-lite-preview-06-17",
   "gemini-2.5-pro-preview-05-06",
   "gemini-2.5-flash-preview-05-20",
+  "gemini-2.0-flash",
   "gemini-2.0-pro-exp-02-05",
   "gemini-2.0-flash-001",
   "gemini-2.0-flash-lite-preview-02-05",
@@ -349,6 +354,9 @@ export const vertexAIModels = [
 
 // WARNING: The first entry in the array is chosen as the default model to add LLM API keys. Make sure it supports top_p, max_tokens and temperature.
 export const googleAIStudioModels = [
+  "gemini-2.5-flash",
+  "gemini-2.5-pro",
+  "gemini-2.5-flash-lite-preview-06-17",
   "gemini-2.5-pro-preview-05-06",
   "gemini-2.5-flash-preview-05-20",
   "gemini-2.0-flash",
@@ -406,11 +414,17 @@ export type LLMApiKey =
     ? z.infer<typeof LLMApiKeySchema>
     : never;
 
+// NOTE: This string is whitelisted in the TS SDK to allow ingestion of traces by Langfuse. Please mirror edits to this string in https://github.com/langfuse/langfuse-js/blob/main/langfuse-core/src/index.ts.
+export const PROMPT_EXPERIMENT_ENVIRONMENT =
+  "langfuse-prompt-experiment" as const;
+
+type PromptExperimentEnvironment = typeof PROMPT_EXPERIMENT_ENVIRONMENT;
+
 export type TraceParams = {
   traceName: string;
   traceId: string;
   projectId: string;
-  tags: string[];
+  environment: PromptExperimentEnvironment;
   tokenCountDelegate: TokenCountDelegate;
   authCheck: AuthHeaderValidVerificationResult;
 };

packages/shared/src/server/queries/clickhouse-sql/clickhouse-filter.ts

@@ -432,19 +432,23 @@ export class FilterList {
     this.filters.push(...filter);
   }
 
-  find(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  find(predicate: (filter: Filter) => boolean) {
     return this.filters.find(predicate);
   }
 
-  filter(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  filter(predicate: (filter: Filter) => boolean) {
     return new FilterList(this.filters.filter(predicate));
   }
 
-  some(predicate: (filter: Filter) => boolean) { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  some(predicate: (filter: Filter) => boolean) {
     return this.filters.some(predicate);
   }
 
-  forEach(callback: (filter: Filter) => void) { // eslint-disable-line no-unused-vars
+  // eslint-disable-next-line no-unused-vars
+  forEach(callback: (filter: Filter) => void) {
     this.filters.forEach(callback);
   }
 

packages/shared/src/server/queries/clickhouse-sql/search.ts

@@ -6,6 +6,7 @@ export const clickhouseSearchCondition = (
   query?: string,
   searchType?: TracingSearchType[],
   tablePrefix?: string,
+  useTracesAmtCompatMode: boolean = false,
 ) => {
   const prefix = tablePrefix ? `${tablePrefix}.` : "";
 
@@ -13,9 +14,11 @@ export const clickhouseSearchCondition = (
     !searchType || searchType.includes("id")
       ? `${prefix}id ILIKE {searchString: String} OR user_id ILIKE {searchString: String} OR ${prefix}name ILIKE {searchString: String}`
       : null,
-    searchType && searchType.includes("content")
+    searchType && searchType.includes("content") && !useTracesAmtCompatMode
       ? `${prefix}input ILIKE {searchString: String} OR ${prefix}output ILIKE {searchString: String}`
-      : null,
+      : searchType && searchType.includes("content") && useTracesAmtCompatMode
+        ? `finalizeAggregation(${prefix}input) ILIKE {searchString: String} OR finalizeAggregation(${prefix}output) ILIKE {searchString: String}`
+        : null,
   ].filter(Boolean);
 
   return {

packages/shared/src/server/queues.ts

@@ -6,6 +6,8 @@ import {
   BatchActionType,
 } from "../features/batchAction/types";
 import { BatchTableNames } from "../interfaces/tableNames";
+import { EventActionSchema } from "../domain";
+import { PromptDomainSchema } from "../domain/prompts";
 
 export const IngestionEvent = z.object({
   data: z.object({
@@ -38,6 +40,21 @@ export const ScoresQueueEventSchema = z.object({
   projectId: z.string(),
   scoreIds: z.array(z.string()),
 });
+export const DatasetQueueEventSchema = z.discriminatedUnion("deletionType", [
+  // Delete all run items for a specific dataset
+  z.object({
+    deletionType: z.literal("dataset"),
+    projectId: z.string(),
+    datasetId: z.string(),
+  }),
+  // Delete all run items for multiple dataset runs (also used for single run deletion)
+  z.object({
+    deletionType: z.literal("dataset-runs"),
+    projectId: z.string(),
+    datasetId: z.string(),
+    datasetRunIds: z.array(z.string()),
+  }),
+]);
 export const ProjectQueueEventSchema = z.object({
   projectId: z.string(),
   orgId: z.string(),
@@ -99,6 +116,15 @@ export const BatchActionProcessingEventSchema = z.discriminatedUnion(
       targetId: z.string().optional(),
       type: z.enum(BatchActionType),
     }),
+    z.object({
+      actionId: z.literal("session-add-to-annotation-queue"),
+      projectId: z.string(),
+      query: BatchActionQuerySchema,
+      tableName: z.enum(BatchTableNames),
+      cutoffCreatedAt: z.date(),
+      targetId: z.string().optional(),
+      type: z.enum(BatchActionType),
+    }),
     z.object({
       actionId: z.literal("eval-create"),
       targetObject: z.enum(["trace", "dataset"]),
@@ -129,6 +155,33 @@ export const DeadLetterRetryQueueEventSchema = z.object({
   timestamp: z.date(),
 });
 
+export const WebhookOutboundEnvelopeSchema = z.object({
+  prompt: PromptDomainSchema,
+  action: EventActionSchema,
+  type: z.literal("prompt-version"),
+});
+
+export const WebhookInputSchema = z.object({
+  projectId: z.string(),
+  automationId: z.string(),
+  executionId: z.string(),
+  payload: WebhookOutboundEnvelopeSchema,
+});
+
+export const EntityChangeEventSchema = z.discriminatedUnion("entityType", [
+  z.object({
+    entityType: z.literal("prompt-version"),
+    projectId: z.string(),
+    promptId: z.string(),
+    action: EventActionSchema,
+    prompt: PromptDomainSchema,
+  }),
+  // Add other entity types here in the future
+]);
+
+export type WebhookInput = z.infer<typeof WebhookInputSchema>;
+export type EntityChangeEventType = z.infer<typeof EntityChangeEventSchema>;
+
 export type CreateEvalQueueEventType = z.infer<
   typeof CreateEvalQueueEventSchema
 >;
@@ -136,6 +189,7 @@ export type BatchExportJobType = z.infer<typeof BatchExportJobSchema>;
 export type TraceQueueEventType = z.infer<typeof TraceQueueEventSchema>;
 export type TracesQueueEventType = z.infer<typeof TracesQueueEventSchema>;
 export type ScoresQueueEventType = z.infer<typeof ScoresQueueEventSchema>;
+export type DatasetQueueEventType = z.infer<typeof DatasetQueueEventSchema>;
 export type ProjectQueueEventType = z.infer<typeof ProjectQueueEventSchema>;
 export type DatasetRunItemUpsertEventType = z.infer<
   typeof DatasetRunItemUpsertEventSchema
@@ -161,6 +215,15 @@ export type DeadLetterRetryQueueEventType = z.infer<
   typeof DeadLetterRetryQueueEventSchema
 >;
 
+export type WebhookQueueEventType = z.infer<typeof WebhookInputSchema>;
+
+export const RetryBaggage = z.object({
+  originalJobTimestamp: z.date(),
+  attempt: z.number(),
+});
+
+export type RetryBaggage = z.infer<typeof RetryBaggage>;
+
 export enum QueueName {
   TraceUpsert = "trace-upsert", // Ingestion pipeline adds events on each Trace upsert
   TraceDelete = "trace-delete",
@@ -183,7 +246,10 @@ export enum QueueName {
   BatchActionQueue = "batch-action-queue",
   CreateEvalQueue = "create-eval-queue",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-queue",
   DeadLetterRetryQueue = "dead-letter-retry-queue",
+  WebhookQueue = "webhook-queue",
+  EntityChangeQueue = "entity-change-queue",
 }
 
 export enum QueueJobs {
@@ -208,7 +274,10 @@ export enum QueueJobs {
   BatchActionProcessingJob = "batch-action-processing-job",
   CreateEvalJob = "create-eval-job",
   ScoreDelete = "score-delete",
+  DatasetDelete = "dataset-delete-job",
   DeadLetterRetryJob = "dead-letter-retry-job",
+  WebhookJob = "webhook-job",
+  EntityChangeJob = "entity-change-job",
 }
 
 export type TQueueJobTypes = {
@@ -230,6 +299,12 @@ export type TQueueJobTypes = {
     payload: ScoresQueueEventType;
     name: QueueJobs.ScoreDelete;
   };
+  [QueueName.DatasetDelete]: {
+    timestamp: Date;
+    id: string;
+    payload: DatasetQueueEventType;
+    name: QueueJobs.DatasetDelete;
+  };
   [QueueName.ProjectDelete]: {
     timestamp: Date;
     id: string;
@@ -247,6 +322,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: EvalExecutionEventType;
     name: QueueJobs.EvaluationExecution;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.BatchExport]: {
     timestamp: Date;
@@ -271,6 +347,7 @@ export type TQueueJobTypes = {
     id: string;
     payload: ExperimentCreateEventType;
     name: QueueJobs.ExperimentCreateJob;
+    retryBaggage?: RetryBaggage;
   };
   [QueueName.PostHogIntegrationProcessingQueue]: {
     timestamp: Date;
@@ -308,4 +385,16 @@ export type TQueueJobTypes = {
     payload: DeadLetterRetryQueueEventType;
     name: QueueJobs.DeadLetterRetryJob;
   };
+  [QueueName.WebhookQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: WebhookInput;
+    name: QueueJobs.WebhookJob;
+  };
+  [QueueName.EntityChangeQueue]: {
+    timestamp: Date;
+    id: string;
+    payload: EntityChangeEventType;
+    name: QueueJobs.EntityChangeJob;
+  };
 };

packages/shared/src/server/redis/batchExport.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName, TQueueJobTypes } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class BatchExportQueue {

packages/shared/src/server/redis/blobStorageIntegrationProcessingQueue.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class BlobStorageIntegrationProcessingQueue {
@@ -18,8 +22,10 @@ export class BlobStorageIntegrationProcessingQueue {
 
     BlobStorageIntegrationProcessingQueue.instance = newRedis
       ? new Queue(QueueName.BlobStorageIntegrationProcessingQueue, {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.BlobStorageIntegrationProcessingQueue),
+          connection: newRedis,
+          prefix: getQueuePrefix(
+            QueueName.BlobStorageIntegrationProcessingQueue,
+          ),
           defaultJobOptions: {
             removeOnComplete: true,
             removeOnFail: 100_000,
@@ -38,4 +44,4 @@ export class BlobStorageIntegrationProcessingQueue {
 
     return BlobStorageIntegrationProcessingQueue.instance;
   }
-} 
+}

packages/shared/src/server/redis/cloudUsageMeteringQueue.ts

@@ -1,7 +1,11 @@
 import { Queue } from "bullmq";
 import { env } from "../../env";
 import { QueueName, QueueJobs } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class CloudUsageMeteringQueue {

packages/shared/src/server/redis/coreDataS3ExportQueue.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName, QueueJobs } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 import { env } from "../../env";
 
@@ -23,8 +27,8 @@ export class CoreDataS3ExportQueue {
 
     CoreDataS3ExportQueue.instance = newRedis
       ? new Queue(QueueName.CoreDataS3ExportQueue, {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.CoreDataS3ExportQueue),
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.CoreDataS3ExportQueue),
           defaultJobOptions: {
             removeOnComplete: true,
             removeOnFail: 100,

packages/shared/src/server/redis/createEvalQueue.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class CreateEvalQueue {

packages/shared/src/server/redis/dataRetentionProcessingQueue.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class DataRetentionProcessingQueue {
@@ -18,8 +22,8 @@ export class DataRetentionProcessingQueue {
 
     DataRetentionProcessingQueue.instance = newRedis
       ? new Queue(QueueName.DataRetentionProcessingQueue, {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.DataRetentionProcessingQueue),
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.DataRetentionProcessingQueue),
           defaultJobOptions: {
             removeOnComplete: true,
             removeOnFail: 10000,

packages/shared/src/server/redis/dataRetentionQueue.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName, QueueJobs } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class DataRetentionQueue {
@@ -18,8 +22,8 @@ export class DataRetentionQueue {
 
     DataRetentionQueue.instance = newRedis
       ? new Queue(QueueName.DataRetentionQueue, {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.DataRetentionQueue),
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.DataRetentionQueue),
           defaultJobOptions: {
             removeOnComplete: true,
             removeOnFail: 100,

packages/shared/src/server/redis/datasetDelete.ts

@@ -0,0 +1,50 @@
+import { QueueName, TQueueJobTypes } from "../queues";
+import { Queue } from "bullmq";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
+import { logger } from "../logger";
+
+export class DatasetDeleteQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.DatasetDelete]
+  > | null {
+    if (DatasetDeleteQueue.instance) return DatasetDeleteQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    DatasetDeleteQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.DatasetDelete]>(
+          QueueName.DatasetDelete,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.DatasetDelete),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 2,
+              backoff: {
+                type: "exponential",
+                delay: 30_000,
+              },
+            },
+          },
+        )
+      : null;
+
+    DatasetDeleteQueue.instance?.on("error", (err) => {
+      logger.error("DatasetDeleteQueue error", err);
+    });
+
+    return DatasetDeleteQueue.instance;
+  }
+}

packages/shared/src/server/redis/datasetRunItemUpsert.ts

@@ -1,6 +1,10 @@
 import { QueueName, TQueueJobTypes } from "../queues";
 import { Queue } from "bullmq";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class DatasetRunItemUpsertQueue {

packages/shared/src/server/redis/dlqRetryQueue.ts

@@ -1,6 +1,10 @@
 import { Queue } from "bullmq";
 import { QueueName, QueueJobs } from "../queues";
-import { createNewRedisInstance, redisQueueRetryOptions, getQueuePrefix } from "./redis";
+import {
+  createNewRedisInstance,
+  redisQueueRetryOptions,
+  getQueuePrefix,
+} from "./redis";
 import { logger } from "../logger";
 
 export class DeadLetterRetryQueue {
@@ -18,8 +22,8 @@ export class DeadLetterRetryQueue {
 
     DeadLetterRetryQueue.instance = newRedis
       ? new Queue(QueueName.DeadLetterRetryQueue, {
-            connection: newRedis,
-            prefix: getQueuePrefix(QueueName.DeadLetterRetryQueue),
+          connection: newRedis,
+          prefix: getQueuePrefix(QueueName.DeadLetterRetryQueue),
           defaultJobOptions: {
             removeOnComplete: true,
             removeOnFail: 100,

packages/shared/src/server/redis/entityChangeQueue.ts

@@ -0,0 +1,50 @@
+import { Queue } from "bullmq";
+import { QueueName, TQueueJobTypes } from "../queues";
+import {
+  createNewRedisInstance,
+  getQueuePrefix,
+  redisQueueRetryOptions,
+} from "./redis";
+import { logger } from "../logger";
+
+export class EntityChangeQueue {
+  private static instance: Queue<
+    TQueueJobTypes[QueueName.EntityChangeQueue]
+  > | null = null;
+
+  public static getInstance(): Queue<
+    TQueueJobTypes[QueueName.EntityChangeQueue]
+  > | null {
+    if (EntityChangeQueue.instance) return EntityChangeQueue.instance;
+
+    const newRedis = createNewRedisInstance({
+      enableOfflineQueue: false,
+      ...redisQueueRetryOptions,
+    });
+
+    EntityChangeQueue.instance = newRedis
+      ? new Queue<TQueueJobTypes[QueueName.EntityChangeQueue]>(
+          QueueName.EntityChangeQueue,
+          {
+            connection: newRedis,
+            prefix: getQueuePrefix(QueueName.EntityChangeQueue),
+            defaultJobOptions: {
+              removeOnComplete: true,
+              removeOnFail: 100_000,
+              attempts: 5,
+              backoff: {
+                type: "exponential",
+                delay: 5000,
+              },
+            },
+          },
+        )
+      : null;
+
+    EntityChangeQueue.instance?.on("error", (err) => {
+      logger.error("EntityChangeQueue error", err);
+    });
+
+    return EntityChangeQueue.instance;
+  }
+}